49 lines
1.8 KiB
Go
49 lines
1.8 KiB
Go
// This file implements the HTML→Markdown reduction used by the webfetch tool
|
|
// (US-012, #128). It wraps the JohannesKaufmann/html-to-markdown/v2 library: the
|
|
// base plugin already strips head/script/style/link/meta/iframe/noscript/input,
|
|
// and we additionally register the remaining page chrome (nav/footer/header/
|
|
// aside/form/svg/template) for removal so only readable content survives. The
|
|
// commonmark plugin renders headings, links, lists, code, emphasis, and tables.
|
|
package agenttool
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
|
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
|
|
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
|
|
)
|
|
|
|
// chromeElements are dropped whole (element + subtree) in addition to the base
|
|
// plugin's defaults: they carry no readable body content for a text extraction.
|
|
var chromeElements = []string{
|
|
"nav", "footer", "header", "aside", "svg", "form", "template",
|
|
}
|
|
|
|
// mdConverter is the shared, configured converter. It is built once — NewConverter
|
|
// registers plugins and tag handlers, which is wasteful to repeat per call, and
|
|
// the converter is safe for concurrent ConvertString use.
|
|
var mdConverter = sync.OnceValue(func() *converter.Converter {
|
|
conv := converter.NewConverter(
|
|
converter.WithPlugins(
|
|
base.NewBasePlugin(),
|
|
commonmark.NewCommonmarkPlugin(),
|
|
),
|
|
)
|
|
for _, tag := range chromeElements {
|
|
conv.Register.TagType(tag, converter.TagTypeRemove, converter.PriorityStandard)
|
|
}
|
|
return conv
|
|
})
|
|
|
|
// htmlToMarkdown converts body (HTML) to a simplified Markdown string. On a
|
|
// conversion error (rare — the parser is lenient) it falls back to the raw bytes
|
|
// so the caller always gets usable text.
|
|
func htmlToMarkdown(body []byte) string {
|
|
md, err := mdConverter().ConvertString(string(body))
|
|
if err != nil {
|
|
return string(body)
|
|
}
|
|
return md
|
|
}
|