first commit

pigo/internal/agenttool/htmlmarkdown.go

@@ -0,0 +1,48 @@
+// This file implements the HTML→Markdown reduction used by the webfetch tool
+// (US-012, #128). It wraps the JohannesKaufmann/html-to-markdown/v2 library: the
+// base plugin already strips head/script/style/link/meta/iframe/noscript/input,
+// and we additionally register the remaining page chrome (nav/footer/header/
+// aside/form/svg/template) for removal so only readable content survives. The
+// commonmark plugin renders headings, links, lists, code, emphasis, and tables.
+package agenttool
+
+import (
+	"sync"
+
+	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
+)
+
+// chromeElements are dropped whole (element + subtree) in addition to the base
+// plugin's defaults: they carry no readable body content for a text extraction.
+var chromeElements = []string{
+	"nav", "footer", "header", "aside", "svg", "form", "template",
+}
+
+// mdConverter is the shared, configured converter. It is built once — NewConverter
+// registers plugins and tag handlers, which is wasteful to repeat per call, and
+// the converter is safe for concurrent ConvertString use.
+var mdConverter = sync.OnceValue(func() *converter.Converter {
+	conv := converter.NewConverter(
+		converter.WithPlugins(
+			base.NewBasePlugin(),
+			commonmark.NewCommonmarkPlugin(),
+		),
+	)
+	for _, tag := range chromeElements {
+		conv.Register.TagType(tag, converter.TagTypeRemove, converter.PriorityStandard)
+	}
+	return conv
+})
+
+// htmlToMarkdown converts body (HTML) to a simplified Markdown string. On a
+// conversion error (rare — the parser is lenient) it falls back to the raw bytes
+// so the caller always gets usable text.
+func htmlToMarkdown(body []byte) string {
+	md, err := mdConverter().ConvertString(string(body))
+	if err != nil {
+		return string(body)
+	}
+	return md
+}