zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-13 04:13:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(parser): preserve loose body text in HTMLParser (#17290)

Browse Source
This commit is contained in:
WOLIKIMCHENG
2026-07-24 16:10:10 +08:00
committed by GitHub
parent bd355deaa9
commit 008fa3e10e
2 changed files with 124 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
internal/parser/parser/html_parser.go
View File

@@ -103,11 +103,17 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
// walkHTMLBlocks emits one normalized item per block-level
// descendant of root. Inline elements (b, i, a, span, …) are
// collapsed into the parent's text via leafText. <script> and
// <style> blocks are skipped entirely so they don't pollute the
// downstream chunker input.
// collapsed into the parent's text via leafText. <script>,
// <style>, and <noscript> blocks are skipped entirely so they
// don't pollute the downstream chunker input.
func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
for child := root.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
if emitsLooseHTMLText(root) {
appendHTMLTextItem(out, child.Data, "text")
}
continue
}
if child.Type != html.ElementNode {
continue
}
@@ -116,23 +122,35 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
case "script", "style", "noscript":
// Skip executable / stylistic blocks entirely.
continue
case "html", "head", "body":
case "head":
// Skip document metadata so it does not pollute body text.
continue
case "html", "body":
// Wrapper elements: descend into their children.
walkHTMLBlocks(child, out)
continue
}
text := htmlLeafText(child)
if strings.TrimSpace(text) == "" {
continue
}
*out = append(*out, map[string]any{
"text": strings.TrimSpace(text),
"doc_type_kwd": "text",
"ck_type": htmlTagToCkType(tag),
})
appendHTMLTextItem(out, text, htmlTagToCkType(tag))
}
}
func emitsLooseHTMLText(root *html.Node) bool {
return root.Type == html.ElementNode && root.Data == "body"
}
func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
text = strings.TrimSpace(text)
if text == "" {
return
}
*out = append(*out, map[string]any{
"text": text,
"doc_type_kwd": "text",
"ck_type": ckType,
})
}
// htmlTagToCkType maps HTML block tags to the python `ck_type`
// vocabulary used downstream by TitleChunker and similar
// components. Tags not in the map fall back to "text".
@@ -157,10 +175,9 @@ func htmlTagToCkType(tag string) string {
}
// htmlLeafText joins the visible text of an HTML node and its
// descendants. <script>/<style> subtrees are skipped (mirrors
// the python html.parser behaviour). The output preserves
// whitespace runs so headings like "<h1>Hello world</h1>"
// round-trip with their spacing intact.
// descendants. <script>/<style>/<noscript> subtrees are skipped.
// The output preserves whitespace runs so headings like
// "<h1>Hello world</h1>" round-trip with their spacing intact.
func htmlLeafText(n *html.Node) string {
var b strings.Builder
walkHTMLLeaf(n, &b)
@@ -172,7 +189,7 @@ func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
case html.TextNode:
b.WriteString(n.Data)
case html.ElementNode:
if n.Data == "script" || n.Data == "style" {
if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
return
}
// Add a line break between block children so headings,