zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-24 17:36:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(parser): preserve loose body text in HTMLParser (#17290)

Browse Source
This commit is contained in:
WOLIKIMCHENG
2026-07-24 16:10:10 +08:00
committed by GitHub
parent bd355deaa9
commit 008fa3e10e
2 changed files with 124 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
internal/parser/parser/html_parser.go
View File

@@ -103,11 +103,17 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
// walkHTMLBlocks emits one normalized item per block-level
// descendant of root. Inline elements (b, i, a, span, …) are
// collapsed into the parent's text via leafText. <script> and
// <style> blocks are skipped entirely so they don't pollute the
// downstream chunker input.
// collapsed into the parent's text via leafText. <script>,
// <style>, and <noscript> blocks are skipped entirely so they
// don't pollute the downstream chunker input.
func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
for child := root.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
if emitsLooseHTMLText(root) {
appendHTMLTextItem(out, child.Data, "text")
}
continue
}
if child.Type != html.ElementNode {
continue
}
@@ -116,23 +122,35 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
case "script", "style", "noscript":
// Skip executable / stylistic blocks entirely.
continue
case "html", "head", "body":
case "head":
// Skip document metadata so it does not pollute body text.
continue
case "html", "body":
// Wrapper elements: descend into their children.
walkHTMLBlocks(child, out)
continue
}
text := htmlLeafText(child)
if strings.TrimSpace(text) == "" {
continue
}
*out = append(*out, map[string]any{
"text": strings.TrimSpace(text),
"doc_type_kwd": "text",
"ck_type": htmlTagToCkType(tag),
})
appendHTMLTextItem(out, text, htmlTagToCkType(tag))
}
}
func emitsLooseHTMLText(root *html.Node) bool {
return root.Type == html.ElementNode && root.Data == "body"
}
func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
text = strings.TrimSpace(text)
if text == "" {
return
}
*out = append(*out, map[string]any{
"text": text,
"doc_type_kwd": "text",
"ck_type": ckType,
})
}
// htmlTagToCkType maps HTML block tags to the python `ck_type`
// vocabulary used downstream by TitleChunker and similar
// components. Tags not in the map fall back to "text".
@@ -157,10 +175,9 @@ func htmlTagToCkType(tag string) string {
}
// htmlLeafText joins the visible text of an HTML node and its
// descendants. <script>/<style> subtrees are skipped (mirrors
// the python html.parser behaviour). The output preserves
// whitespace runs so headings like "<h1>Hello world</h1>"
// round-trip with their spacing intact.
// descendants. <script>/<style>/<noscript> subtrees are skipped.
// The output preserves whitespace runs so headings like
// "<h1>Hello world</h1>" round-trip with their spacing intact.
func htmlLeafText(n *html.Node) string {
var b strings.Builder
walkHTMLLeaf(n, &b)
@@ -172,7 +189,7 @@ func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
case html.TextNode:
b.WriteString(n.Data)
case html.ElementNode:
if n.Data == "script" || n.Data == "style" {
if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
return
}
// Add a line break between block children so headings,

95
internal/parser/parser/parse_with_result_test.go
View File

@@ -36,6 +36,8 @@ import (
"strings"
"testing"
"golang.org/x/net/html"
"ragflow/internal/utility"
)
@@ -158,9 +160,87 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
}
}
func TestHTMLParser_ParseWithResult_PreservesLooseText(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
src := []byte(`<!DOCTYPE html><html><head>
<title>Head metadata</title>
</head><body>
Intro text
<h1>Title</h1>
Between blocks
<p>Body <span>inline</span>.<noscript>Inline fallback</noscript></p>
<script>alert("x")</script>
<style>body { color: red; }</style>
<noscript>Fallback text</noscript>
Tail text
</body></html>`)
res := p.ParseWithResult(ctx, "doc.html", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
want := []struct {
text string
ckType string
}{
{"Intro text", "text"},
{"Title", "heading"},
{"Between blocks", "text"},
{"Body inline.", "paragraph"},
{"Tail text", "text"},
}
if len(res.JSON) != len(want) {
t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, w := range want {
if got := res.JSON[i]["text"]; got != w.text {
t.Errorf("JSON[%d].text = %v, want %v", i, got, w.text)
}
if got := res.JSON[i]["doc_type_kwd"]; got != "text" {
t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)
}
if got := res.JSON[i]["ck_type"]; got != w.ckType {
t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)
}
}
}
func TestHTMLParser_ParseWithResult_PreservesLooseTextWithoutExplicitBody(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
src := []byte(`<!DOCTYPE html>
Intro text
<h1>Title</h1>
Tail text`)
res := p.ParseWithResult(ctx, "doc.html", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
want := []string{"Intro text", "Title", "Tail text"}
if len(res.JSON) != len(want) {
t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, text := range want {
if got := res.JSON[i]["text"]; got != text {
t.Errorf("JSON[%d].text = %v, want %v", i, got, text)
}
}
}
func TestWalkHTMLBlocks_SkipsHeadLooseText(t *testing.T) {
head := &html.Node{Type: html.ElementNode, Data: "head"}
head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})
var items []map[string]any
walkHTMLBlocks(head, &items)
if len(items) != 0 {
t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)
}
}
// TestHTMLParser_ParseWithResult_SkipsScriptAndStyle pins the
// rule that <script> / <style> subtrees are skipped entirely so
// they don't pollute the downstream chunker input.
// rule that <script> / <style> / <noscript> subtrees are skipped
// entirely so they don't pollute the downstream chunker input.
func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
@@ -168,17 +248,22 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
<p>Visible.</p>
<script>alert("x")</script>
<style>body { color: red; }</style>
<p>Also <script>inline alert</script><style>.inline { color: blue; }</style><noscript>fallback</noscript> visible.</p>
<p>Also visible.</p>
</body></html>`)
res := p.ParseWithResult(ctx, "doc.html", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if len(res.JSON) != 2 {
t.Errorf("JSON len = %d, want 2 (script+style skipped)", len(res.JSON))
if len(res.JSON) != 3 {
t.Errorf("JSON len = %d, want 3 (script+style+noscript skipped)", len(res.JSON))
}
for _, it := range res.JSON {
if txt, _ := it["text"].(string); strings.Contains(txt, "alert") || strings.Contains(txt, "color") {
if txt, _ := it["text"].(string); strings.Contains(txt, "alert") ||
strings.Contains(txt, "color") ||
strings.Contains(txt, "inline alert") ||
strings.Contains(txt, "blue") ||
strings.Contains(txt, "fallback") {
t.Errorf("item text leaks script/style content: %q", txt)
}
}