From 008fa3e10e1c1c555c28c1777e9abd3c18d9c78a Mon Sep 17 00:00:00 2001
From: WOLIKIMCHENG <35391914+WOLIKIMCHENG@users.noreply.github.com>
Date: Fri, 24 Jul 2026 16:10:10 +0800
Subject: [PATCH] fix(parser): preserve loose body text in HTMLParser (#17290)

---
 internal/parser/parser/html_parser.go         | 51 ++++++----
 .../parser/parser/parse_with_result_test.go   | 95 ++++++++++++++++++-
 2 files changed, 124 insertions(+), 22 deletions(-)

diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index a6c831bdc2..f613528a70 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -103,11 +103,17 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
 
 // walkHTMLBlocks emits one normalized item per block-level
 // descendant of root. Inline elements (b, i, a, span, …) are
-// collapsed into the parent's text via leafText. <script> and
-// <style> blocks are skipped entirely so they don't pollute the
-// downstream chunker input.
+// collapsed into the parent's text via leafText. <script>,
+// <style>, and <noscript> blocks are skipped entirely so they
+// don't pollute the downstream chunker input.
 func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
 	for child := root.FirstChild; child != nil; child = child.NextSibling {
+		if child.Type == html.TextNode {
+			if emitsLooseHTMLText(root) {
+				appendHTMLTextItem(out, child.Data, "text")
+			}
+			continue
+		}
 		if child.Type != html.ElementNode {
 			continue
 		}
@@ -116,23 +122,35 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
 		case "script", "style", "noscript":
 			// Skip executable / stylistic blocks entirely.
 			continue
-		case "html", "head", "body":
+		case "head":
+			// Skip document metadata so it does not pollute body text.
+			continue
+		case "html", "body":
 			// Wrapper elements: descend into their children.
 			walkHTMLBlocks(child, out)
 			continue
 		}
 		text := htmlLeafText(child)
-		if strings.TrimSpace(text) == "" {
-			continue
-		}
-		*out = append(*out, map[string]any{
-			"text":         strings.TrimSpace(text),
-			"doc_type_kwd": "text",
-			"ck_type":      htmlTagToCkType(tag),
-		})
+		appendHTMLTextItem(out, text, htmlTagToCkType(tag))
 	}
 }
 
+func emitsLooseHTMLText(root *html.Node) bool {
+	return root.Type == html.ElementNode && root.Data == "body"
+}
+
+func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return
+	}
+	*out = append(*out, map[string]any{
+		"text":         text,
+		"doc_type_kwd": "text",
+		"ck_type":      ckType,
+	})
+}
+
 // htmlTagToCkType maps HTML block tags to the python `ck_type`
 // vocabulary used downstream by TitleChunker and similar
 // components. Tags not in the map fall back to "text".
@@ -157,10 +175,9 @@ func htmlTagToCkType(tag string) string {
 }
 
 // htmlLeafText joins the visible text of an HTML node and its
-// descendants. <script>/<style> subtrees are skipped (mirrors
-// the python html.parser behaviour). The output preserves
-// whitespace runs so headings like "<h1>Hello   world</h1>"
-// round-trip with their spacing intact.
+// descendants. <script>/<style>/<noscript> subtrees are skipped.
+// The output preserves whitespace runs so headings like
+// "<h1>Hello   world</h1>" round-trip with their spacing intact.
 func htmlLeafText(n *html.Node) string {
 	var b strings.Builder
 	walkHTMLLeaf(n, &b)
@@ -172,7 +189,7 @@ func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
 	case html.TextNode:
 		b.WriteString(n.Data)
 	case html.ElementNode:
-		if n.Data == "script" || n.Data == "style" {
+		if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
 			return
 		}
 		// Add a line break between block children so headings,
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 2c523a6361..8ab285dd69 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -36,6 +36,8 @@ import (
 	"strings"
 	"testing"
 
+	"golang.org/x/net/html"
+
 	"ragflow/internal/utility"
 )
 
@@ -158,9 +160,87 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
 	}
 }
 
+func TestHTMLParser_ParseWithResult_PreservesLooseText(t *testing.T) {
+	ctx := t.Context()
+	p := NewHTMLParser()
+	src := []byte(`<!DOCTYPE html><html><head>
+<title>Head metadata</title>
+</head><body>
+Intro text
+<h1>Title</h1>
+Between blocks
+<p>Body <span>inline</span>.<noscript>Inline fallback</noscript></p>
+<script>alert("x")</script>
+<style>body { color: red; }</style>
+<noscript>Fallback text</noscript>
+Tail text
+</body></html>`)
+	res := p.ParseWithResult(ctx, "doc.html", src)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	want := []struct {
+		text   string
+		ckType string
+	}{
+		{"Intro text", "text"},
+		{"Title", "heading"},
+		{"Between blocks", "text"},
+		{"Body inline.", "paragraph"},
+		{"Tail text", "text"},
+	}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, w := range want {
+		if got := res.JSON[i]["text"]; got != w.text {
+			t.Errorf("JSON[%d].text = %v, want %v", i, got, w.text)
+		}
+		if got := res.JSON[i]["doc_type_kwd"]; got != "text" {
+			t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)
+		}
+		if got := res.JSON[i]["ck_type"]; got != w.ckType {
+			t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)
+		}
+	}
+}
+
+func TestHTMLParser_ParseWithResult_PreservesLooseTextWithoutExplicitBody(t *testing.T) {
+	ctx := t.Context()
+	p := NewHTMLParser()
+	src := []byte(`<!DOCTYPE html>
+Intro text
+<h1>Title</h1>
+Tail text`)
+	res := p.ParseWithResult(ctx, "doc.html", src)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	want := []string{"Intro text", "Title", "Tail text"}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, text := range want {
+		if got := res.JSON[i]["text"]; got != text {
+			t.Errorf("JSON[%d].text = %v, want %v", i, got, text)
+		}
+	}
+}
+
+func TestWalkHTMLBlocks_SkipsHeadLooseText(t *testing.T) {
+	head := &html.Node{Type: html.ElementNode, Data: "head"}
+	head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})
+
+	var items []map[string]any
+	walkHTMLBlocks(head, &items)
+	if len(items) != 0 {
+		t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)
+	}
+}
+
 // TestHTMLParser_ParseWithResult_SkipsScriptAndStyle pins the
-// rule that <script> / <style> subtrees are skipped entirely so
-// they don't pollute the downstream chunker input.
+// rule that <script> / <style> / <noscript> subtrees are skipped
+// entirely so they don't pollute the downstream chunker input.
 func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
 	ctx := t.Context()
 	p := NewHTMLParser()
@@ -168,17 +248,22 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
 <p>Visible.</p>
 <script>alert("x")</script>
 <style>body { color: red; }</style>
+<p>Also <script>inline alert</script><style>.inline { color: blue; }</style><noscript>fallback</noscript> visible.</p>
 <p>Also visible.</p>
 </body></html>`)
 	res := p.ParseWithResult(ctx, "doc.html", src)
 	if res.Err != nil {
 		t.Fatalf("ParseWithResult: %v", res.Err)
 	}
-	if len(res.JSON) != 2 {
-		t.Errorf("JSON len = %d, want 2 (script+style skipped)", len(res.JSON))
+	if len(res.JSON) != 3 {
+		t.Errorf("JSON len = %d, want 3 (script+style+noscript skipped)", len(res.JSON))
 	}
 	for _, it := range res.JSON {
-		if txt, _ := it["text"].(string); strings.Contains(txt, "alert") || strings.Contains(txt, "color") {
+		if txt, _ := it["text"].(string); strings.Contains(txt, "alert") ||
+			strings.Contains(txt, "color") ||
+			strings.Contains(txt, "inline alert") ||
+			strings.Contains(txt, "blue") ||
+			strings.Contains(txt, "fallback") {
 			t.Errorf("item text leaks script/style content: %q", txt)
 		}
 	}