Fix(parser): keep real line breaks when merging HTML fragments and PDF boxes (#17856)

Net effect: inline prose stays on one line (`Hello World`), real `<br>` boundaries survive (including before tags and repeated breaks), and source formatting whitespace no longer over-splits.

internal/parser/parser/html_parser.go

@@ -110,7 +110,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
 	for child := root.FirstChild; child != nil; child = child.NextSibling {
 		if child.Type == html.TextNode {
 			if emitsLooseHTMLText(root) {
-				appendHTMLTextItem(out, child.Data, "text")
+				appendHTMLTextItem(out, child.Data, "text", true)
 			}
 			continue
 		}
@@ -131,7 +131,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
 			continue
 		}
 		text := htmlLeafText(child)
-		appendHTMLTextItem(out, text, htmlTagToCkType(tag))
+		appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")
 	}
 }
 
@@ -139,8 +139,10 @@ func emitsLooseHTMLText(root *html.Node) bool {
 	return root.Type == html.ElementNode && root.Data == "body"
 }
 
-func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
-	text = strings.TrimSpace(text)
+func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {
+	if trim {
+		text = strings.TrimSpace(text)
+	}
 	if text == "" {
 		return
 	}
@@ -174,38 +176,116 @@ func htmlTagToCkType(tag string) string {
 	return "text"
 }
 
+// leafWriter accumulates the visible text of an HTML subtree while applying
+// CSS whitespace folding (the default white-space: normal rules):
+//   - collapsible whitespace runs collapse to a single space;
+//   - leading/trailing whitespace of a line is dropped;
+//   - a <br> forces a hard line break (and resets the leading-whitespace state);
+//   - <pre>/<textarea> are emitted verbatim (no folding, no injected breaks).
+type leafWriter struct {
+	b         *bytes.Buffer
+	lastSpace bool // last written rune was a collapsed single space
+	lineStart bool // at the start of a line, so leading whitespace is dropped
+	endsNL    bool // builder currently ends with a hard line break
+	pre       bool // inside <pre>/<textarea>: emit verbatim
+}
+
+func isCollapsibleWS(r rune) bool {
+	return r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f'
+}
+
+// writeText appends s, folding collapsible whitespace unless in pre mode.
+func (w *leafWriter) writeText(s string) {
+	if w.pre {
+		for _, r := range s {
+			w.b.WriteRune(r)
+			w.endsNL = r == '\n'
+		}
+		w.lastSpace = false
+		w.lineStart = false
+		return
+	}
+	for _, r := range s {
+		if isCollapsibleWS(r) {
+			if w.lineStart || w.lastSpace {
+				continue
+			}
+			w.b.WriteRune(' ')
+			w.lastSpace = true
+			w.lineStart = false
+			w.endsNL = false
+			continue
+		}
+		w.b.WriteRune(r)
+		w.lastSpace = false
+		w.lineStart = false
+		w.endsNL = false
+	}
+}
+
+// hardBreak inserts a forced line break (a <br> or block boundary). Per CSS,
+// whitespace immediately before a break is dropped (so "Hello <br>" yields
+// "Hello\n", not "Hello \n"). Inside <pre>/<textarea> whitespace is preserved,
+// so the preceding space is kept.
+func (w *leafWriter) hardBreak() {
+	if !w.pre && w.lastSpace && w.b.Len() > 0 {
+		w.b.Truncate(w.b.Len() - 1)
+	}
+	w.b.WriteByte('\n')
+	w.lastSpace = false
+	w.lineStart = true
+	w.endsNL = true
+}
+
 // htmlLeafText joins the visible text of an HTML node and its
-// descendants. <script>/<style>/<noscript> subtrees are skipped.
-// The output preserves whitespace runs so headings like
-// "<h1>Hello   world</h1>" round-trip with their spacing intact.
+// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace
+// is folded per CSS rules (so "<h1>Hello   world</h1>" becomes "Hello world"
+// and "<br>" survives as a real line break), while <pre>/<textarea> keep
+// their source formatting verbatim.
 func htmlLeafText(n *html.Node) string {
-	var b strings.Builder
-	walkHTMLLeaf(n, &b)
+	var b bytes.Buffer
+	w := &leafWriter{b: &b}
+	walkHTMLLeaf(n, w)
 	return b.String()
 }
 
-func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
+func walkHTMLLeaf(n *html.Node, w *leafWriter) {
 	switch n.Type {
 	case html.TextNode:
-		b.WriteString(n.Data)
+		w.writeText(n.Data)
 	case html.ElementNode:
 		if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
 			return
 		}
-		// Add a line break between block children so headings,
-		// paragraphs, and list items don't run together.
-		switch n.Data {
-		case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
-			"tr", "blockquote":
-			if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
-				b.WriteString("\n")
+		if n.Data == "br" {
+			w.hardBreak()
+			return
+		}
+		if n.Data == "pre" || n.Data == "textarea" {
+			// Verbatim: no folding, no injected block breaks.
+			w.pre = true
+			for child := n.FirstChild; child != nil; child = child.NextSibling {
+				walkHTMLLeaf(child, w)
+			}
+			w.pre = false
+			return
+		}
+		// Add a line break between block children so headings, paragraphs,
+		// and list items don't run together.
+		if !w.pre {
+			switch n.Data {
+			case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
+				"tr", "blockquote":
+				if w.b.Len() > 0 && !w.endsNL {
+					w.hardBreak()
+				}
 			}
 		}
 		for child := n.FirstChild; child != nil; child = child.NextSibling {
-			walkHTMLLeaf(child, b)
+			walkHTMLLeaf(child, w)
 		}
-		if isBlockTag(n.Data) && b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
-			b.WriteString("\n")
+		if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
+			w.hardBreak()
 		}
 	}
 }

internal/parser/parser/html_parser_unified_test.go

@@ -0,0 +1,114 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	_ "embed"
+	"encoding/json"
+	"testing"
+)
+
+// unifiedHTMLCasesJSON is the single source of truth for the browser-faithful
+// HTML parsing semantics that BOTH the Go and Python parsers must converge on.
+// It is embedded from testdata/unified_html_cases.json, which the Python mirror
+// (test/unit_test/deepdoc/parser/test_html_parser.py) also loads — so the two
+// engines share one fixture and can no longer drift.
+//
+// Each case wraps its content in a single block element so that
+// ParseWithResult emits exactly one item and the Python merge_block_text emits
+// exactly one block string, enabling a 1:1 byte comparison between engines.
+//
+//go:embed testdata/unified_html_cases.json
+var unifiedHTMLCasesJSON []byte
+
+type unifiedHTMLCase struct {
+	Name string `json:"name"`
+	HTML string `json:"html"`
+	Want string `json:"want"`
+}
+
+func loadUnifiedHTMLCases(t *testing.T) []unifiedHTMLCase {
+	t.Helper()
+	var cases []unifiedHTMLCase
+	if err := json.Unmarshal(unifiedHTMLCasesJSON, &cases); err != nil {
+		t.Fatalf("unmarshal unified html cases: %v", err)
+	}
+	return cases
+}
+
+// TestHTMLParser_ParseWithResult_UnifiedSemantics asserts the browser-faithful
+// semantics on the Go engine. The cases are loaded from the shared embedded
+// fixture, so this test and its Python mirror stay in lockstep.
+func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {
+	for _, tc := range loadUnifiedHTMLCases(t) {
+		t.Run(tc.Name, func(t *testing.T) {
+			p := NewHTMLParser()
+			res := p.ParseWithResult(t.Context(), "doc.html", []byte(tc.HTML))
+			if res.Err != nil {
+				t.Fatalf("ParseWithResult: %v", res.Err)
+			}
+			if len(res.JSON) != 1 {
+				t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)
+			}
+			if got := res.JSON[0]["text"].(string); got != tc.Want {
+				t.Errorf("got %q, want %q", got, tc.Want)
+			}
+		})
+	}
+}
+
+// TestHTMLParser_ParseWithResult_RealisticSmoke exercises the Go HTML walker on
+// a realistic multi-block document: a heading, a paragraph with an inline
+// <b> and a <br> line break, a CJK paragraph with an inline element, and a
+// verbatim <pre> block. It guards the leafWriter CSS-folding rewrite against
+// hidden regressions specific to the Go reimplementation:
+//   - <br> becomes a hard line break;
+//   - inline boundaries join verbatim, with NO inserted space even for CJK;
+//   - block-internal whitespace collapses to a single space and is trimmed;
+//   - <pre> keeps its source whitespace verbatim (leading/trailing included).
+func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {
+	const html = `<h1>产品说明 Product Guide</h1>
+<p>第一步：打开应用<br>第二步：点击<b>设置</b>按钮完成配置。</p>
+<p>欢迎使用我们的<b>智能助手</b>，它能帮您快速处理任务。</p>
+<pre>  code
+  block</pre>`
+	p := NewHTMLParser()
+	res := p.ParseWithResult(t.Context(), "doc.html", []byte(html))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	var texts []string
+	for _, item := range res.JSON {
+		texts = append(texts, item["text"].(string))
+	}
+	want := []string{
+		// Heading: whitespace folded, no injected break.
+		"产品说明 Product Guide",
+		// <br> => hard break; inline <b> joined verbatim (无空格).
+		"第一步：打开应用\n第二步：点击设置按钮完成配置。",
+		// CJK inline joined verbatim: 我们的 + 智能助手, no space.
+		"欢迎使用我们的智能助手，它能帮您快速处理任务。",
+		// <pre> preserved verbatim, leading/trailing whitespace intact.
+		"  code\n  block",
+	}
+	if len(texts) != len(want) {
+		t.Fatalf("block count = %d, want %d; got %#v", len(texts), len(want), texts)
+	}
+	for i := range want {
+		if texts[i] != want[i] {
+			t.Errorf("block %d: got %q, want %q", i, texts[i], want[i])
+		}
+	}
+}

internal/parser/parser/testdata/unified_html_cases.json

@@ -0,0 +1,12 @@
+[
+  {"name": "br_basic", "html": "<p>line1<br>line2</p>", "want": "line1\nline2"},
+  {"name": "br_surrounding_space", "html": "<p>Hello <br> World</p>", "want": "Hello\nWorld"},
+  {"name": "br_double", "html": "<p>A<br><br>B</p>", "want": "A\n\nB"},
+  {"name": "br_before_inline", "html": "<p>Line1<br><span>Line2</span></p>", "want": "Line1\nLine2"},
+  {"name": "inline_no_space_latin", "html": "<p>Hello<b>World</b></p>", "want": "HelloWorld"},
+  {"name": "inline_no_space_cjk", "html": "<p>你好<b>世界</b></p>", "want": "你好世界"},
+  {"name": "inline_with_space", "html": "<p>Hello <b>World</b></p>", "want": "Hello World"},
+  {"name": "inline_three", "html": "<p>First<b>Second</b>Third</p>", "want": "FirstSecondThird"},
+  {"name": "whitespace_collapse", "html": "<p>\n  Hello\n  <b>World</b>\n</p>", "want": "Hello World"},
+  {"name": "pre_preserved", "html": "<pre>  code\n  block</pre>", "want": "  code\n  block"}
+]