refactor(parser): preserve HTML <table> structure (inline + structured item) (#18129)

Stop flattening `<table>` into a single text blob. A `<table>` now emits:
1. an inlined `doc_type_kwd:"text"` item keeping the `<table>…</table>`
markup (row/column structure survives for embedding/retrieval/LLM rendering),
2. a structured `doc_type_kwd:"table"` / `ck_type:"table"` item appended
after the walk, consumed by the downstream chunker.

internal/parser/parser/html_parser.go

@@ -107,6 +107,10 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
 // <style>, and <noscript> blocks are skipped entirely so they
 // don't pollute the downstream chunker input.
 func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
+	// tableItems collects the structured doc_type_kwd:"table" items so they
+	// can be appended after the walk (mirrors markdown_parser.go:366-367,
+	// which appends tables after all sections to match Python's ordering).
+	var tableItems []map[string]any
 	for child := root.FirstChild; child != nil; child = child.NextSibling {
 		if child.Type == html.TextNode {
 			if emitsLooseHTMLText(root) {
@@ -129,10 +133,53 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
 			// Wrapper elements: descend into their children.
 			walkHTMLBlocks(child, out)
 			continue
+		case "table":
+			// Keep the table as its full HTML markup (NOT flattened) so
+			// row/column structure survives into embedding, retrieval, and
+			// LLM rendering. This mirrors the markdown table handling
+			// (markdown_parser.go:305-328) and Python's HtmlParser, which
+			// keeps the <table>…</table> string as the section text.
+			markup := renderTableHTML(child)
+			if strings.TrimSpace(markup) != "" {
+				// Inlined copy in document order (matches Python's <table>
+				// section text and markdown's text-flow copy).
+				*out = append(*out, map[string]any{
+					"text":         markup,
+					"doc_type_kwd": "text",
+				})
+				// Structured table item, appended after the walk.
+				tableItems = append(tableItems, map[string]any{
+					"text":         markup,
+					"doc_type_kwd": "table",
+					"ck_type":      "table",
+				})
+			}
+			continue
 		}
-		text := htmlLeafText(child)
+		text := htmlLeafText(child, &tableItems)
 		appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")
 	}
+	if len(tableItems) > 0 {
+		*out = append(*out, tableItems...)
+	}
+}
+
+// renderTableHTML serializes a <table> node back to its outer HTML markup
+// (tags preserved), mirroring Python's HtmlParser which keeps the full
+// <table>…</table> string as the section text. This preserves row/column
+// structure for embedding, retrieval, and LLM rendering, instead of
+// flattening cells into a single text blob. It is used both for top-level
+// tables (walkHTMLBlocks) and for tables reached via the leaf-text extractor
+// (walkHTMLLeaf, i.e. a <table> nested in a div/section/…). On any rendering
+// error it returns "" so callers skip the table rather than risk a render
+// loop through the leaf extractor — html.Render only fails on unsupported
+// node kinds, and a parsed <table> never triggers it.
+func renderTableHTML(n *html.Node) string {
+	var b bytes.Buffer
+	if err := html.Render(&b, n); err != nil {
+		return ""
+	}
+	return b.String()
 }
 
 func emitsLooseHTMLText(root *html.Node) bool {
@@ -241,15 +288,18 @@ func (w *leafWriter) hardBreak() {
 // descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace
 // is folded per CSS rules (so "<h1>Hello   world</h1>" becomes "Hello world"
 // and "<br>" survives as a real line break), while <pre>/<textarea> keep
-// their source formatting verbatim.
-func htmlLeafText(n *html.Node) string {
+// their source formatting verbatim. Any <table> encountered in the subtree is
+// rendered as its <table>…</table> markup (so row/column structure survives)
+// and registered as a structured doc_type_kwd:"table" item via tableItems —
+// see walkHTMLLeaf's "table" case.
+func htmlLeafText(n *html.Node, tableItems *[]map[string]any) string {
 	var b bytes.Buffer
 	w := &leafWriter{b: &b}
-	walkHTMLLeaf(n, w)
+	walkHTMLLeaf(n, w, tableItems)
 	return b.String()
 }
 
-func walkHTMLLeaf(n *html.Node, w *leafWriter) {
+func walkHTMLLeaf(n *html.Node, w *leafWriter, tableItems *[]map[string]any) {
 	switch n.Type {
 	case html.TextNode:
 		w.writeText(n.Data)
@@ -265,11 +315,31 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter) {
 			// Verbatim: no folding, no injected block breaks.
 			w.pre = true
 			for child := n.FirstChild; child != nil; child = child.NextSibling {
-				walkHTMLLeaf(child, w)
+				walkHTMLLeaf(child, w, tableItems)
 			}
 			w.pre = false
 			return
 		}
+		if n.Data == "table" {
+			// Keep the table as its full HTML markup (NOT flattened) so
+			// row/column structure survives into the wrapper's text item,
+			// embedding, retrieval, and LLM rendering. This mirrors the
+			// top-level walkHTMLBlocks "table" case, and covers tables nested
+			// in div/section/article/… (which the walkHTMLBlocks case never
+			// reaches). The structured table item is collected for the
+			// downstream chunker (appended after the walk by the caller, same
+			// as the top-level path).
+			markup := renderTableHTML(n)
+			if strings.TrimSpace(markup) != "" {
+				w.writeText(markup)
+				*tableItems = append(*tableItems, map[string]any{
+					"text":         markup,
+					"doc_type_kwd": "table",
+					"ck_type":      "table",
+				})
+			}
+			return
+		}
 		// Add a line break between block children so headings, paragraphs,
 		// and list items don't run together.
 		if !w.pre {
@@ -282,7 +352,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter) {
 			}
 		}
 		for child := n.FirstChild; child != nil; child = child.NextSibling {
-			walkHTMLLeaf(child, w)
+			walkHTMLLeaf(child, w, tableItems)
 		}
 		if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
 			w.hardBreak()

internal/parser/parser/html_parser_align_test.go

@@ -0,0 +1,380 @@
+package parser
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+)
+
+// TestHTMLParser_TableProducesStructuredItems asserts the HTML table
+// alignment fix (approach a, mirroring markdown_parser.go:305-328 and
+// :366-367): a <table> must NOT be flattened into a single text blob. The
+// parser must emit two items:
+//
+//  1. an inlined copy in the text flow (doc_type_kwd:"text") whose text keeps
+//     the <table> markup, so embedding/retrieval/LLM rendering preserve the
+//     row/column structure (no "collapse");
+//  2. a structured table item (doc_type_kwd:"table", ck_type:"table") appended
+//     after the walk, consumed by the downstream chunker (attachMediaContext).
+//
+// Before the fix, walkHTMLBlocks ran htmlLeafText over the <table>, producing
+// ONE flat item (e.g. "Name Age Alice 30") tagged ck_type:"table" but with no
+// markup — this test fails on that behavior.
+func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
+	const html = `<html><body>
+<h1>Employee Table</h1>
+<table>
+<tr><th>Name</th><th>Age</th></tr>
+<tr><td>Alice</td><td>30</td></tr>
+</table>
+<p>Trailing paragraph</p>
+</body></html>`
+
+	p := NewHTMLParser()
+	res := p.ParseWithResult(context.Background(), "doc.html", []byte(html))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+
+	var inlinedText, structuredText string
+	inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1
+	for i, it := range res.JSON {
+		text, _ := it["text"].(string)
+		switch it["doc_type_kwd"] {
+		case "text":
+			if strings.Contains(text, "<table") {
+				inlinedText = text
+				inlinedIdx = i
+			}
+			if text == "Trailing paragraph" {
+				trailingIdx = i
+			}
+		case "table":
+			structuredText = text
+			structuredIdx = i
+		}
+	}
+
+	if inlinedIdx < 0 {
+		t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup; got items: %#v", res.JSON)
+	}
+	if !strings.Contains(inlinedText, "<table") ||
+		!strings.Contains(inlinedText, "Name") ||
+		!strings.Contains(inlinedText, "Alice") {
+		t.Errorf("inlined table copy missing markup/cells: %q", inlinedText)
+	}
+
+	if structuredIdx < 0 {
+		t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)
+	}
+	if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {
+		t.Errorf("structured table item ck_type = %v, want %v", got, want)
+	}
+	if structuredText != inlinedText {
+		t.Errorf("structured table text differs from inlined copy:\n inlined=%q\n struct =%q", inlinedText, structuredText)
+	}
+
+	// The structured table item must be appended after the walk, i.e. after
+	// the in-walk text items (mirrors markdown_parser.go:366-367). The trailing
+	// <p> is an in-walk text item, so the structured item must come after it.
+	if trailingIdx < 0 {
+		t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)
+	}
+	if structuredIdx <= trailingIdx {
+		t.Errorf("structured table item at index %d must come after the trailing paragraph at index %d", structuredIdx, trailingIdx)
+	}
+}
+
+// TestHTMLParser_NestedTableProducesStructuredItems asserts the table fix also
+// covers tables that are NOT direct children of <body> — i.e. a <table> wrapped
+// in a layout container such as <div>/<section>/<article> (the common real-world
+// case). walkHTMLBlocks only special-cases a <table> that is a direct child of
+// the walk root; a nested <table> is reached via the default-branch leaf-text
+// extractor (htmlLeafText → walkHTMLLeaf). Before the fix, that path ran
+// htmlLeafText over the wrapper, flattening the table into a single text blob
+// ("Name Age Alice 30") with no markup — the same collapse as the top-level case,
+// just hidden one level deeper. After the fix, walkHTMLLeaf must render the
+// <table> markup inline (so it survives inside the wrapper's single text item)
+// AND register a structured doc_type_kwd:"table"/ck_type:"table" item (appended
+// after the walk like the top-level path).
+func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
+	const html = `<html><body>
+<h1>Heading</h1>
+<div class="content">
+<table>
+<tr><th>Name</th><th>Age</th></tr>
+<tr><td>Alice</td><td>30</td></tr>
+</table>
+</div>
+<p>Trailing paragraph</p>
+</body></html>`
+
+	p := NewHTMLParser()
+	res := p.ParseWithResult(context.Background(), "nested.html", []byte(html))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+
+	var inlinedText, structuredText string
+	inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1
+	for i, it := range res.JSON {
+		text, _ := it["text"].(string)
+		switch it["doc_type_kwd"] {
+		case "text":
+			if strings.Contains(text, "<table") {
+				inlinedText = text
+				inlinedIdx = i
+			}
+			if text == "Trailing paragraph" {
+				trailingIdx = i
+			}
+		case "table":
+			structuredText = text
+			structuredIdx = i
+		}
+	}
+
+	if inlinedIdx < 0 {
+		t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup (nested table flattened?); got items: %#v", res.JSON)
+	}
+	if !strings.Contains(inlinedText, "<table") ||
+		!strings.Contains(inlinedText, "Name") ||
+		!strings.Contains(inlinedText, "Alice") {
+		t.Errorf("inlined nested table copy missing markup/cells: %q", inlinedText)
+	}
+
+	if structuredIdx < 0 {
+		t.Fatalf("no structured doc_type_kwd:\"table\" item emitted for nested table; got items: %#v", res.JSON)
+	}
+	if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {
+		t.Errorf("structured nested table item ck_type = %v, want %v", got, want)
+	}
+	if !strings.Contains(structuredText, "<table") ||
+		!strings.Contains(structuredText, "Name") ||
+		!strings.Contains(structuredText, "Alice") {
+		t.Errorf("structured nested table item missing markup/cells: %q", structuredText)
+	}
+
+	// The structured table item must be appended after the walk, i.e. after
+	// every in-walk text item (mirrors the top-level path and markdown).
+	if trailingIdx < 0 {
+		t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)
+	}
+	if structuredIdx <= inlinedIdx || structuredIdx <= trailingIdx {
+		t.Errorf("structured nested table item at index %d must come after the inlined text item at %d and trailing paragraph at %d", structuredIdx, inlinedIdx, trailingIdx)
+	}
+}
+
+// TestHTMLParser_NestedListItemsPreserved verifies a nested <ul> (a list
+// containing a sublist) does NOT lose any item text — i.e. it is NOT subject to
+// the same kind of collapse the table path once had. Unlike tables (whose markup is
+// dropped and cells can fuse), list items are separated by hard breaks inside
+// walkHTMLLeaf, so every <li> text survives in document order. This mirrors
+// Python's RAGFlowHtmlParser: read_text_recursively (html_parser.py:108-160)
+// assigns each block its own block_id and emits flat, depth-less records for
+// nested <li>s — so BOTH sides preserve item content and NEITHER represents
+// nesting depth. The accepted divergence is granularity: Python emits one
+// record per <li> while Go emits the whole <ul> as a single text item (its
+// "one item per block-level element" model); the primary alignment gate is
+// content equivalence (PARSER_ALIGNMENT_HANDOFF.md §2.3), which this test
+// guards. A regression here would be Go dropping or fusing list items.
+func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {
+	const html = `<html><body>
+<ul>
+  <li>Item 1</li>
+  <li>Item 2
+    <ul>
+      <li>Sub A</li>
+      <li>Sub B</li>
+    </ul>
+  </li>
+</ul>
+</body></html>`
+
+	p := NewHTMLParser()
+	res := p.ParseWithResult(context.Background(), "nested-list.html", []byte(html))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+
+	// Join every item's text in order to check content + order + fusion.
+	var all string
+	for _, it := range res.JSON {
+		if txt, _ := it["text"].(string); txt != "" {
+			all += txt + "\n"
+		}
+	}
+
+	for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {
+		if !strings.Contains(all, want) {
+			t.Errorf("nested list lost item %q; joined output: %q", want, all)
+		}
+	}
+
+	// Document order must survive: the sublist items must follow their parent.
+	i2 := strings.Index(all, "Item 2")
+	ia := strings.Index(all, "Sub A")
+	ib := strings.Index(all, "Sub B")
+	if i2 < 0 || ia < 0 || ib < 0 || !(ia > i2 && ib > i2) {
+		t.Errorf("nested list order not preserved (parent must precede sub-items): %q", all)
+	}
+
+	// No collapse: items must not be fused without a separator
+	// (e.g. "Item 2" directly adjacent to "Sub A" with no break).
+	if strings.Contains(all, "Item 2Sub") || strings.Contains(all, "Sub ASub") {
+		t.Errorf("nested list items fused together (collapse): %q", all)
+	}
+
+	// The whole top-level <ul> is emitted as ONE text item tagged
+	// ck_type:"list" (Go's "one item per block-level element" model). Guard
+	// against a regression that would drop the list tag or split it
+	// unexpectedly. Note: a list nested inside a non-list container
+	// (div/section/…) is folded into that container's single text item and
+	// therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,
+	// not a collapse, since downstream chunking reads doc_type_kwd not ck_type.
+	var listText string
+	for _, it := range res.JSON {
+		if it["ck_type"] == "list" {
+			listText, _ = it["text"].(string)
+			break
+		}
+	}
+	if listText == "" {
+		t.Fatalf("top-level <ul> not tagged ck_type:\"list\"; got items: %#v", res.JSON)
+	}
+	for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {
+		if !strings.Contains(listText, want) {
+			t.Errorf("ck_type:\"list\" item missing %q: %q", want, listText)
+		}
+	}
+}
+
+// TestHTMLParser_PrePreservesVerbatim guards that <pre> (and <textarea>) blocks
+// are emitted verbatim — indentation and internal newlines must survive, NOT be
+// CSS-collapsed into single spaces. Code is high-value content: if the verbatim
+// path regressed to the folded leaf-text path, indented code would lose its
+// structure and the LLM / user would see broken code. This mirrors Python's
+// RAGFlowHtmlParser, which keeps pre/textarea verbatim (_PRE_TAGS in
+// deepdoc/parser/html_parser.py). Both blocks keep doc_type_kwd:"text"; <pre>
+// is tagged ck_type:"code" while <textarea> falls back to ck_type:"text"
+// (htmlTagToCkType has no textarea case) — a labeling nuance, not a content
+// divergence. Note: the single leading newline immediately after <pre>/<textarea>
+// is dropped by the HTML parser per spec (pre/textarea eat one leading newline);
+// that is correct browser behavior, not folding. What must survive is the
+// indentation and the internal newlines.
+func TestHTMLParser_PrePreservesVerbatim(t *testing.T) {
+	const html = `<html><body>
+<pre>
+def foo():
+    if x:
+        return 1
+    return 0
+</pre>
+<textarea>
+SELECT * FROM t
+  WHERE a = 1
+</textarea>
+</body></html>`
+
+	p := NewHTMLParser()
+	res := p.ParseWithResult(context.Background(), "pre.html", []byte(html))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+
+	var preItem, textareaItem map[string]any
+	for _, it := range res.JSON {
+		txt, _ := it["text"].(string)
+		switch {
+		case strings.Contains(txt, "def foo()"):
+			preItem = it
+		case strings.Contains(txt, "SELECT * FROM t"):
+			textareaItem = it
+		}
+	}
+
+	if preItem == nil {
+		t.Fatalf("no <pre> item found; got items: %#v", res.JSON)
+	}
+	if preItem["ck_type"] != "code" {
+		t.Errorf("<pre> item ck_type = %v, want code", preItem["ck_type"])
+	}
+	preText, _ := preItem["text"].(string)
+	// 4-space and 8-space indents must survive verbatim (a folded path would
+	// collapse them to a single space).
+	if !strings.Contains(preText, "    if x:") {
+		t.Errorf("<pre> lost 4-space indent (folded?): %q", preText)
+	}
+	if !strings.Contains(preText, "        return 1") {
+		t.Errorf("<pre> lost 8-space indent (folded?): %q", preText)
+	}
+	// Internal newlines must survive (line structure preserved).
+	if !strings.Contains(preText, "def foo():\n") {
+		t.Errorf("<pre> lost internal newline (folded?): %q", preText)
+	}
+
+	if textareaItem == nil {
+		t.Fatalf("no <textarea> item found; got items: %#v", res.JSON)
+	}
+	taText, _ := textareaItem["text"].(string)
+	// 2-space indent must survive verbatim.
+	if !strings.Contains(taText, "  WHERE a = 1") {
+		t.Errorf("<textarea> lost 2-space indent (folded?): %q", taText)
+	}
+}
+
+// TestHTMLParser_AlignmentGolden verifies Go's ParseWithResult output is
+// content-equivalent to Python's RAGFlowHtmlParser on the shared sample, using
+// the shared concatenation-normalization alignment tool (align_test.go).
+// Python keeps raw HTML and splits on block elements (deepdoc/parser/html_parser.py:
+// read_text_recursively + merge_block_text); Go emits clean per-block text.
+// The comparison normalizes both (HTML heading markers "#{1,6}", HTML tags,
+// whitespace collapsed) and ignores "table" items, which are accepted
+// representation differences (the <table> markup stays inline as a "text"
+// item in both, see TestHTMLParser_TableProducesStructuredItems). The
+// accepted divergences are declared in the golden's meta block, not hardcoded
+// here.
+//
+// The baseline lives in testdata/html.python.en.golden.json / testdata/html.python.zh.golden.json as {meta, items}
+// document (see its "meta" block for how to regenerate it from the Python
+// engine — no committed generator script).
+func TestHTMLParser_AlignmentGolden(t *testing.T) {
+	ctx := t.Context()
+	p := NewHTMLParser()
+
+	cases := []struct {
+		name   string
+		sample string
+		golden string
+	}{
+		{"en", "testdata/html.sample.en.html", "testdata/html.python.en.golden.json"},
+		{"zh", "testdata/html.sample.zh.html", "testdata/html.python.zh.golden.json"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			sample, err := os.ReadFile(tc.sample)
+			if err != nil {
+				t.Fatalf("read sample: %v", err)
+			}
+			res := p.ParseWithResult(ctx, tc.sample, sample)
+			if res.Err != nil {
+				t.Fatalf("ParseWithResult: %v", res.Err)
+			}
+
+			doc := LoadGoldenDoc(t, tc.golden)
+
+			// Drop the meta-declared accepted divergences on both sides (here "table"):
+			// Go appends a structured doc_type_kwd:"table" item, while Python keeps the
+			// <table> markup inline as a "text" item. The inline markup remains in both
+			// runs and is still compared — that is exactly what guards against collapse.
+			drop := AcceptedDivergences(doc.Meta)
+			goText := FilterOutDocTypes(res.JSON, drop)
+			pyText := FilterOutDocTypes(doc.Items, drop)
+
+			if ok, diff := CompareAlignment(goText, pyText, HTMLAlignOptions()); !ok {
+				t.Fatalf("html parser not aligned with Python golden:%s", diff)
+			}
+		})
+	}
+}

internal/parser/parser/testdata/html.python.en.golden.json

@@ -0,0 +1,33 @@
+{
+  "meta": {
+    "python_engine": "deepdoc.parser.html_parser.RAGFlowHtmlParser",
+    "generator": "rag/flow/parser/parser.py:_html (HtmlParser()(name, blob, 512))",
+    "sample": "internal/parser/parser/testdata/html.sample.en.html",
+    "delimiter": "\\n!?;。；！？",
+    "separate_tables": false,
+    "accepted_divergences": ["table"],
+    "note": "No generator script is committed. To regenerate: call RAGFlowHtmlParser.read_text_recursively on the sample at chunk_token_num=512, apply the TITLE_TAGS heading prefix (\"# \"/\"## \" …) inline, emit every block (including the inline <table>…</table> markup) as a {\"text\": …, \"doc_type_kwd\": \"text\"} record, then wrap in {meta, items}. Go also appends a structured doc_type_kwd:\"table\" item; that extra record is excluded via meta.accepted_divergences in the alignment test."
+  },
+  "items": [
+    {
+      "text": "# Product Guide",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "Step 1: Open the app. Step 2: Click Settings to complete the setup.",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "<table>\n<tr><th>Name</th><th>Age</th></tr>\n<tr><td>Alice</td><td>30</td></tr>\n<tr><td>Bob</td><td>25</td></tr>\n</table>",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "## Terms of Use",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "Welcome to our smart assistant, which helps you complete tasks quickly.",
+      "doc_type_kwd": "text"
+    }
+  ]
+}

internal/parser/parser/testdata/html.python.zh.golden.json

@@ -0,0 +1,33 @@
+{
+  "meta": {
+    "python_engine": "deepdoc.parser.html_parser.RAGFlowHtmlParser",
+    "generator": "rag/flow/parser/parser.py:_html (HtmlParser()(name, blob, 512))",
+    "sample": "internal/parser/parser/testdata/html.sample.zh.html",
+    "delimiter": "\\n!?;。；！？",
+    "separate_tables": false,
+    "accepted_divergences": ["table"],
+    "note": "No generator script is committed. To regenerate: call RAGFlowHtmlParser.read_text_recursively on the sample at chunk_token_num=512, apply the TITLE_TAGS heading prefix (\"# \"/\"## \" …) inline, emit every block (including the inline <table>…</table> markup) as a {\"text\": …, \"doc_type_kwd\": \"text\"} record, then wrap in {meta, items}. Go also appends a structured doc_type_kwd:\"table\" item; that extra record is excluded via meta.accepted_divergences in the alignment test."
+  },
+  "items": [
+    {
+      "text": "# 产品指南",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "第一步：打开应用。第二步：点击设置完成配置。",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "<table>\n<tr><th>姓名</th><th>年龄</th></tr>\n<tr><td>张三</td><td>30</td></tr>\n<tr><td>李四</td><td>25</td></tr>\n</table>",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "## 使用条款",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "欢迎使用我们的智能助手，它能帮你快速完成任务。",
+      "doc_type_kwd": "text"
+    }
+  ]
+}

internal/parser/parser/testdata/html.sample.en.html

@@ -0,0 +1,11 @@
+<html><body>
+<h1>Product Guide</h1>
+<p>Step 1: Open the app. Step 2: Click Settings to complete the setup.</p>
+<table>
+<tr><th>Name</th><th>Age</th></tr>
+<tr><td>Alice</td><td>30</td></tr>
+<tr><td>Bob</td><td>25</td></tr>
+</table>
+<h2>Terms of Use</h2>
+<p>Welcome to our smart assistant, which helps you complete tasks quickly.</p>
+</body></html>

internal/parser/parser/testdata/html.sample.zh.html

@@ -0,0 +1,11 @@
+<html><body>
+<h1>产品指南</h1>
+<p>第一步：打开应用。第二步：点击设置完成配置。</p>
+<table>
+<tr><th>姓名</th><th>年龄</th></tr>
+<tr><td>张三</td><td>30</td></tr>
+<tr><td>李四</td><td>25</td></tr>
+</table>
+<h2>使用条款</h2>
+<p>欢迎使用我们的智能助手，它能帮你快速完成任务。</p>
+</body></html>