test(parser): split markdown alignment golden into en/zh, cover CJK

- Rename the English sample/golden to *_en.* and add a Chinese counterpart (*_zh.*) so markdown parsing is exercised in both Latin and CJK contexts; the Chinese sample also covers the full-width delimiters in the default delimiter set (\n!?;。；！？). - Both baselines keep the {meta, items} form with accepted_divergences declared in meta; the test is now table-driven over en/zh. - The generator script is not committed; regeneration is described in each golden's meta block. Stacks on the markdown golden meta-driven refactor (c84236ace).
2026-08-15 13:14:28 +08:00 · 2026-08-11 17:52:54 +08:00
parent 9d0e400093
commit 95cab76ee6
5 changed files with 125 additions and 21 deletions
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -531,33 +531,50 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
 // stripped; whitespace collapsed) and ignores the doc types the golden declares
 // as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1).
 //
-// No generator script is committed. The baseline is reproducible from the
-// golden's meta block alone (see markdown.python.golden.json: generator, sample,
-// delimiter, accepted_divergences): call the python flow _markdown on the sample
-// with the default delimiter set, then project each merged section to
-// {"text": section[0], "doc_type_kwd": "text"}.
+// Both an English (markdown.sample.en.md) and a Chinese (markdown.sample.zh.md)
+// sample are checked so markdown parsing is exercised in both Latin and CJK
+// contexts — the Chinese sample also covers the full-width delimiters in the
+// default delimiter set (\n!?;。；！？). Each baseline is a {meta, items}
+// document whose "meta" block records how it was produced (generator
+// rag/flow/parser/parser.py:_markdown, sample, delimiter, accepted
+// divergences). No generator script is committed — to regenerate, call
+// _markdown on the sample and dump {meta, items}. The baseline is
+// reproducible from the metadata alone (an AI or human can recreate the thin
+// wrapper on demand).
 func TestMarkdownParser_AlignmentGolden(t *testing.T) {
 	ctx := t.Context()
 	p, _ := NewMarkdownParser(GoMarkdown)

-	sample, err := os.ReadFile("testdata/markdown.sample.md")
-	if err != nil {
-		t.Fatalf("read sample: %v", err)
-	}
-	res := p.ParseWithResult(ctx, "markdown.sample.md", sample)
-	if res.Err != nil {
-		t.Fatalf("ParseWithResult: %v", res.Err)
+	cases := []struct {
+		name   string
+		sample string
+		golden string
+	}{
+		{"en", "testdata/markdown.sample.en.md", "testdata/markdown.python.en.golden.json"},
+		{"zh", "testdata/markdown.sample.zh.md", "testdata/markdown.python.zh.golden.json"},
 	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			sample, err := os.ReadFile(tc.sample)
+			if err != nil {
+				t.Fatalf("read sample: %v", err)
+			}
+			res := p.ParseWithResult(ctx, tc.sample, sample)
+			if res.Err != nil {
+				t.Fatalf("ParseWithResult: %v", res.Err)
+			}

-	gd := LoadGoldenDoc(t, "testdata/markdown.python.golden.json")
+			gd := LoadGoldenDoc(t, tc.golden)

-	// Exclude the doc types the golden declares as accepted divergences
-	// (meta.accepted_divergences) on both sides — no hardcoded list in the test.
-	ignore := AcceptedDivergences(gd.Meta)
-	goText := FilterOutDocTypes(res.JSON, ignore)
-	pyText := FilterOutDocTypes(gd.Items, ignore)
+			// Exclude the doc types the golden declares as accepted divergences
+			// (meta.accepted_divergences) on both sides — no hardcoded list in the test.
+			ignore := AcceptedDivergences(gd.Meta)
+			goText := FilterOutDocTypes(res.JSON, ignore)
+			pyText := FilterOutDocTypes(gd.Items, ignore)

-	if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
-		t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
+			if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
+				t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
+			}
+		})
 	}
 }
--- a/internal/parser/parser/testdata/markdown.python.en.golden.json
+++ b/internal/parser/parser/testdata/markdown.python.en.golden.json
@@ -1,7 +1,7 @@
 {
  "meta": {
    "generator": "rag/flow/parser/parser.py:_markdown",
-    "sample": "internal/parser/parser/testdata/markdown.sample.md",
+    "sample": "internal/parser/parser/testdata/markdown.sample.en.md",
    "delimiter": "\n!?;。；！？",
    "separate_tables": false,
    "accepted_divergences": ["table", "image"],
--- a/internal/parser/parser/testdata/markdown.python.zh.golden.json
+++ b/internal/parser/parser/testdata/markdown.python.zh.golden.json
@@ -0,0 +1,61 @@
+{
+  "meta": {
+    "generator": "rag/flow/parser/parser.py:_markdown",
+    "sample": "internal/parser/parser/testdata/markdown.sample.zh.md",
+    "delimiter": "\n!?;。；！？",
+    "separate_tables": false,
+    "accepted_divergences": ["table", "image"],
+    "python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser",
+    "note": "No generator script is committed. To regenerate: call _markdown on the sample and dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)."
+  },
+  "items": [
+    {
+      "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "## 套餐明细",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "注意：所有套餐均需空腹",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "## 注意事项",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "- 体检前三天清淡饮食",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "- 避免剧烈运动！",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "下面是示例配置：",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "```yaml\nname: health-check\nversion: 1\n```",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "示意图",
+      "doc_type_kwd": "image"
+    },
+    {
+      "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n",
+      "doc_type_kwd": "table"
+    },
+    {
+      "text": "\n<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>\n",
+      "doc_type_kwd": "table"
+    }
+  ]
+}
--- a/internal/parser/parser/testdata/markdown.sample.en.md
+++ b/internal/parser/parser/testdata/markdown.sample.en.md
--- a/internal/parser/parser/testdata/markdown.sample.zh.md
+++ b/internal/parser/parser/testdata/markdown.sample.zh.md
@@ -0,0 +1,26 @@
+# 健康检查套餐对比
+
+本文比较两种体检套餐，包含表格、列表与代码块
+
+## 套餐明细
+
+| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |
+| --- | --- | --- |
+| 血常规 / 尿常规 | 包含 | 包含 |
+| 心电图 | 不包含 | 包含 |
+
+注意：所有套餐均需空腹。
+
+## 注意事项
+
+- 体检前三天清淡饮食。
+- 避免剧烈运动！
+
+下面是示例配置：
+
+```yaml
+name: health-check
+version: 1
+```
+
+![示意图](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC)