From 95cab76ee6cb4504e1d65a95c0cf94900e885ba0 Mon Sep 17 00:00:00 2001 From: xugangqiang Date: Tue, 11 Aug 2026 17:52:54 +0800 Subject: [PATCH] test(parser): split markdown alignment golden into en/zh, cover CJK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename the English sample/golden to *_en.* and add a Chinese counterpart (*_zh.*) so markdown parsing is exercised in both Latin and CJK contexts; the Chinese sample also covers the full-width delimiters in the default delimiter set (\n!?;。；！？). - Both baselines keep the {meta, items} form with accepted_divergences declared in meta; the test is now table-driven over en/zh. - The generator script is not committed; regeneration is described in each golden's meta block. Stacks on the markdown golden meta-driven refactor (c84236ace). --- .../parser/parser/markdown_parser_test.go | 57 +++++++++++------ ...en.json => markdown.python.en.golden.json} | 2 +- .../testdata/markdown.python.zh.golden.json | 61 +++++++++++++++++++ ...rkdown.sample.md => markdown.sample.en.md} | 0 .../parser/testdata/markdown.sample.zh.md | 26 ++++++++ 5 files changed, 125 insertions(+), 21 deletions(-) rename internal/parser/parser/testdata/{markdown.python.golden.json => markdown.python.en.golden.json} (97%) create mode 100644 internal/parser/parser/testdata/markdown.python.zh.golden.json rename internal/parser/parser/testdata/{markdown.sample.md => markdown.sample.en.md} (100%) create mode 100644 internal/parser/parser/testdata/markdown.sample.zh.md diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index b97b750e7d..e0d5413b7a 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -531,33 +531,50 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) { // stripped; whitespace collapsed) and ignores the doc types the golden declares // as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1). // -// No generator script is committed. The baseline is reproducible from the -// golden's meta block alone (see markdown.python.golden.json: generator, sample, -// delimiter, accepted_divergences): call the python flow _markdown on the sample -// with the default delimiter set, then project each merged section to -// {"text": section[0], "doc_type_kwd": "text"}. +// Both an English (markdown.sample.en.md) and a Chinese (markdown.sample.zh.md) +// sample are checked so markdown parsing is exercised in both Latin and CJK +// contexts — the Chinese sample also covers the full-width delimiters in the +// default delimiter set (\n!?;。；！？). Each baseline is a {meta, items} +// document whose "meta" block records how it was produced (generator +// rag/flow/parser/parser.py:_markdown, sample, delimiter, accepted +// divergences). No generator script is committed — to regenerate, call +// _markdown on the sample and dump {meta, items}. The baseline is +// reproducible from the metadata alone (an AI or human can recreate the thin +// wrapper on demand). func TestMarkdownParser_AlignmentGolden(t *testing.T) { ctx := t.Context() p, _ := NewMarkdownParser(GoMarkdown) - sample, err := os.ReadFile("testdata/markdown.sample.md") - if err != nil { - t.Fatalf("read sample: %v", err) - } - res := p.ParseWithResult(ctx, "markdown.sample.md", sample) - if res.Err != nil { - t.Fatalf("ParseWithResult: %v", res.Err) + cases := []struct { + name string + sample string + golden string + }{ + {"en", "testdata/markdown.sample.en.md", "testdata/markdown.python.en.golden.json"}, + {"zh", "testdata/markdown.sample.zh.md", "testdata/markdown.python.zh.golden.json"}, } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sample, err := os.ReadFile(tc.sample) + if err != nil { + t.Fatalf("read sample: %v", err) + } + res := p.ParseWithResult(ctx, tc.sample, sample) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } - gd := LoadGoldenDoc(t, "testdata/markdown.python.golden.json") + gd := LoadGoldenDoc(t, tc.golden) - // Exclude the doc types the golden declares as accepted divergences - // (meta.accepted_divergences) on both sides — no hardcoded list in the test. - ignore := AcceptedDivergences(gd.Meta) - goText := FilterOutDocTypes(res.JSON, ignore) - pyText := FilterOutDocTypes(gd.Items, ignore) + // Exclude the doc types the golden declares as accepted divergences + // (meta.accepted_divergences) on both sides — no hardcoded list in the test. + ignore := AcceptedDivergences(gd.Meta) + goText := FilterOutDocTypes(res.JSON, ignore) + pyText := FilterOutDocTypes(gd.Items, ignore) - if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { - t.Fatalf("markdown parser not aligned with Python golden:%s", diff) + if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { + t.Fatalf("markdown parser not aligned with Python golden:%s", diff) + } + }) } } diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.en.golden.json similarity index 97% rename from internal/parser/parser/testdata/markdown.python.golden.json rename to internal/parser/parser/testdata/markdown.python.en.golden.json index cb6e062750..9cff9029cd 100644 --- a/internal/parser/parser/testdata/markdown.python.golden.json +++ b/internal/parser/parser/testdata/markdown.python.en.golden.json @@ -1,7 +1,7 @@ { "meta": { "generator": "rag/flow/parser/parser.py:_markdown", - "sample": "internal/parser/parser/testdata/markdown.sample.md", + "sample": "internal/parser/parser/testdata/markdown.sample.en.md", "delimiter": "\n!?;。；！？", "separate_tables": false, "accepted_divergences": ["table", "image"], diff --git a/internal/parser/parser/testdata/markdown.python.zh.golden.json b/internal/parser/parser/testdata/markdown.python.zh.golden.json new file mode 100644 index 0000000000..73edab6dda --- /dev/null +++ b/internal/parser/parser/testdata/markdown.python.zh.golden.json @@ -0,0 +1,61 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_markdown", + "sample": "internal/parser/parser/testdata/markdown.sample.zh.md", + "delimiter": "\n!?;。；！？", + "separate_tables": false, + "accepted_divergences": ["table", "image"], + "python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser", + "note": "No generator script is committed. To regenerate: call _markdown on the sample and dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)." + }, + "items": [ + { + "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块", + "doc_type_kwd": "text" + }, + { + "text": "## 套餐明细", + "doc_type_kwd": "text" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
", + "doc_type_kwd": "text" + }, + { + "text": "注意：所有套餐均需空腹", + "doc_type_kwd": "text" + }, + { + "text": "## 注意事项", + "doc_type_kwd": "text" + }, + { + "text": "- 体检前三天清淡饮食", + "doc_type_kwd": "text" + }, + { + "text": "- 避免剧烈运动！", + "doc_type_kwd": "text" + }, + { + "text": "下面是示例配置：", + "doc_type_kwd": "text" + }, + { + "text": "```yaml\nname: health-check\nversion: 1\n```", + "doc_type_kwd": "text" + }, + { + "text": "示意图", + "doc_type_kwd": "image" + }, + { + "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n", + "doc_type_kwd": "table" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
\n", + "doc_type_kwd": "table" + } + ] +} diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.en.md similarity index 100% rename from internal/parser/parser/testdata/markdown.sample.md rename to internal/parser/parser/testdata/markdown.sample.en.md diff --git a/internal/parser/parser/testdata/markdown.sample.zh.md b/internal/parser/parser/testdata/markdown.sample.zh.md new file mode 100644 index 0000000000..c89b573778 --- /dev/null +++ b/internal/parser/parser/testdata/markdown.sample.zh.md @@ -0,0 +1,26 @@ +# 健康检查套餐对比 + +本文比较两种体检套餐，包含表格、列表与代码块 + +## 套餐明细 + +| 检查项目 | 基础版 699 元 | 进阶版 1299 元 | +| --- | --- | --- | +| 血常规 / 尿常规 | 包含 | 包含 | +| 心电图 | 不包含 | 包含 | + +注意：所有套餐均需空腹。 + +## 注意事项 + +- 体检前三天清淡饮食。 +- 避免剧烈运动！ + +下面是示例配置： + +```yaml +name: health-check +version: 1 +``` + +![示意图](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC)