diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index 82891932dd..e0d5413b7a 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -528,32 +528,53 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) { // shared concatenation-normalization alignment tool (align_test.go). Python // keeps raw Markdown and splits on the delimiter set; Go emits clean per-block // text. The comparison normalizes both (Markdown syntax, html tags, delimiters -// stripped; whitespace collapsed) and ignores "table"/"image" items, which are -// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1). +// stripped; whitespace collapsed) and ignores the doc types the golden declares +// as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1). // -// Regenerate the baseline with: -// -// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py +// Both an English (markdown.sample.en.md) and a Chinese (markdown.sample.zh.md) +// sample are checked so markdown parsing is exercised in both Latin and CJK +// contexts — the Chinese sample also covers the full-width delimiters in the +// default delimiter set (\n!?;。；！？). Each baseline is a {meta, items} +// document whose "meta" block records how it was produced (generator +// rag/flow/parser/parser.py:_markdown, sample, delimiter, accepted +// divergences). No generator script is committed — to regenerate, call +// _markdown on the sample and dump {meta, items}. The baseline is +// reproducible from the metadata alone (an AI or human can recreate the thin +// wrapper on demand). func TestMarkdownParser_AlignmentGolden(t *testing.T) { ctx := t.Context() p, _ := NewMarkdownParser(GoMarkdown) - sample, err := os.ReadFile("testdata/markdown.sample.md") - if err != nil { - t.Fatalf("read sample: %v", err) - } - res := p.ParseWithResult(ctx, "markdown.sample.md", sample) - if res.Err != nil { - t.Fatalf("ParseWithResult: %v", res.Err) + cases := []struct { + name string + sample string + golden string + }{ + {"en", "testdata/markdown.sample.en.md", "testdata/markdown.python.en.golden.json"}, + {"zh", "testdata/markdown.sample.zh.md", "testdata/markdown.python.zh.golden.json"}, } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sample, err := os.ReadFile(tc.sample) + if err != nil { + t.Fatalf("read sample: %v", err) + } + res := p.ParseWithResult(ctx, tc.sample, sample) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } - golden := LoadGolden(t, "testdata/markdown.python.golden.json") + gd := LoadGoldenDoc(t, tc.golden) - // Ignore "table"/"image" items on both sides (accepted divergences). - goText := FilterByDocType(res.JSON, "text") - pyText := FilterByDocType(golden, "text") + // Exclude the doc types the golden declares as accepted divergences + // (meta.accepted_divergences) on both sides — no hardcoded list in the test. + ignore := AcceptedDivergences(gd.Meta) + goText := FilterOutDocTypes(res.JSON, ignore) + pyText := FilterOutDocTypes(gd.Items, ignore) - if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { - t.Fatalf("markdown parser not aligned with Python golden:%s", diff) + if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { + t.Fatalf("markdown parser not aligned with Python golden:%s", diff) + } + }) } } diff --git a/internal/parser/parser/testdata/gen_markdown_golden.py b/internal/parser/parser/testdata/gen_markdown_golden.py deleted file mode 100644 index fc0c113812..0000000000 --- a/internal/parser/parser/testdata/gen_markdown_golden.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json. - -Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the -same engine rag/flow/parser/parser.py:_markdown delegates to via -rag/app/naive.Markdown) so the golden is a faithful baseline rather than a -hand approximation. - -Requires the project virtualenv (uv) because deepdoc needs markdown / -beartype / etc.: - - .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py - -It mirrors _markdown with separate_tables=False and the default delimiter -set, then assembles json items exactly as _markdown does: - - * each extracted section -> {"text": , "doc_type_kwd": "text"} - * each standalone table -> {"text": , "doc_type_kwd": "table"} - * an image section -> {"text": , "doc_type_kwd": "image"} - -The Go alignment test then strips markdown syntax, html tags, and delimiters -before comparing, and ignores "table"/"image" items (those representations are -accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1). -""" - -import json -import os -import re -import sys - -# Make the repo root importable when run as a standalone script from testdata. -# Script lives at /internal/parser/parser/testdata/, so five dirname hops. -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))) - -SAMPLE = "internal/parser/parser/testdata/markdown.sample.md" -OUT = "internal/parser/parser/testdata/markdown.python.golden.json" -DELIM = "\n!?;。；！？" -IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") -SENTINEL = "@@IMAGE@@" - - -def main(): - from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor - - with open(SAMPLE, encoding="utf-8") as f: - raw = f.read() - - # Model _markdown's return_section_images: the image is extracted as its - # own item (alt text only). Replace the markdown with a delimiter-free - # sentinel so the extractor does not split it. - alts = [] - - def _repl(m): - alts.append(m.group(1)) - return SENTINEL - - prepared = IMG_RE.sub(_repl, raw) - - parser = RAGFlowMarkdownParser() - remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False) - extractor = MarkdownElementExtractor(remainder) - sections = extractor.extract_elements(DELIM, include_meta=True) - - items = [] - for s in sections: - content = s["content"] - if SENTINEL in content: - items.append({"text": alts.pop(0), "doc_type_kwd": "image"}) - continue - items.append({"text": content, "doc_type_kwd": "text"}) - - for tbl in tables: - # _markdown (rag/flow/parser/parser.py:1103-1111) appends each - # extracted table as a duplicate "table" item even when inlined, - # carrying the table's raw text (GFM source or raw HTML). - items.append({"text": tbl, "doc_type_kwd": "table"}) - - with open(OUT, "w", encoding="utf-8") as f: - json.dump(items, f, ensure_ascii=False, indent=2) - print("wrote %d items to %s" % (len(items), OUT)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/internal/parser/parser/testdata/markdown.python.en.golden.json b/internal/parser/parser/testdata/markdown.python.en.golden.json new file mode 100644 index 0000000000..9cff9029cd --- /dev/null +++ b/internal/parser/parser/testdata/markdown.python.en.golden.json @@ -0,0 +1,61 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_markdown", + "sample": "internal/parser/parser/testdata/markdown.sample.en.md", + "delimiter": "\n!?;。；！？", + "separate_tables": false, + "accepted_divergences": ["table", "image"], + "python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser", + "note": "No generator script is committed. To regenerate: call _markdown on sample, then dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)." + }, + "items": [ + { + "text": "# Health Check Plan Comparison\nThis document compares two health check plans, covering tables, lists, and code blocks", + "doc_type_kwd": "text" + }, + { + "text": "## Plan Details", + "doc_type_kwd": "text" + }, + { + "text": "
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Check ItemBasic 699Advanced 1299
Blood routine / Urine routineIncludedIncluded
ECGNot includedIncluded
", + "doc_type_kwd": "text" + }, + { + "text": "Note: all plans require fasting.", + "doc_type_kwd": "text" + }, + { + "text": "## Precautions", + "doc_type_kwd": "text" + }, + { + "text": "- Eat a light diet for three days before the exam?", + "doc_type_kwd": "text" + }, + { + "text": "- Avoid strenuous exercise!", + "doc_type_kwd": "text" + }, + { + "text": "Here is a sample configuration:", + "doc_type_kwd": "text" + }, + { + "text": "```yaml\nname: health-check\nversion: 1\n```", + "doc_type_kwd": "text" + }, + { + "text": "diagram", + "doc_type_kwd": "image" + }, + { + "text": "\n| Check Item | Basic 699 | Advanced 1299 |\n| --- | --- | --- |\n| Blood routine / Urine routine | Included | Included |\n| ECG | Not included | Included |\n", + "doc_type_kwd": "table" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Check ItemBasic 699Advanced 1299
Blood routine / Urine routineIncludedIncluded
ECGNot includedIncluded
\n", + "doc_type_kwd": "table" + } + ] +} diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.golden.json deleted file mode 100644 index 73f5168110..0000000000 --- a/internal/parser/parser/testdata/markdown.python.golden.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块", - "doc_type_kwd": "text" - }, - { - "text": "## 套餐明细", - "doc_type_kwd": "text" - }, - { - "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
", - "doc_type_kwd": "text" - }, - { - "text": "注意：所有套餐均需空腹", - "doc_type_kwd": "text" - }, - { - "text": "## 注意事项", - "doc_type_kwd": "text" - }, - { - "text": "- 体检前三天清淡饮食", - "doc_type_kwd": "text" - }, - { - "text": "- 避免剧烈运动", - "doc_type_kwd": "text" - }, - { - "text": "下面是示例配置：", - "doc_type_kwd": "text" - }, - { - "text": "```yaml\nname: health-check\nversion: 1\n```", - "doc_type_kwd": "text" - }, - { - "text": "示意图", - "doc_type_kwd": "image" - }, - { - "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n", - "doc_type_kwd": "table" - }, - { - "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
\n", - "doc_type_kwd": "table" - } -] diff --git a/internal/parser/parser/testdata/markdown.python.zh.golden.json b/internal/parser/parser/testdata/markdown.python.zh.golden.json new file mode 100644 index 0000000000..73edab6dda --- /dev/null +++ b/internal/parser/parser/testdata/markdown.python.zh.golden.json @@ -0,0 +1,61 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_markdown", + "sample": "internal/parser/parser/testdata/markdown.sample.zh.md", + "delimiter": "\n!?;。；！？", + "separate_tables": false, + "accepted_divergences": ["table", "image"], + "python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser", + "note": "No generator script is committed. To regenerate: call _markdown on the sample and dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)." + }, + "items": [ + { + "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块", + "doc_type_kwd": "text" + }, + { + "text": "## 套餐明细", + "doc_type_kwd": "text" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
", + "doc_type_kwd": "text" + }, + { + "text": "注意：所有套餐均需空腹", + "doc_type_kwd": "text" + }, + { + "text": "## 注意事项", + "doc_type_kwd": "text" + }, + { + "text": "- 体检前三天清淡饮食", + "doc_type_kwd": "text" + }, + { + "text": "- 避免剧烈运动！", + "doc_type_kwd": "text" + }, + { + "text": "下面是示例配置：", + "doc_type_kwd": "text" + }, + { + "text": "```yaml\nname: health-check\nversion: 1\n```", + "doc_type_kwd": "text" + }, + { + "text": "示意图", + "doc_type_kwd": "image" + }, + { + "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n", + "doc_type_kwd": "table" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
\n", + "doc_type_kwd": "table" + } + ] +} diff --git a/internal/parser/parser/testdata/markdown.sample.en.md b/internal/parser/parser/testdata/markdown.sample.en.md new file mode 100644 index 0000000000..a31480a65b --- /dev/null +++ b/internal/parser/parser/testdata/markdown.sample.en.md @@ -0,0 +1,26 @@ +# Health Check Plan Comparison + +This document compares two health check plans, covering tables, lists, and code blocks + +## Plan Details + +| Check Item | Basic 699 | Advanced 1299 | +| --- | --- | --- | +| Blood routine / Urine routine | Included | Included | +| ECG | Not included | Included | + +Note: all plans require fasting. + +## Precautions + +- Eat a light diet for three days before the exam? +- Avoid strenuous exercise! + +Here is a sample configuration: + +```yaml +name: health-check +version: 1 +``` + +![diagram](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC) diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.zh.md similarity index 98% rename from internal/parser/parser/testdata/markdown.sample.md rename to internal/parser/parser/testdata/markdown.sample.zh.md index 7aec22ee26..c89b573778 100644 --- a/internal/parser/parser/testdata/markdown.sample.md +++ b/internal/parser/parser/testdata/markdown.sample.zh.md @@ -1,6 +1,6 @@ # 健康检查套餐对比 -本文比较两种体检套餐，包含表格、列表与代码块。 +本文比较两种体检套餐，包含表格、列表与代码块 ## 套餐明细