From 9d0e400093231e0b4f1f9ce78b46a27ea2ecb69c Mon Sep 17 00:00:00 2001 From: xugangqiang Date: Tue, 11 Aug 2026 15:47:38 +0800 Subject: [PATCH] refactor(parser): make markdown golden meta-driven, drop generator script Move the markdown golden baseline to the {meta, items} format and drive accepted divergences from meta.accepted_divergences instead of hardcoded lists. No production parser code changes. - markdown.python.golden.json is now {meta, items}; meta records how it was produced so it is reproducible without a committed script. - TestMarkdownParser_AlignmentGolden uses LoadGoldenDoc / AcceptedDivergences / FilterOutDocTypes (from the foundation helpers PR). - Deletes testdata/gen_markdown_golden.py. Depends on #18014 and the foundation helpers PR (align_test.go). Files: internal/parser/parser/markdown_parser_test.go internal/parser/parser/testdata/markdown.python.golden.json internal/parser/parser/testdata/markdown.sample.md internal/parser/parser/testdata/gen_markdown_golden.py (deleted) --- .../parser/parser/markdown_parser_test.go | 22 ++-- .../parser/testdata/gen_markdown_golden.py | 84 -------------- .../testdata/markdown.python.golden.json | 109 ++++++++++-------- .../parser/parser/testdata/markdown.sample.md | 24 ++-- 4 files changed, 85 insertions(+), 154 deletions(-) delete mode 100644 internal/parser/parser/testdata/gen_markdown_golden.py diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index 82891932dd..b97b750e7d 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -528,12 +528,14 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) { // shared concatenation-normalization alignment tool (align_test.go). Python // keeps raw Markdown and splits on the delimiter set; Go emits clean per-block // text. The comparison normalizes both (Markdown syntax, html tags, delimiters -// stripped; whitespace collapsed) and ignores "table"/"image" items, which are -// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1). +// stripped; whitespace collapsed) and ignores the doc types the golden declares +// as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1). // -// Regenerate the baseline with: -// -// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py +// No generator script is committed. The baseline is reproducible from the +// golden's meta block alone (see markdown.python.golden.json: generator, sample, +// delimiter, accepted_divergences): call the python flow _markdown on the sample +// with the default delimiter set, then project each merged section to +// {"text": section[0], "doc_type_kwd": "text"}. func TestMarkdownParser_AlignmentGolden(t *testing.T) { ctx := t.Context() p, _ := NewMarkdownParser(GoMarkdown) @@ -547,11 +549,13 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) { t.Fatalf("ParseWithResult: %v", res.Err) } - golden := LoadGolden(t, "testdata/markdown.python.golden.json") + gd := LoadGoldenDoc(t, "testdata/markdown.python.golden.json") - // Ignore "table"/"image" items on both sides (accepted divergences). - goText := FilterByDocType(res.JSON, "text") - pyText := FilterByDocType(golden, "text") + // Exclude the doc types the golden declares as accepted divergences + // (meta.accepted_divergences) on both sides — no hardcoded list in the test. + ignore := AcceptedDivergences(gd.Meta) + goText := FilterOutDocTypes(res.JSON, ignore) + pyText := FilterOutDocTypes(gd.Items, ignore) if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { t.Fatalf("markdown parser not aligned with Python golden:%s", diff) diff --git a/internal/parser/parser/testdata/gen_markdown_golden.py b/internal/parser/parser/testdata/gen_markdown_golden.py deleted file mode 100644 index fc0c113812..0000000000 --- a/internal/parser/parser/testdata/gen_markdown_golden.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json. - -Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the -same engine rag/flow/parser/parser.py:_markdown delegates to via -rag/app/naive.Markdown) so the golden is a faithful baseline rather than a -hand approximation. - -Requires the project virtualenv (uv) because deepdoc needs markdown / -beartype / etc.: - - .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py - -It mirrors _markdown with separate_tables=False and the default delimiter -set, then assembles json items exactly as _markdown does: - - * each extracted section -> {"text": , "doc_type_kwd": "text"} - * each standalone table -> {"text": , "doc_type_kwd": "table"} - * an image section -> {"text": , "doc_type_kwd": "image"} - -The Go alignment test then strips markdown syntax, html tags, and delimiters -before comparing, and ignores "table"/"image" items (those representations are -accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1). -""" - -import json -import os -import re -import sys - -# Make the repo root importable when run as a standalone script from testdata. -# Script lives at /internal/parser/parser/testdata/, so five dirname hops. -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))) - -SAMPLE = "internal/parser/parser/testdata/markdown.sample.md" -OUT = "internal/parser/parser/testdata/markdown.python.golden.json" -DELIM = "\n!?;。；！？" -IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") -SENTINEL = "@@IMAGE@@" - - -def main(): - from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor - - with open(SAMPLE, encoding="utf-8") as f: - raw = f.read() - - # Model _markdown's return_section_images: the image is extracted as its - # own item (alt text only). Replace the markdown with a delimiter-free - # sentinel so the extractor does not split it. - alts = [] - - def _repl(m): - alts.append(m.group(1)) - return SENTINEL - - prepared = IMG_RE.sub(_repl, raw) - - parser = RAGFlowMarkdownParser() - remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False) - extractor = MarkdownElementExtractor(remainder) - sections = extractor.extract_elements(DELIM, include_meta=True) - - items = [] - for s in sections: - content = s["content"] - if SENTINEL in content: - items.append({"text": alts.pop(0), "doc_type_kwd": "image"}) - continue - items.append({"text": content, "doc_type_kwd": "text"}) - - for tbl in tables: - # _markdown (rag/flow/parser/parser.py:1103-1111) appends each - # extracted table as a duplicate "table" item even when inlined, - # carrying the table's raw text (GFM source or raw HTML). - items.append({"text": tbl, "doc_type_kwd": "table"}) - - with open(OUT, "w", encoding="utf-8") as f: - json.dump(items, f, ensure_ascii=False, indent=2) - print("wrote %d items to %s" % (len(items), OUT)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.golden.json index 73f5168110..cb6e062750 100644 --- a/internal/parser/parser/testdata/markdown.python.golden.json +++ b/internal/parser/parser/testdata/markdown.python.golden.json @@ -1,50 +1,61 @@ -[ - { - "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块", - "doc_type_kwd": "text" +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_markdown", + "sample": "internal/parser/parser/testdata/markdown.sample.md", + "delimiter": "\n!?;。；！？", + "separate_tables": false, + "accepted_divergences": ["table", "image"], + "python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser", + "note": "No generator script is committed. To regenerate: call _markdown on sample, then dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)." }, - { - "text": "## 套餐明细", - "doc_type_kwd": "text" - }, - { - "text": "
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
", - "doc_type_kwd": "text" - }, - { - "text": "注意：所有套餐均需空腹", - "doc_type_kwd": "text" - }, - { - "text": "## 注意事项", - "doc_type_kwd": "text" - }, - { - "text": "- 体检前三天清淡饮食", - "doc_type_kwd": "text" - }, - { - "text": "- 避免剧烈运动", - "doc_type_kwd": "text" - }, - { - "text": "下面是示例配置：", - "doc_type_kwd": "text" - }, - { - "text": "```yaml\nname: health-check\nversion: 1\n```", - "doc_type_kwd": "text" - }, - { - "text": "示意图", - "doc_type_kwd": "image" - }, - { - "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n", - "doc_type_kwd": "table" - }, - { - "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
\n", - "doc_type_kwd": "table" - } -] + "items": [ + { + "text": "# Health Check Plan Comparison\nThis document compares two health check plans, covering tables, lists, and code blocks", + "doc_type_kwd": "text" + }, + { + "text": "## Plan Details", + "doc_type_kwd": "text" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Check ItemBasic 699Advanced 1299
Blood routine / Urine routineIncludedIncluded
ECGNot includedIncluded
", + "doc_type_kwd": "text" + }, + { + "text": "Note: all plans require fasting.", + "doc_type_kwd": "text" + }, + { + "text": "## Precautions", + "doc_type_kwd": "text" + }, + { + "text": "- Eat a light diet for three days before the exam?", + "doc_type_kwd": "text" + }, + { + "text": "- Avoid strenuous exercise!", + "doc_type_kwd": "text" + }, + { + "text": "Here is a sample configuration:", + "doc_type_kwd": "text" + }, + { + "text": "```yaml\nname: health-check\nversion: 1\n```", + "doc_type_kwd": "text" + }, + { + "text": "diagram", + "doc_type_kwd": "image" + }, + { + "text": "\n| Check Item | Basic 699 | Advanced 1299 |\n| --- | --- | --- |\n| Blood routine / Urine routine | Included | Included |\n| ECG | Not included | Included |\n", + "doc_type_kwd": "table" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Check ItemBasic 699Advanced 1299
Blood routine / Urine routineIncludedIncluded
ECGNot includedIncluded
\n", + "doc_type_kwd": "table" + } + ] +} diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.md index 7aec22ee26..a31480a65b 100644 --- a/internal/parser/parser/testdata/markdown.sample.md +++ b/internal/parser/parser/testdata/markdown.sample.md @@ -1,26 +1,26 @@ -# 健康检查套餐对比 +# Health Check Plan Comparison -本文比较两种体检套餐，包含表格、列表与代码块。 +This document compares two health check plans, covering tables, lists, and code blocks -## 套餐明细 +## Plan Details -| 检查项目 | 基础版 699 元 | 进阶版 1299 元 | +| Check Item | Basic 699 | Advanced 1299 | | --- | --- | --- | -| 血常规 / 尿常规 | 包含 | 包含 | -| 心电图 | 不包含 | 包含 | +| Blood routine / Urine routine | Included | Included | +| ECG | Not included | Included | -注意：所有套餐均需空腹。 +Note: all plans require fasting. -## 注意事项 +## Precautions -- 体检前三天清淡饮食。 -- 避免剧烈运动！ +- Eat a light diet for three days before the exam? +- Avoid strenuous exercise! -下面是示例配置： +Here is a sample configuration: ```yaml name: health-check version: 1 ``` -![示意图](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC) +![diagram](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC)