mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-14 12:47:14 +08:00
refactor(parser): make markdown golden meta-driven, drop generator script (#18125)
This commit is contained in:
Jack
@@ -528,32 +528,53 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
|
||||
// shared concatenation-normalization alignment tool (align_test.go). Python
|
||||
// keeps raw Markdown and splits on the delimiter set; Go emits clean per-block
|
||||
// text. The comparison normalizes both (Markdown syntax, html tags, delimiters
|
||||
// stripped; whitespace collapsed) and ignores "table"/"image" items, which are
|
||||
// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1).
|
||||
// stripped; whitespace collapsed) and ignores the doc types the golden declares
|
||||
// as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1).
|
||||
//
|
||||
// Regenerate the baseline with:
|
||||
//
|
||||
// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py
|
||||
// Both an English (markdown.sample.en.md) and a Chinese (markdown.sample.zh.md)
|
||||
// sample are checked so markdown parsing is exercised in both Latin and CJK
|
||||
// contexts — the Chinese sample also covers the full-width delimiters in the
|
||||
// default delimiter set (\n!?;。；！？). Each baseline is a {meta, items}
|
||||
// document whose "meta" block records how it was produced (generator
|
||||
// rag/flow/parser/parser.py:_markdown, sample, delimiter, accepted
|
||||
// divergences). No generator script is committed — to regenerate, call
|
||||
// _markdown on the sample and dump {meta, items}. The baseline is
|
||||
// reproducible from the metadata alone (an AI or human can recreate the thin
|
||||
// wrapper on demand).
|
||||
func TestMarkdownParser_AlignmentGolden(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
p, _ := NewMarkdownParser(GoMarkdown)
|
||||
|
||||
sample, err := os.ReadFile("testdata/markdown.sample.md")
|
||||
if err != nil {
|
||||
t.Fatalf("read sample: %v", err)
|
||||
}
|
||||
res := p.ParseWithResult(ctx, "markdown.sample.md", sample)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
cases := []struct {
|
||||
name string
|
||||
sample string
|
||||
golden string
|
||||
}{
|
||||
{"en", "testdata/markdown.sample.en.md", "testdata/markdown.python.en.golden.json"},
|
||||
{"zh", "testdata/markdown.sample.zh.md", "testdata/markdown.python.zh.golden.json"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sample, err := os.ReadFile(tc.sample)
|
||||
if err != nil {
|
||||
t.Fatalf("read sample: %v", err)
|
||||
}
|
||||
res := p.ParseWithResult(ctx, tc.sample, sample)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
|
||||
golden := LoadGolden(t, "testdata/markdown.python.golden.json")
|
||||
gd := LoadGoldenDoc(t, tc.golden)
|
||||
|
||||
// Ignore "table"/"image" items on both sides (accepted divergences).
|
||||
goText := FilterByDocType(res.JSON, "text")
|
||||
pyText := FilterByDocType(golden, "text")
|
||||
// Exclude the doc types the golden declares as accepted divergences
|
||||
// (meta.accepted_divergences) on both sides — no hardcoded list in the test.
|
||||
ignore := AcceptedDivergences(gd.Meta)
|
||||
goText := FilterOutDocTypes(res.JSON, ignore)
|
||||
pyText := FilterOutDocTypes(gd.Items, ignore)
|
||||
|
||||
if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
|
||||
t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
|
||||
if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
|
||||
t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json.
|
||||
|
||||
Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the
|
||||
same engine rag/flow/parser/parser.py:_markdown delegates to via
|
||||
rag/app/naive.Markdown) so the golden is a faithful baseline rather than a
|
||||
hand approximation.
|
||||
|
||||
Requires the project virtualenv (uv) because deepdoc needs markdown /
|
||||
beartype / etc.:
|
||||
|
||||
.venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py
|
||||
|
||||
It mirrors _markdown with separate_tables=False and the default delimiter
|
||||
set, then assembles json items exactly as _markdown does:
|
||||
|
||||
* each extracted section -> {"text": <raw markdown section>, "doc_type_kwd": "text"}
|
||||
* each standalone table -> {"text": <html table>, "doc_type_kwd": "table"}
|
||||
* an image section -> {"text": <alt>, "doc_type_kwd": "image"}
|
||||
|
||||
The Go alignment test then strips markdown syntax, html tags, and delimiters
|
||||
before comparing, and ignores "table"/"image" items (those representations are
|
||||
accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Make the repo root importable when run as a standalone script from testdata.
|
||||
# Script lives at <repo>/internal/parser/parser/testdata/, so five dirname hops.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))))
|
||||
|
||||
SAMPLE = "internal/parser/parser/testdata/markdown.sample.md"
|
||||
OUT = "internal/parser/parser/testdata/markdown.python.golden.json"
|
||||
DELIM = "\n!?;。；！？"
|
||||
IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
SENTINEL = "@@IMAGE@@"
|
||||
|
||||
|
||||
def main():
|
||||
from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor
|
||||
|
||||
with open(SAMPLE, encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
|
||||
# Model _markdown's return_section_images: the image is extracted as its
|
||||
# own item (alt text only). Replace the markdown with a delimiter-free
|
||||
# sentinel so the extractor does not split it.
|
||||
alts = []
|
||||
|
||||
def _repl(m):
|
||||
alts.append(m.group(1))
|
||||
return SENTINEL
|
||||
|
||||
prepared = IMG_RE.sub(_repl, raw)
|
||||
|
||||
parser = RAGFlowMarkdownParser()
|
||||
remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False)
|
||||
extractor = MarkdownElementExtractor(remainder)
|
||||
sections = extractor.extract_elements(DELIM, include_meta=True)
|
||||
|
||||
items = []
|
||||
for s in sections:
|
||||
content = s["content"]
|
||||
if SENTINEL in content:
|
||||
items.append({"text": alts.pop(0), "doc_type_kwd": "image"})
|
||||
continue
|
||||
items.append({"text": content, "doc_type_kwd": "text"})
|
||||
|
||||
for tbl in tables:
|
||||
# _markdown (rag/flow/parser/parser.py:1103-1111) appends each
|
||||
# extracted table as a duplicate "table" item even when inlined,
|
||||
# carrying the table's raw text (GFM source or raw <table> HTML).
|
||||
items.append({"text": tbl, "doc_type_kwd": "table"})
|
||||
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
json.dump(items, f, ensure_ascii=False, indent=2)
|
||||
print("wrote %d items to %s" % (len(items), OUT))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
61
internal/parser/parser/testdata/markdown.python.en.golden.json
vendored
Normal file
61
internal/parser/parser/testdata/markdown.python.en.golden.json
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"meta": {
|
||||
"generator": "rag/flow/parser/parser.py:_markdown",
|
||||
"sample": "internal/parser/parser/testdata/markdown.sample.en.md",
|
||||
"delimiter": "\n!?;。；！？",
|
||||
"separate_tables": false,
|
||||
"accepted_divergences": ["table", "image"],
|
||||
"python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser",
|
||||
"note": "No generator script is committed. To regenerate: call _markdown on sample, then dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)."
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"text": "# Health Check Plan Comparison\nThis document compares two health check plans, covering tables, lists, and code blocks",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## Plan Details",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "<table>\n<thead>\n<tr>\n<th>Check Item</th>\n<th>Basic 699</th>\n<th>Advanced 1299</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>Blood routine / Urine routine</td>\n<td>Included</td>\n<td>Included</td>\n</tr>\n<tr>\n<td>ECG</td>\n<td>Not included</td>\n<td>Included</td>\n</tr>\n</tbody>\n</table>",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "Note: all plans require fasting.",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## Precautions",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- Eat a light diet for three days before the exam?",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- Avoid strenuous exercise!",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "Here is a sample configuration:",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "```yaml\nname: health-check\nversion: 1\n```",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "diagram",
|
||||
"doc_type_kwd": "image"
|
||||
},
|
||||
{
|
||||
"text": "\n| Check Item | Basic 699 | Advanced 1299 |\n| --- | --- | --- |\n| Blood routine / Urine routine | Included | Included |\n| ECG | Not included | Included |\n",
|
||||
"doc_type_kwd": "table"
|
||||
},
|
||||
{
|
||||
"text": "\n<table>\n<thead>\n<tr>\n<th>Check Item</th>\n<th>Basic 699</th>\n<th>Advanced 1299</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>Blood routine / Urine routine</td>\n<td>Included</td>\n<td>Included</td>\n</tr>\n<tr>\n<td>ECG</td>\n<td>Not included</td>\n<td>Included</td>\n</tr>\n</tbody>\n</table>\n",
|
||||
"doc_type_kwd": "table"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
[
|
||||
{
|
||||
"text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## 套餐明细",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "注意：所有套餐均需空腹",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## 注意事项",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- 体检前三天清淡饮食",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- 避免剧烈运动",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "下面是示例配置：",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "```yaml\nname: health-check\nversion: 1\n```",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "示意图",
|
||||
"doc_type_kwd": "image"
|
||||
},
|
||||
{
|
||||
"text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n",
|
||||
"doc_type_kwd": "table"
|
||||
},
|
||||
{
|
||||
"text": "\n<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>\n",
|
||||
"doc_type_kwd": "table"
|
||||
}
|
||||
]
|
||||
61
internal/parser/parser/testdata/markdown.python.zh.golden.json
vendored
Normal file
61
internal/parser/parser/testdata/markdown.python.zh.golden.json
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"meta": {
|
||||
"generator": "rag/flow/parser/parser.py:_markdown",
|
||||
"sample": "internal/parser/parser/testdata/markdown.sample.zh.md",
|
||||
"delimiter": "\n!?;。；！？",
|
||||
"separate_tables": false,
|
||||
"accepted_divergences": ["table", "image"],
|
||||
"python_engine": "deepdoc.parser.markdown_parser.RAGFlowMarkdownParser",
|
||||
"note": "No generator script is committed. To regenerate: call _markdown on the sample and dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)."
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## 套餐明细",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "注意：所有套餐均需空腹",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "## 注意事项",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- 体检前三天清淡饮食",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "- 避免剧烈运动！",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "下面是示例配置：",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "```yaml\nname: health-check\nversion: 1\n```",
|
||||
"doc_type_kwd": "text"
|
||||
},
|
||||
{
|
||||
"text": "示意图",
|
||||
"doc_type_kwd": "image"
|
||||
},
|
||||
{
|
||||
"text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n",
|
||||
"doc_type_kwd": "table"
|
||||
},
|
||||
{
|
||||
"text": "\n<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>\n",
|
||||
"doc_type_kwd": "table"
|
||||
}
|
||||
]
|
||||
}
|
||||
26
internal/parser/parser/testdata/markdown.sample.en.md
vendored
Normal file
26
internal/parser/parser/testdata/markdown.sample.en.md
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# Health Check Plan Comparison
|
||||
|
||||
This document compares two health check plans, covering tables, lists, and code blocks
|
||||
|
||||
## Plan Details
|
||||
|
||||
| Check Item | Basic 699 | Advanced 1299 |
|
||||
| --- | --- | --- |
|
||||
| Blood routine / Urine routine | Included | Included |
|
||||
| ECG | Not included | Included |
|
||||
|
||||
Note: all plans require fasting.
|
||||
|
||||
## Precautions
|
||||
|
||||
- Eat a light diet for three days before the exam?
|
||||
- Avoid strenuous exercise!
|
||||
|
||||
Here is a sample configuration:
|
||||
|
||||
```yaml
|
||||
name: health-check
|
||||
version: 1
|
||||
```
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# 健康检查套餐对比
|
||||
|
||||
本文比较两种体检套餐，包含表格、列表与代码块。
|
||||
本文比较两种体检套餐，包含表格、列表与代码块
|
||||
|
||||
## 套餐明细
|
||||
|
||||
Reference in New Issue
Block a user