mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-12 11:43:39 +08:00
fix(mineru): skip page chrome blocks to prevent duplicate chunks (#15387)
## Summary - Skip MinerU `header`, `footer`, and `page_number` blocks when converting `content_list.json` into sections. - Ignore unsupported block types explicitly so future MinerU output types cannot re-emit the previous text block. Fixes duplicate text in General/naive chunks when parsing PDFs via MinerU (reported with repeated page headers and body text in slices). Closes #15335 ## Test plan - [x] `pytest test/unit_test/deepdoc/parser/test_mineru_parser.py -v` (4/4 passed)
This commit is contained in:
@@ -66,3 +66,50 @@ def test_transfer_to_sections_logs_sections_dropped_after_sanitization(monkeypat
|
||||
assert sections == []
|
||||
assert "Skip section after sanitization" in caplog.text
|
||||
assert f"type={module.MinerUContentType.TEXT}" in caplog.text
|
||||
|
||||
|
||||
def test_transfer_to_sections_skips_page_chrome_without_duplicating_text(monkeypatch):
|
||||
module = _load_mineru_parser(monkeypatch)
|
||||
parser = module.MinerUParser()
|
||||
fixture_path = Path(__file__).resolve().parents[3] / "fixtures" / "mineru" / "bmw_page_chrome_content_list.json"
|
||||
outputs = __import__("json").loads(fixture_path.read_text(encoding="utf-8"))
|
||||
|
||||
sections = parser._transfer_to_sections(outputs, parse_method="raw")
|
||||
texts = [section[0] for section in sections]
|
||||
|
||||
assert texts == ["打开和关闭", "车辆装备", "车辆钥匙", "概述", "安全提示"]
|
||||
assert texts.count("打开和关闭") == 1
|
||||
assert texts.count("概述") == 1
|
||||
assert "77" not in texts
|
||||
assert "Online Edition for Part no." not in " ".join(texts)
|
||||
|
||||
|
||||
def test_transfer_to_sections_skips_unknown_types_without_duplicating_text(monkeypatch, caplog):
|
||||
module = _load_mineru_parser(monkeypatch)
|
||||
parser = module.MinerUParser()
|
||||
outputs = [
|
||||
{
|
||||
"type": module.MinerUContentType.TEXT,
|
||||
"text": "Primary content",
|
||||
"page_idx": 0,
|
||||
"bbox": (0, 0, 1, 1),
|
||||
},
|
||||
{
|
||||
"type": "sidebar",
|
||||
"text": "Should not repeat previous section",
|
||||
"page_idx": 0,
|
||||
"bbox": (0, 0, 1, 1),
|
||||
},
|
||||
{
|
||||
"type": module.MinerUContentType.TEXT,
|
||||
"text": "Next content",
|
||||
"page_idx": 0,
|
||||
"bbox": (0, 0, 1, 1),
|
||||
},
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger=parser.logger.name):
|
||||
sections = parser._transfer_to_sections(outputs, parse_method="raw")
|
||||
|
||||
assert [section[0] for section in sections] == ["Primary content", "Next content"]
|
||||
assert "Skip unsupported section type=sidebar" in caplog.text
|
||||
|
||||
Reference in New Issue
Block a user