zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-09-08 02:08:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(parser): handle MinerU chart blocks instead of silently dropping them (#19096)

Browse Source
This commit is contained in:
Paul Yao
2026-09-05 11:03:27 +08:00
committed by GitHub
parent 27faeae983
commit 0c28d59ea1
4 changed files with 186 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
.github/workflows/sep-tests.yml vendored
View File

@@ -281,6 +281,84 @@ jobs:
uv sync --python 3.13 --group test --frozen
uv pip install -e sdk/python
- name: Provision NLTK data for unit tests
if: steps.detect_changes.outputs.has_python_changes == 'true'
run: |
set -euo pipefail
# The self-hosted runner stores NLTK packages as root-owned zip files.
# Reading them directly as the runner user fails with EACCES, while
# downloading replacements is blocked by the runner's SSRF policy.
# Copy the three required archives into the workspace with sudo, then
# unpack and validate them as the runner user.
NLTK_TARGET="${GITHUB_WORKSPACE}/nltk_data"
rm -rf "${NLTK_TARGET}"
mkdir -p "${NLTK_TARGET}/tokenizers" "${NLTK_TARGET}/corpora"
copy_nltk_archive() {
local relative_path="$1"
local source=""
for root in /usr/share/nltk_data /usr/local/share/nltk_data; do
if sudo test -f "${root}/${relative_path}"; then
source="${root}/${relative_path}"
break
fi
done
if [ -z "${source}" ]; then
echo "Missing required NLTK archive: ${relative_path}" >&2
return 1
fi
echo "Copying ${source}"
sudo cp "${source}" "${NLTK_TARGET}/${relative_path}"
sudo chown "$(id -u):$(id -g)" "${NLTK_TARGET}/${relative_path}"
chmod 0644 "${NLTK_TARGET}/${relative_path}"
}
copy_nltk_archive tokenizers/punkt_tab.zip
copy_nltk_archive tokenizers/punkt.zip
copy_nltk_archive corpora/wordnet.zip
NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
import os
import zipfile
root = os.environ["NLTK_DATA"]
archives = (
("tokenizers", "punkt_tab"),
("tokenizers", "punkt"),
("corpora", "wordnet"),
)
for category, package in archives:
archive = os.path.join(root, category, f"{package}.zip")
destination = os.path.join(root, category)
with zipfile.ZipFile(archive) as package_zip:
bad_member = package_zip.testzip()
if bad_member is not None:
raise RuntimeError(f"Corrupt NLTK archive {archive}: {bad_member}")
members = [name for name in package_zip.namelist() if not name.startswith("__MACOSX/")]
prefix = f"{package}/"
if not members or not all(name == package or name.startswith(prefix) for name in members):
raise RuntimeError(f"Unexpected layout in {archive}: expected entries below {prefix}")
package_zip.extractall(destination)
print(f"Unpacked {archive} -> {os.path.join(destination, package)}")
PYEOF
echo "NLTK_DATA=${NLTK_TARGET}" >> "${GITHUB_ENV}"
# Validate real tokenizer/lemmatizer calls, not only directory names.
# The command exits non-zero if any resource is absent or malformed.
NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
import nltk
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize
for resource in ("tokenizers/punkt_tab", "tokenizers/punkt", "corpora/wordnet"):
print(f"[OK] {resource} -> {nltk.data.find(resource)}")
assert word_tokenize("NLTK data is ready.")
assert wordnet.synsets("document")
print("NLTK tokenizer and WordNet smoke checks passed")
PYEOF
- name: Run unit test
if: steps.detect_changes.outputs.has_python_changes == 'true'
run: |

35
deepdoc/parser/mineru_parser.py
View File

@@ -56,6 +56,11 @@ class MinerUContentType(StrEnum):
FOOTER = "footer"
PAGE_NUMBER = "page_number"
DISCARDED = "discarded"
# MinerU 3.4.x VLM backend emits chart blocks for figures whose visual is a
# chart rather than a plain image. They carry chart_caption / chart_footnote
# / sub_type / img_path / bbox / page_idx — the same shape as IMAGE blocks —
# so they are routed through the image pipeline instead of being dropped.
CHART = "chart"
# Mapping from language names to MinerU language codes
@@ -848,7 +853,9 @@ class MinerUParser(RAGFlowPdfParser):
output_type = output.get("type")
# These chunkers consume tables and images separately, so exclude
# media from their text sections. Raw consumers keep legacy sections.
if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)):
# Charts are routed the same way as images: MinerU emits them as
# visual blocks that _transfer_to_tables turns into image chunks.
if parse_method in {"naive", "manual", "paper"} and (output_type in {MinerUContentType.IMAGE, MinerUContentType.CHART} or (output_type == MinerUContentType.TABLE and table_enable)):
continue
match output_type:
@@ -867,6 +874,11 @@ class MinerUParser(RAGFlowPdfParser):
vlm_description = (output.get("vlm_description") or "").strip()
if vlm_description:
section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description
case MinerUContentType.CHART:
section = "".join(output.get("chart_caption", [])) + "\n" + "".join(output.get("chart_footnote", []))
vlm_description = (output.get("vlm_description") or "").strip()
if vlm_description:
section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description
case MinerUContentType.EQUATION:
section = output.get("text", "")
case MinerUContentType.CODE:
@@ -886,7 +898,7 @@ class MinerUParser(RAGFlowPdfParser):
case MinerUContentType.HEADER | MinerUContentType.FOOTER | MinerUContentType.PAGE_NUMBER | MinerUContentType.DISCARDED:
continue
case _:
self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type"))
self.logger.warning("[MinerU] Skip unsupported section type=%s", output.get("type"))
continue
# Only flatten table HTML when table extraction is disabled; the
@@ -915,7 +927,7 @@ class MinerUParser(RAGFlowPdfParser):
tables = []
for output in outputs:
output_type = output.get("type")
if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}:
if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE, MinerUContentType.CHART}:
continue
if output_type == MinerUContentType.TABLE and not table_enable:
continue
@@ -933,7 +945,12 @@ class MinerUParser(RAGFlowPdfParser):
tables.append(((None, text), positions))
continue
texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]
# IMAGE and CHART share the same visual pipeline; only the caption
# field names differ (image_caption/image_footnote vs chart_*).
if output_type == MinerUContentType.CHART:
texts = [*output.get("chart_caption", []), *output.get("chart_footnote", [])]
else:
texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]
vlm_description = (output.get("vlm_description") or "").strip()
if vlm_description:
texts.append(vlm_description)
@@ -957,15 +974,17 @@ class MinerUParser(RAGFlowPdfParser):
def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):
"""Generate semantic descriptions for image blocks via the tenant's
VISION model, mirroring deepdoc's VisionFigureParser. Each
IMAGE block with a readable img_path gets a ``vlm_description``
field that ``_transfer_to_sections`` then folds into the chunk
text — closing issue #14869.
IMAGE or CHART block with a readable img_path gets a
``vlm_description`` field that ``_transfer_to_sections`` then folds
into the chunk text — closing issue #14869.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
from rag.app.picture import vision_llm_chunk
from rag.prompts.generator import vision_llm_figure_describe_prompt
image_jobs = [(idx, item) for idx, item in enumerate(outputs) if item.get("type") == MinerUContentType.IMAGE and item.get("img_path") and os.path.exists(item["img_path"])]
image_jobs = [
(idx, item) for idx, item in enumerate(outputs) if item.get("type") in {MinerUContentType.IMAGE, MinerUContentType.CHART} and item.get("img_path") and os.path.exists(item["img_path"])
]
if not image_jobs:
return

5
test/unit_test/conftest.py
View File

@@ -49,13 +49,10 @@ if _LOCAL_NLTK_DATA not in nltk.data.path:
nltk.data.path.insert(0, _LOCAL_NLTK_DATA)
# (download name, resource path used by nltk.data.find)
# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data
# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises
# LookupError at load time; `omw-1.4` must also be present.
_REQUIRED_NLTK_DATA = (
("punkt_tab", "tokenizers/punkt_tab"),
("punkt", "tokenizers/punkt"),
("wordnet", "corpora/wordnet"),
("omw-1.4", "corpora/omw-1.4"),
)
for _name, _find_path in _REQUIRED_NLTK_DATA:
try:

80
test/unit_test/deepdoc/parser/test_mineru_parser.py
View File

@@ -412,6 +412,86 @@ def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path):
assert [chunk["page_num_int"] for chunk in chunks] == [[13, 14], [13], [14]]
@pytest.mark.p1
def test_transfer_to_tables_emits_chart_as_image_chunk(monkeypatch, tmp_path):
"""MinerU 3.4.x VLM chart blocks (chart_caption/chart_footnote/img_path)
must surface as image chunks instead of being silently dropped (#19080)."""
module = _load_mineru_parser(monkeypatch)
parser = module.MinerUParser()
parser.page_from = 0
chart_path = tmp_path / "chart.png"
module.Image.new("RGB", (2, 2), "blue").save(chart_path)
outputs = [
{
"type": module.MinerUContentType.CHART,
"img_path": str(chart_path),
"chart_caption": ["Figure 3"],
"chart_footnote": ["Source: dataset"],
"sub_type": "line",
"vlm_description": "A blue square",
"page_idx": 0,
"bbox": (1, 2, 3, 4),
},
{
"type": module.MinerUContentType.CHART,
"chart_caption": ["Caption without image"],
"chart_footnote": [],
},
]
media = parser._transfer_to_tables(outputs)
# The chart with a readable image becomes an image chunk; the chart without
# an img_path is skipped (mirrors how IMAGE blocks behave).
assert len(media) == 1
image, texts = media[0][0]
chart_path.unlink()
assert isinstance(image, module.Image.Image)
assert image.getpixel((0, 0)) == (0, 0, 255)
assert texts == ["Figure 3", "Source: dataset", "A blue square"]
@pytest.mark.p1
def test_transfer_to_sections_routes_chart_like_image_per_parse_method(monkeypatch):
module = _load_mineru_parser(monkeypatch)
parser = module.MinerUParser()
outputs = [
{"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)},
{
"type": module.MinerUContentType.CHART,
"chart_caption": ["figure"],
"chart_footnote": [],
"page_idx": 0,
"bbox": (0, 2, 1, 3),
},
]
# app chunkers consume media separately: the chart is excluded from text sections.
for app_method in ("naive", "manual", "paper"):
sections = parser._transfer_to_sections(outputs, parse_method=app_method, table_enable=True)
assert len(sections) == 1
assert sections[0][0].startswith("Body")
# raw consumers keep the chart as a text section (caption/footnote), like IMAGE.
raw_sections = parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True)
assert len(raw_sections) == 2
assert raw_sections[1][0].strip() == "figure"
@pytest.mark.p1
def test_transfer_to_sections_warns_on_unknown_type(monkeypatch, caplog):
module = _load_mineru_parser(monkeypatch)
parser = module.MinerUParser()
outputs = [
{"type": "sidebar", "text": "ignored", "page_idx": 0, "bbox": (0, 0, 1, 1)},
]
with caplog.at_level(logging.WARNING, logger=parser.logger.name):
parser._transfer_to_sections(outputs, parse_method="raw")
assert "Skip unsupported section type=sidebar" in caplog.text
@pytest.mark.p1
def test_tokenize_table_uses_payload_type_instead_of_html_content():
from PIL import Image