diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 996ce0ce67..b5f3d706d3 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -281,6 +281,84 @@ jobs: uv sync --python 3.13 --group test --frozen uv pip install -e sdk/python + - name: Provision NLTK data for unit tests + if: steps.detect_changes.outputs.has_python_changes == 'true' + run: | + set -euo pipefail + + # The self-hosted runner stores NLTK packages as root-owned zip files. + # Reading them directly as the runner user fails with EACCES, while + # downloading replacements is blocked by the runner's SSRF policy. + # Copy the three required archives into the workspace with sudo, then + # unpack and validate them as the runner user. + NLTK_TARGET="${GITHUB_WORKSPACE}/nltk_data" + rm -rf "${NLTK_TARGET}" + mkdir -p "${NLTK_TARGET}/tokenizers" "${NLTK_TARGET}/corpora" + + copy_nltk_archive() { + local relative_path="$1" + local source="" + for root in /usr/share/nltk_data /usr/local/share/nltk_data; do + if sudo test -f "${root}/${relative_path}"; then + source="${root}/${relative_path}" + break + fi + done + if [ -z "${source}" ]; then + echo "Missing required NLTK archive: ${relative_path}" >&2 + return 1 + fi + echo "Copying ${source}" + sudo cp "${source}" "${NLTK_TARGET}/${relative_path}" + sudo chown "$(id -u):$(id -g)" "${NLTK_TARGET}/${relative_path}" + chmod 0644 "${NLTK_TARGET}/${relative_path}" + } + + copy_nltk_archive tokenizers/punkt_tab.zip + copy_nltk_archive tokenizers/punkt.zip + copy_nltk_archive corpora/wordnet.zip + + NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF' + import os + import zipfile + + root = os.environ["NLTK_DATA"] + archives = ( + ("tokenizers", "punkt_tab"), + ("tokenizers", "punkt"), + ("corpora", "wordnet"), + ) + for category, package in archives: + archive = os.path.join(root, category, f"{package}.zip") + destination = os.path.join(root, category) + with zipfile.ZipFile(archive) as package_zip: + bad_member = package_zip.testzip() + if bad_member is not None: + raise RuntimeError(f"Corrupt NLTK archive {archive}: {bad_member}") + members = [name for name in package_zip.namelist() if not name.startswith("__MACOSX/")] + prefix = f"{package}/" + if not members or not all(name == package or name.startswith(prefix) for name in members): + raise RuntimeError(f"Unexpected layout in {archive}: expected entries below {prefix}") + package_zip.extractall(destination) + print(f"Unpacked {archive} -> {os.path.join(destination, package)}") + PYEOF + + echo "NLTK_DATA=${NLTK_TARGET}" >> "${GITHUB_ENV}" + + # Validate real tokenizer/lemmatizer calls, not only directory names. + # The command exits non-zero if any resource is absent or malformed. + NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF' + import nltk + from nltk.corpus import wordnet + from nltk.tokenize import word_tokenize + + for resource in ("tokenizers/punkt_tab", "tokenizers/punkt", "corpora/wordnet"): + print(f"[OK] {resource} -> {nltk.data.find(resource)}") + assert word_tokenize("NLTK data is ready.") + assert wordnet.synsets("document") + print("NLTK tokenizer and WordNet smoke checks passed") + PYEOF + - name: Run unit test if: steps.detect_changes.outputs.has_python_changes == 'true' run: | diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py index 373d0e2c6d..cf40876820 100644 --- a/deepdoc/parser/mineru_parser.py +++ b/deepdoc/parser/mineru_parser.py @@ -56,6 +56,11 @@ class MinerUContentType(StrEnum): FOOTER = "footer" PAGE_NUMBER = "page_number" DISCARDED = "discarded" + # MinerU 3.4.x VLM backend emits chart blocks for figures whose visual is a + # chart rather than a plain image. They carry chart_caption / chart_footnote + # / sub_type / img_path / bbox / page_idx — the same shape as IMAGE blocks — + # so they are routed through the image pipeline instead of being dropped. + CHART = "chart" # Mapping from language names to MinerU language codes @@ -848,7 +853,9 @@ class MinerUParser(RAGFlowPdfParser): output_type = output.get("type") # These chunkers consume tables and images separately, so exclude # media from their text sections. Raw consumers keep legacy sections. - if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)): + # Charts are routed the same way as images: MinerU emits them as + # visual blocks that _transfer_to_tables turns into image chunks. + if parse_method in {"naive", "manual", "paper"} and (output_type in {MinerUContentType.IMAGE, MinerUContentType.CHART} or (output_type == MinerUContentType.TABLE and table_enable)): continue match output_type: @@ -867,6 +874,11 @@ class MinerUParser(RAGFlowPdfParser): vlm_description = (output.get("vlm_description") or "").strip() if vlm_description: section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description + case MinerUContentType.CHART: + section = "".join(output.get("chart_caption", [])) + "\n" + "".join(output.get("chart_footnote", [])) + vlm_description = (output.get("vlm_description") or "").strip() + if vlm_description: + section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description case MinerUContentType.EQUATION: section = output.get("text", "") case MinerUContentType.CODE: @@ -886,7 +898,7 @@ class MinerUParser(RAGFlowPdfParser): case MinerUContentType.HEADER | MinerUContentType.FOOTER | MinerUContentType.PAGE_NUMBER | MinerUContentType.DISCARDED: continue case _: - self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type")) + self.logger.warning("[MinerU] Skip unsupported section type=%s", output.get("type")) continue # Only flatten table HTML when table extraction is disabled; the @@ -915,7 +927,7 @@ class MinerUParser(RAGFlowPdfParser): tables = [] for output in outputs: output_type = output.get("type") - if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}: + if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE, MinerUContentType.CHART}: continue if output_type == MinerUContentType.TABLE and not table_enable: continue @@ -933,7 +945,12 @@ class MinerUParser(RAGFlowPdfParser): tables.append(((None, text), positions)) continue - texts = [*output.get("image_caption", []), *output.get("image_footnote", [])] + # IMAGE and CHART share the same visual pipeline; only the caption + # field names differ (image_caption/image_footnote vs chart_*). + if output_type == MinerUContentType.CHART: + texts = [*output.get("chart_caption", []), *output.get("chart_footnote", [])] + else: + texts = [*output.get("image_caption", []), *output.get("image_footnote", [])] vlm_description = (output.get("vlm_description") or "").strip() if vlm_description: texts.append(vlm_description) @@ -957,15 +974,17 @@ class MinerUParser(RAGFlowPdfParser): def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"): """Generate semantic descriptions for image blocks via the tenant's VISION model, mirroring deepdoc's VisionFigureParser. Each - IMAGE block with a readable img_path gets a ``vlm_description`` - field that ``_transfer_to_sections`` then folds into the chunk - text — closing issue #14869. + IMAGE or CHART block with a readable img_path gets a + ``vlm_description`` field that ``_transfer_to_sections`` then folds + into the chunk text — closing issue #14869. """ from concurrent.futures import ThreadPoolExecutor, as_completed from rag.app.picture import vision_llm_chunk from rag.prompts.generator import vision_llm_figure_describe_prompt - image_jobs = [(idx, item) for idx, item in enumerate(outputs) if item.get("type") == MinerUContentType.IMAGE and item.get("img_path") and os.path.exists(item["img_path"])] + image_jobs = [ + (idx, item) for idx, item in enumerate(outputs) if item.get("type") in {MinerUContentType.IMAGE, MinerUContentType.CHART} and item.get("img_path") and os.path.exists(item["img_path"]) + ] if not image_jobs: return diff --git a/test/unit_test/conftest.py b/test/unit_test/conftest.py index 512398c591..19a57c8471 100644 --- a/test/unit_test/conftest.py +++ b/test/unit_test/conftest.py @@ -49,13 +49,10 @@ if _LOCAL_NLTK_DATA not in nltk.data.path: nltk.data.path.insert(0, _LOCAL_NLTK_DATA) # (download name, resource path used by nltk.data.find) -# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data -# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises -# LookupError at load time; `omw-1.4` must also be present. _REQUIRED_NLTK_DATA = ( ("punkt_tab", "tokenizers/punkt_tab"), + ("punkt", "tokenizers/punkt"), ("wordnet", "corpora/wordnet"), - ("omw-1.4", "corpora/omw-1.4"), ) for _name, _find_path in _REQUIRED_NLTK_DATA: try: diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.py index 3a8e206f54..fc974a87b6 100644 --- a/test/unit_test/deepdoc/parser/test_mineru_parser.py +++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py @@ -412,6 +412,86 @@ def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path): assert [chunk["page_num_int"] for chunk in chunks] == [[13, 14], [13], [14]] +@pytest.mark.p1 +def test_transfer_to_tables_emits_chart_as_image_chunk(monkeypatch, tmp_path): + """MinerU 3.4.x VLM chart blocks (chart_caption/chart_footnote/img_path) + must surface as image chunks instead of being silently dropped (#19080).""" + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser() + parser.page_from = 0 + chart_path = tmp_path / "chart.png" + module.Image.new("RGB", (2, 2), "blue").save(chart_path) + outputs = [ + { + "type": module.MinerUContentType.CHART, + "img_path": str(chart_path), + "chart_caption": ["Figure 3"], + "chart_footnote": ["Source: dataset"], + "sub_type": "line", + "vlm_description": "A blue square", + "page_idx": 0, + "bbox": (1, 2, 3, 4), + }, + { + "type": module.MinerUContentType.CHART, + "chart_caption": ["Caption without image"], + "chart_footnote": [], + }, + ] + + media = parser._transfer_to_tables(outputs) + + # The chart with a readable image becomes an image chunk; the chart without + # an img_path is skipped (mirrors how IMAGE blocks behave). + assert len(media) == 1 + image, texts = media[0][0] + chart_path.unlink() + assert isinstance(image, module.Image.Image) + assert image.getpixel((0, 0)) == (0, 0, 255) + assert texts == ["Figure 3", "Source: dataset", "A blue square"] + + +@pytest.mark.p1 +def test_transfer_to_sections_routes_chart_like_image_per_parse_method(monkeypatch): + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser() + outputs = [ + {"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)}, + { + "type": module.MinerUContentType.CHART, + "chart_caption": ["figure"], + "chart_footnote": [], + "page_idx": 0, + "bbox": (0, 2, 1, 3), + }, + ] + + # app chunkers consume media separately: the chart is excluded from text sections. + for app_method in ("naive", "manual", "paper"): + sections = parser._transfer_to_sections(outputs, parse_method=app_method, table_enable=True) + assert len(sections) == 1 + assert sections[0][0].startswith("Body") + + # raw consumers keep the chart as a text section (caption/footnote), like IMAGE. + raw_sections = parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True) + assert len(raw_sections) == 2 + assert raw_sections[1][0].strip() == "figure" + + +@pytest.mark.p1 +def test_transfer_to_sections_warns_on_unknown_type(monkeypatch, caplog): + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser() + outputs = [ + {"type": "sidebar", "text": "ignored", "page_idx": 0, "bbox": (0, 0, 1, 1)}, + ] + + with caplog.at_level(logging.WARNING, logger=parser.logger.name): + parser._transfer_to_sections(outputs, parse_method="raw") + + assert "Skip unsupported section type=sidebar" in caplog.text + + @pytest.mark.p1 def test_tokenize_table_uses_payload_type_instead_of_html_content(): from PIL import Image