diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py index c94b9ee2ac..05ba5052a1 100644 --- a/deepdoc/parser/mineru_parser.py +++ b/deepdoc/parser/mineru_parser.py @@ -269,10 +269,10 @@ class MinerUParser(RAGFlowPdfParser): return True, reason - def _run_mineru(self, input_path: Path, output_dir: Path, options: MinerUParseOptions, callback: Optional[Callable] = None) -> Path: - return self._run_mineru_api(input_path, output_dir, options, callback) + def _run_mineru(self, input_path: Path, output_dir: Path, options: MinerUParseOptions, callback: Optional[Callable] = None, *, page_from: int = 0, page_to: int = MAXIMUM_PAGE_NUMBER) -> Path: + return self._run_mineru_api(input_path, output_dir, options, callback, page_from=page_from, page_to=page_to) - def _run_mineru_api(self, input_path: Path, output_dir: Path, options: MinerUParseOptions, callback: Optional[Callable] = None) -> Path: + def _run_mineru_api(self, input_path: Path, output_dir: Path, options: MinerUParseOptions, callback: Optional[Callable] = None, *, page_from: int = 0, page_to: int = MAXIMUM_PAGE_NUMBER) -> Path: pdf_file_path = str(input_path) if not os.path.exists(pdf_file_path): @@ -282,6 +282,14 @@ class MinerUParser(RAGFlowPdfParser): output_path = tempfile.mkdtemp(prefix=f"{pdf_file_name}_{options.method}_", dir=str(output_dir)) output_zip_path = os.path.join(str(output_dir), f"{Path(output_path).name}.zip") + # RAGFlow's page_to is exclusive (Python slice stop index), MinerU's + # end_page_id is 0-based inclusive. Translate at the API boundary so + # the dispatcher can pass its native range through unchanged. + if page_to == MAXIMUM_PAGE_NUMBER: + end_page_id = 99999 + else: + end_page_id = page_to - 1 + data = { "output_dir": "./output", "lang_list": options.lang, @@ -296,8 +304,8 @@ class MinerUParser(RAGFlowPdfParser): "return_content_list": True, "return_images": True, "response_format_zip": True, - "start_page_id": 0, - "end_page_id": 99999, + "start_page_id": page_from, + "end_page_id": end_page_id, } if options.server_url: @@ -305,6 +313,7 @@ class MinerUParser(RAGFlowPdfParser): elif self.mineru_server_url: data["server_url"] = self.mineru_server_url + self.logger.info(f"[MinerU] resolved page range start_page_id={page_from} end_page_id={end_page_id} (from page_from={page_from} page_to={page_to})") self.logger.info(f"[MinerU] request {data=}") self.logger.info(f"[MinerU] request {options=}") @@ -739,6 +748,8 @@ class MinerUParser(RAGFlowPdfParser): server_url: Optional[str] = None, delete_output: bool = True, parse_method: str = "raw", + page_from: int = 0, + page_to: int = MAXIMUM_PAGE_NUMBER, **kwargs, ) -> tuple: import shutil @@ -802,7 +813,7 @@ class MinerUParser(RAGFlowPdfParser): formula_enable=enable_formula, table_enable=enable_table, ) - final_out_dir = self._run_mineru(pdf, out_dir, options, callback=callback) + final_out_dir = self._run_mineru(pdf, out_dir, options, callback=callback, page_from=page_from, page_to=page_to) outputs = self._read_output(final_out_dir, pdf.stem, method=mineru_method_raw_str, backend=backend) self.logger.info(f"[MinerU] Parsed {len(outputs)} blocks from PDF.") if callback: diff --git a/rag/app/naive.py b/rag/app/naive.py index 93b572168a..5062864f4c 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -172,6 +172,8 @@ def by_mineru( callback=callback, parse_method=parse_method, lang=lang, + page_from=from_page, + page_to=min(to_page, MAXIMUM_PAGE_NUMBER), **kwargs, ) return sections, tables, pdf_parser diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.py index c7055ed31a..1706561672 100644 --- a/test/unit_test/deepdoc/parser/test_mineru_parser.py +++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py @@ -1,6 +1,7 @@ import importlib.util import logging import sys +from io import BytesIO from pathlib import Path from types import ModuleType @@ -113,3 +114,149 @@ def test_transfer_to_sections_skips_unknown_types_without_duplicating_text(monke assert [section[0] for section in sections] == ["Primary content", "Next content"] assert "Skip unsupported section type=sidebar" in caplog.text + + +class _FakeZipResponse: + """Stand-in for the streaming response returned by requests.post. + + Provides the minimum surface that _run_mineru_api touches: status code, + headers (Content-Type), and a `.raw` stream that copyfileobj can drain. + """ + + def __init__(self, body: bytes = b"zip-bytes"): + self._body = body + self.headers = {"Content-Type": "application/zip"} + self.raw = BytesIO(body) + + def raise_for_status(self): + return None + + +class _FakePostContext: + def __init__(self, response: _FakeZipResponse, captured: dict): + self._response = response + self._captured = captured + + def __enter__(self): + return self._response + + def __exit__(self, exc_type, exc, tb): + return False + + +def _capture_run_mineru_api(monkeypatch, module, *, pdf_path: Path, extracted_dir: Path): + """Stub everything around requests.post so _run_mineru_api runs end-to-end + against an in-memory response. Returns the captured kwargs dict. + """ + captured: dict = {} + + def fake_post(url, files, data, headers, timeout, stream): + captured["url"] = url + captured["data"] = data + captured["files"] = files + return _FakePostContext(_FakeZipResponse(), captured) + + monkeypatch.setattr(module.requests, "post", fake_post) + monkeypatch.setattr(module.os.path, "exists", lambda _p: True) + monkeypatch.setattr( + module.MinerUParser, + "_extract_zip_no_root", + lambda self, *_a, **_kw: None, + ) + monkeypatch.setattr( + module.shutil, + "copyfileobj", + lambda _src, _dst: None, + ) + import tempfile + + monkeypatch.setattr(tempfile, "mkdtemp", lambda prefix="", dir=None: str(extracted_dir)) + return captured + + +def test_run_mineru_api_threads_page_range_into_request_payload(monkeypatch, tmp_path): + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser(mineru_api="http://mineru.local") + parser.mineru_server_url = "" + + pdf_path = tmp_path / "sample.pdf" + pdf_path.write_bytes(b"%PDF-1.4 fake") + extracted_dir = tmp_path / "out" + extracted_dir.mkdir() + + captured = _capture_run_mineru_api(monkeypatch, module, pdf_path=pdf_path, extracted_dir=extracted_dir) + options = module.MinerUParseOptions() + + # Mid-document range: pages 0..12 inclusive in RAGFlow slice terms. + parser._run_mineru_api( + pdf_path, + extracted_dir, + options, + callback=None, + page_from=0, + page_to=13, + ) + + assert captured["data"]["start_page_id"] == 0 + assert captured["data"]["end_page_id"] == 12 + + # End-of-document range: still need the full doc to come back. + captured.clear() + parser._run_mineru_api( + pdf_path, + extracted_dir, + options, + callback=None, + page_from=5, + page_to=20, + ) + + assert captured["data"]["start_page_id"] == 5 + assert captured["data"]["end_page_id"] == 19 + + +def test_run_mineru_api_uses_full_document_when_no_range_given(monkeypatch, tmp_path): + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser(mineru_api="http://mineru.local") + parser.mineru_server_url = "" + + pdf_path = tmp_path / "sample.pdf" + pdf_path.write_bytes(b"%PDF-1.4 fake") + extracted_dir = tmp_path / "out" + extracted_dir.mkdir() + + captured = _capture_run_mineru_api(monkeypatch, module, pdf_path=pdf_path, extracted_dir=extracted_dir) + options = module.MinerUParseOptions() + + # No page_from/page_to: defaults should keep the prior behavior (0 / 99999). + parser._run_mineru_api(pdf_path, extracted_dir, options, callback=None) + + assert captured["data"]["start_page_id"] == 0 + assert captured["data"]["end_page_id"] == 99999 + + +def test_end_page_minus_one_normalizes_for_mineru_api(monkeypatch, tmp_path): + module = _load_mineru_parser(monkeypatch) + parser = module.MinerUParser(mineru_api="http://mineru.local") + parser.mineru_server_url = "" + + pdf_path = tmp_path / "sample.pdf" + pdf_path.write_bytes(b"%PDF-1.4 fake") + extracted_dir = tmp_path / "out" + extracted_dir.mkdir() + + captured = _capture_run_mineru_api(monkeypatch, module, pdf_path=pdf_path, extracted_dir=extracted_dir) + options = module.MinerUParseOptions() + + # RAGFlow to_page is exclusive (Python slice stop); MinerU end_page_id is + # 0-based inclusive, so to_page - 1 is the correct translation. + parser._run_mineru_api( + pdf_path, + extracted_dir, + options, + callback=None, + page_from=0, + page_to=13, + ) + + assert captured["data"]["end_page_id"] == 12