diff --git a/deepdoc/parser/docling_parser.py b/deepdoc/parser/docling_parser.py index 56def78034..d0800ab6d2 100644 --- a/deepdoc/parser/docling_parser.py +++ b/deepdoc/parser/docling_parser.py @@ -347,6 +347,26 @@ class DoclingParser(RAGFlowPdfParser): return docs return [] + @staticmethod + def _looks_like_chunk_response(payload: Any) -> bool: + """Return True iff ``payload`` looks like a chunking endpoint response. + + A chunk response is either a non-empty top-level list or a dict that + carries a non-empty ``results`` or ``chunks`` list. A standard + conversion response (``{"document": ..., "status": ...}``) does not + match, so a server that silently ignored the ``do_chunking`` flag is + correctly classified as standard even when the request payload asked + for chunking. + """ + if isinstance(payload, list): + return bool(payload) + if isinstance(payload, dict): + for key in ("results", "chunks"): + value = payload.get(key) + if isinstance(value, list) and value: + return True + return False + def _parse_pdf_remote( self, filepath: str | PathLike[str], @@ -360,9 +380,13 @@ class DoclingParser(RAGFlowPdfParser): """ Parses a PDF document using a remote Docling server. - Prioritizes native chunking endpoints (/v1/chunk/source, /v1alpha/chunk/source) - to prevent token overflow, with a graceful fallback to standard conversion - endpoints if chunking is unavailable. + Sends the document with chunking options first, then falls back to a + standard conversion payload if the server rejects the chunking parameters. + The chunked-vs-standard parsing decision is made from the **response + shape**, not the request shape: Docling Serve silently drops unknown + fields such as ``do_chunking`` and returns a standard conversion + response, so the response is treated as standard even when chunking + was requested. """ server_url = self._effective_server_url(docling_server_url) if not server_url: @@ -437,10 +461,16 @@ class DoclingParser(RAGFlowPdfParser): ) if resp.status_code < 300: response_json = resp.json() - is_chunked_response = chunk_flag + response_is_chunk = self._looks_like_chunk_response(response_json) + is_chunked_response = chunk_flag and response_is_chunk - if chunk_flag: + if chunk_flag and response_is_chunk: self.logger.info(f"[Docling] Successfully used native chunking on: {endpoint}") + elif chunk_flag: + self.logger.warning( + f"[Docling] Server ignored chunking request on {endpoint}; " + "treating response as standard conversion." + ) else: self.logger.info(f"[Docling] Chunking unavailable, fell back to standard: {endpoint}") break diff --git a/test/unit_test/deepdoc/parser/test_docling_parser_remote.py b/test/unit_test/deepdoc/parser/test_docling_parser_remote.py index e3410c65e2..7cf0598491 100644 --- a/test/unit_test/deepdoc/parser/test_docling_parser_remote.py +++ b/test/unit_test/deepdoc/parser/test_docling_parser_remote.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import logging import sys import types from pathlib import Path @@ -15,8 +16,9 @@ class _Response: status_code = 200 text = "" - def __init__(self, payload): + def __init__(self, payload, status_code: int = 200): self._payload = payload + self.status_code = status_code def json(self): return self._payload @@ -74,3 +76,92 @@ def test_remote_chunked_200_standard_payload_falls_back(monkeypatch): assert sections == [("# Parsed\n\nbody", "")] assert tables == [] assert calls[0][0]["options"]["do_chunking"] is True + + +@pytest.mark.p2 +def test_chunk_shape_helper_recognises_chunk_payloads(monkeypatch): + """A response that is chunk-shaped (list, or dict with non-empty results/chunks) + is classified as chunked regardless of which payload was sent.""" + module = _load_docling_parser(monkeypatch) + assert module.DoclingParser._looks_like_chunk_response( + [{"text": "chunk-1"}] + ) is True + assert module.DoclingParser._looks_like_chunk_response( + {"results": [{"text": "chunk-1"}, {"text": "chunk-2"}]} + ) is True + assert module.DoclingParser._looks_like_chunk_response( + {"chunks": [{"text": "chunk-1"}]} + ) is True + + +@pytest.mark.p2 +def test_chunk_shape_helper_rejects_standard_payloads(monkeypatch): + """A standard conversion response, empty containers, and non-payload types + are correctly classified as not-chunked.""" + module = _load_docling_parser(monkeypatch) + standard = {"document": {"md_content": "body"}, "status": "success"} + assert module.DoclingParser._looks_like_chunk_response(standard) is False + assert module.DoclingParser._looks_like_chunk_response({}) is False + assert module.DoclingParser._looks_like_chunk_response({"results": []}) is False + assert module.DoclingParser._looks_like_chunk_response({"chunks": []}) is False + assert module.DoclingParser._looks_like_chunk_response([]) is False + assert module.DoclingParser._looks_like_chunk_response("not-a-payload") is False + assert module.DoclingParser._looks_like_chunk_response(None) is False + assert module.DoclingParser._looks_like_chunk_response(42) is False + + +@pytest.mark.p2 +def test_remote_chunked_request_with_results_list_is_treated_as_chunked(monkeypatch): + """A server that returns a ``results`` list (Docling Serve's native chunk + shape) is treated as chunked and each chunk becomes a section.""" + module = _load_docling_parser(monkeypatch) + + def fake_post(_url, json, timeout): + return _Response({"results": [{"text": "alpha"}, {"text": "beta"}]}) + + monkeypatch.setattr(module.requests, "post", fake_post) + + parser = module.DoclingParser(docling_server_url="http://docling.local") + sections, tables = parser._parse_pdf_remote("sample.pdf", binary=b"%PDF", parse_method="raw") + + assert sections == [("alpha", ""), ("beta", "")] + assert tables == [] + + +@pytest.mark.p2 +def test_remote_top_level_list_response_is_treated_as_chunked(monkeypatch): + """A server that returns a top-level JSON array of chunks is treated + as chunked (matches the existing implicit assumption in the code).""" + module = _load_docling_parser(monkeypatch) + + def fake_post(_url, json, timeout): + return _Response([{"text": "first"}, {"text": "second"}]) + + monkeypatch.setattr(module.requests, "post", fake_post) + + parser = module.DoclingParser(docling_server_url="http://docling.local") + sections, _ = parser._parse_pdf_remote("sample.pdf", binary=b"%PDF", parse_method="raw") + + assert sections == [("first", ""), ("second", "")] + + +@pytest.mark.p2 +def test_remote_chunked_request_with_ignored_flag_does_not_log_success(monkeypatch, caplog): + """When Docling Serve silently drops the ``do_chunking`` flag and returns + a standard conversion response, RAGFlow must not log a chunking-success + message and must log a warning instead.""" + module = _load_docling_parser(monkeypatch) + + def fake_post(_url, json, timeout): + return _Response({"document": {"md_content": "real content"}, "status": "success"}) + + monkeypatch.setattr(module.requests, "post", fake_post) + + parser = module.DoclingParser(docling_server_url="http://docling.local") + with caplog.at_level(logging.DEBUG, logger="DoclingParser"): + sections, _ = parser._parse_pdf_remote("sample.pdf", binary=b"%PDF", parse_method="raw") + + assert sections == [("real content", "")] + flat = " ".join(record.getMessage() for record in caplog.records) + assert "Successfully used native chunking" not in flat + assert "Server ignored chunking request" in flat \ No newline at end of file