mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-17 13:17:20 +08:00
### What problem does this PR solve? Closes #14058. RAGFlow supports multiple PDF parsing backends (DeepDOC, MinerU, Docling, TCADP, PaddleOCR). This PR adds **OpenDataLoader** ([opendataloader-project/opendataloader-pdf](https://github.com/opendataloader-project/opendataloader-pdf)) as a new optional backend, giving users a deterministic, local-first alternative with competitive table extraction accuracy. ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Documentation Update --- ### Changes #### Backend - `deepdoc/parser/opendataloader_parser.py` — new `OpenDataLoaderParser` class inheriting `RAGFlowPdfParser`. Implements `check_installation()` (guards Python package + Java 11+ runtime), `parse_pdf()` with JSON-first extraction (heading/paragraph/table/list/image/formula) and Markdown fallback, position-tag generation compatible with the shared `@@page\tx0\tx1\ty0\ty1##` format, and temp-dir lifecycle with cleanup. - `rag/app/naive.py` — new `by_opendataloader()` wrapper, registered in `PARSERS` dict, added to `chunk_token_num=0` override list. - `rag/flow/parser/parser.py` — `"opendataloader"` branch in the pipeline PDF handler + check validation list. #### Infrastructure - `docker/entrypoint.sh` — `ensure_opendataloader()` function: opt-in via `USE_OPENDATALOADER=true`, skips gracefully if Java is not on PATH. #### Frontend - `web/src/components/layout-recognize-form-field.tsx` — `OpenDataLoader` added to `ParseDocumentType` enum and parser dropdown. Cascades automatically to the pipeline editor's Parser component. #### Docs - `docs/guides/dataset/select_pdf_parser.md` — added OpenDataLoader entry and full env-var reference. --- ### Environment variables | Variable | Default | Description | |---|---|---| | `USE_OPENDATALOADER` | `false` | Set `true` to install `opendataloader-pdf` on container startup | | `OPENDATALOADER_VERSION` | latest | Pin the PyPI release (e.g. `==2.2.1`) | | `OPENDATALOADER_HYBRID` | _(unset)_ | Enable hybrid AI mode (e.g. `docling-fast`) | | `OPENDATALOADER_IMAGE_OUTPUT` | _(unset)_ | `off` / `embedded` / `external` | | `OPENDATALOADER_OUTPUT_DIR` | _(tmp)_ | Persistent output dir; temp dir used + cleaned if unset | | `OPENDATALOADER_DELETE_OUTPUT` | `1` | `0` to retain intermediate files for debugging | | `OPENDATALOADER_SANITIZE` | _(unset)_ | `1` to filter prompt-injection patterns from output | --- ### Dependencies - **Runtime**: `opendataloader-pdf` (PyPI, Apache 2.0) — opt-in, not added to `pyproject.toml` core deps. Installed by `ensure_opendataloader()` at container startup when `USE_OPENDATALOADER=true`. - **System**: Java 11+ on PATH (JVM is the underlying engine). The installer skips with a warning if `java` is not found. --- ### How to test **Standalone parser:** ```bash source .venv/bin/activate uv pip install opendataloader-pdf python3 -c " import sys; sys.path.insert(0, '.') from deepdoc.parser.opendataloader_parser import OpenDataLoaderParser p = OpenDataLoaderParser() print('available:', p.check_installation()) s, t = p.parse_pdf('path/to/test.pdf', parse_method='pipeline') print(f'sections={len(s)} tables={len(t)}') " ``` ### Benchmark vs Docling ``` file parser secs sections tables ---------------------------------------------------------------------- text-heavy.pdf docling 45.29 148 10 text-heavy.pdf opendataloader 3.14 559 0 table-heavy.pdf docling 7.05 76 3 table-heavy.pdf opendataloader 3.71 90 0 complex.pdf docling 42.67 114 8 complex.pdf opendataloader 3.51 180 0 ```
This commit is contained in:
wdeveloper16
@@ -240,7 +240,7 @@ class ParserParam(ProcessParamBase):
|
||||
pdf_parse_method = pdf_config.get("parse_method", "")
|
||||
self.check_empty(pdf_parse_method, "Parse method abnormal.")
|
||||
|
||||
if pdf_parse_method.lower() not in ["deepdoc", "plain_text", "mineru", "docling", "tcadp parser", "paddleocr"]:
|
||||
if pdf_parse_method.lower() not in ["deepdoc", "plain_text", "mineru", "docling", "opendataloader", "tcadp parser", "paddleocr"]:
|
||||
self.check_empty(pdf_config.get("lang", ""), "PDF VLM language")
|
||||
|
||||
pdf_output_format = pdf_config.get("output_format", "")
|
||||
@@ -434,6 +434,70 @@ class Parser(ProcessBase):
|
||||
box["image"] = image
|
||||
bboxes.append(box)
|
||||
|
||||
elif parse_method.lower() == "opendataloader":
|
||||
|
||||
def resolve_opendataloader_llm_name():
|
||||
configured = parser_model_name or conf.get("opendataloader_llm_name")
|
||||
if configured:
|
||||
return configured
|
||||
tenant_id = self._canvas._tenant_id
|
||||
if not tenant_id:
|
||||
return None
|
||||
from api.db.services.tenant_llm_service import TenantLLMService
|
||||
env_name = TenantLLMService.ensure_opendataloader_from_env(tenant_id)
|
||||
candidates = TenantLLMService.query(tenant_id=tenant_id, llm_factory="OpenDataLoader", model_type=LLMType.OCR.value)
|
||||
if candidates:
|
||||
return candidates[0].llm_name
|
||||
return env_name
|
||||
|
||||
parser_model_name = resolve_opendataloader_llm_name()
|
||||
if not parser_model_name:
|
||||
raise RuntimeError("OpenDataLoader model not configured. Please add OpenDataLoader in Model Providers.")
|
||||
|
||||
tenant_id = self._canvas._tenant_id
|
||||
ocr_model_config = get_model_config_by_type_and_name(tenant_id, LLMType.OCR, parser_model_name)
|
||||
ocr_model = LLMBundle(tenant_id, ocr_model_config)
|
||||
pdf_parser = ocr_model.mdl
|
||||
|
||||
lines, odl_tables = pdf_parser.parse_pdf(
|
||||
filepath=name,
|
||||
binary=blob,
|
||||
callback=self.callback,
|
||||
parse_method="pipeline",
|
||||
)
|
||||
bboxes = []
|
||||
for item in lines or []:
|
||||
if not isinstance(item, tuple) or len(item) < 3:
|
||||
continue
|
||||
text, layout_type, poss = item[0], item[1], item[2]
|
||||
box = {
|
||||
"text": text,
|
||||
"layout_type": layout_type or "text",
|
||||
}
|
||||
if isinstance(poss, str) and poss:
|
||||
positions = [[pos[0][-1] + 1, *pos[1:]] for pos in pdf_parser.extract_positions(poss)]
|
||||
if positions:
|
||||
box["positions"] = positions
|
||||
image = pdf_parser.crop(poss, 1)
|
||||
if image is not None:
|
||||
box["image"] = image
|
||||
bboxes.append(box)
|
||||
# Merge tables and images from the second return value.
|
||||
for (img, html_or_caption), positions in odl_tables or []:
|
||||
box = {"layout_type": "table" if not isinstance(html_or_caption, list) else "figure"}
|
||||
if isinstance(html_or_caption, str):
|
||||
box["text"] = html_or_caption
|
||||
elif isinstance(html_or_caption, list):
|
||||
box["text"] = html_or_caption[0] if html_or_caption else ""
|
||||
if img is not None:
|
||||
box["image"] = img
|
||||
if positions:
|
||||
try:
|
||||
box["positions"] = [[p[0] + 1, p[1], p[2], p[3], p[4]] for p in positions]
|
||||
except Exception:
|
||||
pass
|
||||
bboxes.append(box)
|
||||
|
||||
elif parse_method.lower() == "tcadp parser":
|
||||
# ADP is a document parsing tool using Tencent Cloud API
|
||||
table_result_type = conf.get("table_result_type", "1")
|
||||
|
||||
Reference in New Issue
Block a user