mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-01 13:33:48 +08:00
### What problem does this PR solve? Closes #14058. RAGFlow supports multiple PDF parsing backends (DeepDOC, MinerU, Docling, TCADP, PaddleOCR). This PR adds **OpenDataLoader** ([opendataloader-project/opendataloader-pdf](https://github.com/opendataloader-project/opendataloader-pdf)) as a new optional backend, giving users a deterministic, local-first alternative with competitive table extraction accuracy. ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Documentation Update --- ### Changes #### Backend - `deepdoc/parser/opendataloader_parser.py` — new `OpenDataLoaderParser` class inheriting `RAGFlowPdfParser`. Implements `check_installation()` (guards Python package + Java 11+ runtime), `parse_pdf()` with JSON-first extraction (heading/paragraph/table/list/image/formula) and Markdown fallback, position-tag generation compatible with the shared `@@page\tx0\tx1\ty0\ty1##` format, and temp-dir lifecycle with cleanup. - `rag/app/naive.py` — new `by_opendataloader()` wrapper, registered in `PARSERS` dict, added to `chunk_token_num=0` override list. - `rag/flow/parser/parser.py` — `"opendataloader"` branch in the pipeline PDF handler + check validation list. #### Infrastructure - `docker/entrypoint.sh` — `ensure_opendataloader()` function: opt-in via `USE_OPENDATALOADER=true`, skips gracefully if Java is not on PATH. #### Frontend - `web/src/components/layout-recognize-form-field.tsx` — `OpenDataLoader` added to `ParseDocumentType` enum and parser dropdown. Cascades automatically to the pipeline editor's Parser component. #### Docs - `docs/guides/dataset/select_pdf_parser.md` — added OpenDataLoader entry and full env-var reference. --- ### Environment variables | Variable | Default | Description | |---|---|---| | `USE_OPENDATALOADER` | `false` | Set `true` to install `opendataloader-pdf` on container startup | | `OPENDATALOADER_VERSION` | latest | Pin the PyPI release (e.g. `==2.2.1`) | | `OPENDATALOADER_HYBRID` | _(unset)_ | Enable hybrid AI mode (e.g. `docling-fast`) | | `OPENDATALOADER_IMAGE_OUTPUT` | _(unset)_ | `off` / `embedded` / `external` | | `OPENDATALOADER_OUTPUT_DIR` | _(tmp)_ | Persistent output dir; temp dir used + cleaned if unset | | `OPENDATALOADER_DELETE_OUTPUT` | `1` | `0` to retain intermediate files for debugging | | `OPENDATALOADER_SANITIZE` | _(unset)_ | `1` to filter prompt-injection patterns from output | --- ### Dependencies - **Runtime**: `opendataloader-pdf` (PyPI, Apache 2.0) — opt-in, not added to `pyproject.toml` core deps. Installed by `ensure_opendataloader()` at container startup when `USE_OPENDATALOADER=true`. - **System**: Java 11+ on PATH (JVM is the underlying engine). The installer skips with a warning if `java` is not found. --- ### How to test **Standalone parser:** ```bash source .venv/bin/activate uv pip install opendataloader-pdf python3 -c " import sys; sys.path.insert(0, '.') from deepdoc.parser.opendataloader_parser import OpenDataLoaderParser p = OpenDataLoaderParser() print('available:', p.check_installation()) s, t = p.parse_pdf('path/to/test.pdf', parse_method='pipeline') print(f'sections={len(s)} tables={len(t)}') " ``` ### Benchmark vs Docling ``` file parser secs sections tables ---------------------------------------------------------------------- text-heavy.pdf docling 45.29 148 10 text-heavy.pdf opendataloader 3.14 559 0 table-heavy.pdf docling 7.05 76 3 table-heavy.pdf opendataloader 3.71 90 0 complex.pdf docling 42.67 114 8 complex.pdf opendataloader 3.51 180 0 ```
131 lines
3.8 KiB
TypeScript
131 lines
3.8 KiB
TypeScript
import { LlmModelType } from '@/constants/knowledge';
|
|
import { useTranslate } from '@/hooks/common-hooks';
|
|
import { useSelectLlmOptionsByModelType } from '@/hooks/use-llm-request';
|
|
import { cn } from '@/lib/utils';
|
|
import { camelCase } from 'lodash';
|
|
import { ReactNode, useMemo } from 'react';
|
|
import { useFormContext } from 'react-hook-form';
|
|
import { MinerUOptionsFormField } from './mineru-options-form-field';
|
|
import { SelectWithSearch } from './originui/select-with-search';
|
|
import { PaddleOCROptionsFormField } from './paddleocr-options-form-field';
|
|
import {
|
|
FormControl,
|
|
FormField,
|
|
FormItem,
|
|
FormLabel,
|
|
FormMessage,
|
|
} from './ui/form';
|
|
|
|
export const enum ParseDocumentType {
|
|
DeepDOC = 'DeepDOC',
|
|
PlainText = 'Plain Text',
|
|
Docling = 'Docling',
|
|
OpenDataLoader = 'OpenDataLoader',
|
|
TCADPParser = 'TCADP Parser',
|
|
}
|
|
|
|
export function LayoutRecognizeFormField({
|
|
name = 'parser_config.layout_recognize',
|
|
horizontal = true,
|
|
optionsWithoutLLM,
|
|
label,
|
|
showMineruOptions = true,
|
|
showPaddleocrOptions = true,
|
|
testId,
|
|
}: {
|
|
name?: string;
|
|
horizontal?: boolean;
|
|
optionsWithoutLLM?: { value: string; label: string }[];
|
|
label?: ReactNode;
|
|
showMineruOptions?: boolean;
|
|
showPaddleocrOptions?: boolean;
|
|
testId?: string;
|
|
}) {
|
|
const form = useFormContext();
|
|
|
|
const { t } = useTranslate('knowledgeDetails');
|
|
const allOptions = useSelectLlmOptionsByModelType();
|
|
|
|
const options = useMemo(() => {
|
|
const list = optionsWithoutLLM
|
|
? optionsWithoutLLM
|
|
: [
|
|
ParseDocumentType.DeepDOC,
|
|
ParseDocumentType.PlainText,
|
|
ParseDocumentType.Docling,
|
|
ParseDocumentType.OpenDataLoader,
|
|
ParseDocumentType.TCADPParser,
|
|
].map((x) => ({
|
|
label: x === ParseDocumentType.PlainText ? t(camelCase(x)) : x,
|
|
value: x,
|
|
}));
|
|
|
|
const image2TextList = [
|
|
...allOptions[LlmModelType.Image2text],
|
|
...allOptions[LlmModelType.Ocr],
|
|
].map((x) => {
|
|
return {
|
|
...x,
|
|
options: x.options.map((y) => {
|
|
return {
|
|
...y,
|
|
label: (
|
|
<div className="flex justify-between items-center gap-2">
|
|
{y.label}
|
|
<span className="text-red-500 text-sm">Experimental</span>
|
|
</div>
|
|
),
|
|
};
|
|
}),
|
|
};
|
|
});
|
|
|
|
return [...list, ...image2TextList];
|
|
}, [allOptions, optionsWithoutLLM, t]);
|
|
|
|
return (
|
|
<FormField
|
|
control={form.control}
|
|
name={name}
|
|
render={({ field }) => {
|
|
return (
|
|
<>
|
|
<FormItem className={'items-center space-y-0 '}>
|
|
<div
|
|
className={cn('flex', {
|
|
'flex-col ': !horizontal,
|
|
'items-center': horizontal,
|
|
})}
|
|
>
|
|
<FormLabel
|
|
tooltip={t('layoutRecognizeTip')}
|
|
className={cn('text-sm text-text-secondary whitespace-wrap', {
|
|
['w-1/4']: horizontal,
|
|
})}
|
|
>
|
|
{label || t('layoutRecognize')}
|
|
</FormLabel>
|
|
<div className={horizontal ? 'w-3/4' : 'w-full'}>
|
|
<FormControl>
|
|
<SelectWithSearch
|
|
{...field}
|
|
options={options}
|
|
testId={testId}
|
|
></SelectWithSearch>
|
|
</FormControl>
|
|
</div>
|
|
</div>
|
|
<div className="flex pt-1">
|
|
<div className={horizontal ? 'w-1/4' : 'w-full'}></div>
|
|
<FormMessage />
|
|
</div>
|
|
</FormItem>
|
|
{showMineruOptions && <MinerUOptionsFormField />}
|
|
{showPaddleocrOptions && <PaddleOCROptionsFormField />}
|
|
</>
|
|
);
|
|
}}
|
|
/>
|
|
);
|
|
}
|