Feat: add OpenDataLoader PDF parser backend (#14058) (#14097)

### What problem does this PR solve? Closes #14058. RAGFlow supports multiple PDF parsing backends (DeepDOC, MinerU, Docling, TCADP, PaddleOCR). This PR adds **OpenDataLoader** ([opendataloader-project/opendataloader-pdf](https://github.com/opendataloader-project/opendataloader-pdf)) as a new optional backend, giving users a deterministic, local-first alternative with competitive table extraction accuracy. ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Documentation Update --- ### Changes #### Backend - `deepdoc/parser/opendataloader_parser.py` — new `OpenDataLoaderParser` class inheriting `RAGFlowPdfParser`. Implements `check_installation()` (guards Python package + Java 11+ runtime), `parse_pdf()` with JSON-first extraction (heading/paragraph/table/list/image/formula) and Markdown fallback, position-tag generation compatible with the shared `@@page\tx0\tx1\ty0\ty1##` format, and temp-dir lifecycle with cleanup. - `rag/app/naive.py` — new `by_opendataloader()` wrapper, registered in `PARSERS` dict, added to `chunk_token_num=0` override list. - `rag/flow/parser/parser.py` — `"opendataloader"` branch in the pipeline PDF handler + check validation list. #### Infrastructure - `docker/entrypoint.sh` — `ensure_opendataloader()` function: opt-in via `USE_OPENDATALOADER=true`, skips gracefully if Java is not on PATH. #### Frontend - `web/src/components/layout-recognize-form-field.tsx` — `OpenDataLoader` added to `ParseDocumentType` enum and parser dropdown. Cascades automatically to the pipeline editor's Parser component. #### Docs - `docs/guides/dataset/select_pdf_parser.md` — added OpenDataLoader entry and full env-var reference. --- ### Environment variables | Variable | Default | Description | |---|---|---| | `USE_OPENDATALOADER` | `false` | Set `true` to install `opendataloader-pdf` on container startup | | `OPENDATALOADER_VERSION` | latest | Pin the PyPI release (e.g. `==2.2.1`) | | `OPENDATALOADER_HYBRID` | _(unset)_ | Enable hybrid AI mode (e.g. `docling-fast`) | | `OPENDATALOADER_IMAGE_OUTPUT` | _(unset)_ | `off` / `embedded` / `external` | | `OPENDATALOADER_OUTPUT_DIR` | _(tmp)_ | Persistent output dir; temp dir used + cleaned if unset | | `OPENDATALOADER_DELETE_OUTPUT` | `1` | `0` to retain intermediate files for debugging | | `OPENDATALOADER_SANITIZE` | _(unset)_ | `1` to filter prompt-injection patterns from output | --- ### Dependencies - **Runtime**: `opendataloader-pdf` (PyPI, Apache 2.0) — opt-in, not added to `pyproject.toml` core deps. Installed by `ensure_opendataloader()` at container startup when `USE_OPENDATALOADER=true`. - **System**: Java 11+ on PATH (JVM is the underlying engine). The installer skips with a warning if `java` is not found. --- ### How to test **Standalone parser:** ```bash source .venv/bin/activate uv pip install opendataloader-pdf python3 -c " import sys; sys.path.insert(0, '.') from deepdoc.parser.opendataloader_parser import OpenDataLoaderParser p = OpenDataLoaderParser() print('available:', p.check_installation()) s, t = p.parse_pdf('path/to/test.pdf', parse_method='pipeline') print(f'sections={len(s)} tables={len(t)}') " ``` ### Benchmark vs Docling ``` file parser secs sections tables ---------------------------------------------------------------------- text-heavy.pdf docling 45.29 148 10 text-heavy.pdf opendataloader 3.14 559 0 table-heavy.pdf docling 7.05 76 3 table-heavy.pdf opendataloader 3.71 90 0 complex.pdf docling 42.67 114 8 complex.pdf opendataloader 3.51 180 0 ```
2026-07-11 06:05:45 +08:00 · 2026-04-24 18:33:02 +02:00
parent e22cf333ed
commit 78188ce9e9
16 changed files with 1228 additions and 3 deletions
--- a/web/src/components/layout-recognize-form-field.tsx
+++ b/web/src/components/layout-recognize-form-field.tsx
@@ -20,6 +20,7 @@ export const enum ParseDocumentType {
  DeepDOC = 'DeepDOC',
  PlainText = 'Plain Text',
  Docling = 'Docling',
+  OpenDataLoader = 'OpenDataLoader',
  TCADPParser = 'TCADP Parser',
 }

@@ -52,6 +53,7 @@ export function LayoutRecognizeFormField({
          ParseDocumentType.DeepDOC,
          ParseDocumentType.PlainText,
          ParseDocumentType.Docling,
+          ParseDocumentType.OpenDataLoader,
          ParseDocumentType.TCADPParser,
        ].map((x) => ({
          label: x === ParseDocumentType.PlainText ? t(camelCase(x)) : x,
--- a/web/src/constants/llm.ts
+++ b/web/src/constants/llm.ts
@@ -62,6 +62,7 @@ export enum LLMFactory {
  Builtin = 'Builtin',
  MinerU = 'MinerU',
  PaddleOCR = 'PaddleOCR',
+  OpenDataLoader = 'OpenDataLoader',
  N1n = 'n1n',
  Avian = 'Avian',
  RAGcon = 'RAGcon',
--- a/web/src/pages/user-setting/setting-model/hooks.tsx
+++ b/web/src/pages/user-setting/setting-model/hooks.tsx
@@ -807,6 +807,56 @@ export const useSubmitPaddleOCR = () => {
  };
 };

+export const useSubmitOpenDataLoader = () => {
+  const [saveLoading, setSaveLoading] = useState(false);
+  const { addLlm } = useAddLlm();
+  const {
+    visible: opendataloaderVisible,
+    hideModal: hideOpenDataLoaderModal,
+    showModal: showOpenDataLoaderModal,
+  } = useSetModalState();
+
+  const onOpenDataLoaderOk = useCallback(
+    async (payload: any, isVerify = false) => {
+      if (!isVerify) {
+        setSaveLoading(true);
+      }
+      const req: IAddLlmRequestBody = {
+        llm_factory: LLMFactory.OpenDataLoader,
+        llm_name: payload.llm_name,
+        model_type: 'ocr',
+        api_key: { ...payload },
+        api_base: '',
+        max_tokens: 0,
+      };
+      const ret = await addLlm({ ...req, verify: isVerify });
+      if (!isVerify) {
+        setSaveLoading(false);
+        if (ret.code === 0) {
+          hideOpenDataLoaderModal();
+          return true;
+        }
+      }
+      if (isVerify) {
+        return {
+          isValid: !!ret.data?.success,
+          logs: ret.data?.message,
+        } as VerifyResult;
+      }
+      return false;
+    },
+    [addLlm, hideOpenDataLoaderModal, setSaveLoading],
+  );
+
+  return {
+    opendataloaderVisible,
+    hideOpenDataLoaderModal,
+    showOpenDataLoaderModal,
+    onOpenDataLoaderOk,
+    opendataloaderLoading: saveLoading,
+  };
+};
+
 export const useVerifySettings = ({
  onVerify,
 }: {
--- a/web/src/pages/user-setting/setting-model/index.tsx
+++ b/web/src/pages/user-setting/setting-model/index.tsx
@@ -14,6 +14,7 @@ import {
  useSubmitGoogle,
  useSubmitMinerU,
  useSubmitOllama,
+  useSubmitOpenDataLoader,
  useSubmitPaddleOCR,
  useSubmitSpark,
  useSubmitSystemModelSetting,
@@ -30,6 +31,7 @@ import GoogleModal from './modal/google-modal';
 import MinerUModal from './modal/mineru-modal';
 import TencentCloudModal from './modal/next-tencent-modal';
 import OllamaModal from './modal/ollama-modal';
+import OpenDataLoaderModal from './modal/opendataloader-modal';
 import PaddleOCRModal from './modal/paddleocr-modal';
 import SparkModal from './modal/spark-modal';
 import VolcEngineModal from './modal/volcengine-modal';
@@ -139,6 +141,14 @@ const ModelProviders = () => {
    paddleocrLoading,
  } = useSubmitPaddleOCR();

+  const {
+    opendataloaderVisible,
+    hideOpenDataLoaderModal,
+    showOpenDataLoaderModal,
+    onOpenDataLoaderOk,
+    opendataloaderLoading,
+  } = useSubmitOpenDataLoader();
+
  const ModalMap = useMemo(
    () => ({
      [LLMFactory.Bedrock]: showBedrockAddingModal,
@@ -151,6 +161,7 @@ const ModelProviders = () => {
      [LLMFactory.AzureOpenAI]: showAzureAddingModal,
      [LLMFactory.MinerU]: showMineruModal,
      [LLMFactory.PaddleOCR]: showPaddleOCRModal,
+      [LLMFactory.OpenDataLoader]: showOpenDataLoaderModal,
    }),
    [
      showBedrockAddingModal,
@@ -163,6 +174,7 @@ const ModelProviders = () => {
      showAzureAddingModal,
      showMineruModal,
      showPaddleOCRModal,
+      showOpenDataLoaderModal,
    ],
  );

@@ -240,6 +252,9 @@ const ModelProviders = () => {
    if (paddleocrVisible) {
      return onPaddleOCROk;
    }
+    if (opendataloaderVisible) {
+      return onOpenDataLoaderOk;
+    }
    if (GoogleAddingVisible) {
      return onGoogleAddingOk;
    }
@@ -269,6 +284,8 @@ const ModelProviders = () => {
    onMineruOk,
    paddleocrVisible,
    onPaddleOCROk,
+    opendataloaderVisible,
+    onOpenDataLoaderOk,
  ]);

  const { onApiKeyVerifying } = useVerifySettings({
@@ -391,6 +408,13 @@ const ModelProviders = () => {
        loading={paddleocrLoading}
        onVerify={onApiKeyVerifying}
      ></PaddleOCRModal>
+      <OpenDataLoaderModal
+        visible={opendataloaderVisible}
+        hideModal={hideOpenDataLoaderModal}
+        onOk={onOpenDataLoaderOk}
+        loading={opendataloaderLoading}
+        onVerify={onApiKeyVerifying}
+      ></OpenDataLoaderModal>
    </div>
  );
 };
--- a/web/src/pages/user-setting/setting-model/modal/opendataloader-modal/index.tsx
+++ b/web/src/pages/user-setting/setting-model/modal/opendataloader-modal/index.tsx
@@ -0,0 +1,137 @@
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { Button, ButtonLoading } from '@/components/ui/button';
+import {
+  Dialog,
+  DialogContent,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from '@/components/ui/dialog';
+import { Form } from '@/components/ui/form';
+import { Input } from '@/components/ui/input';
+import { LLMFactory } from '@/constants/llm';
+import { VerifyResult } from '@/pages/user-setting/setting-model/hooks';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { memo, useMemo } from 'react';
+import { useForm } from 'react-hook-form';
+import { useTranslation } from 'react-i18next';
+import { z } from 'zod';
+import { LLMHeader } from '../../components/llm-header';
+import VerifyButton from '../verify-button';
+
+export type OpenDataLoaderFormValues = {
+  llm_name: string;
+  opendataloader_apiserver: string;
+  opendataloader_api_key?: string;
+};
+
+export interface IModalProps<T> {
+  visible: boolean;
+  hideModal: () => void;
+  onOk?: (data: T) => Promise<boolean>;
+  onVerify?: (
+    postBody: any,
+  ) => Promise<boolean | void | VerifyResult | undefined>;
+  loading?: boolean;
+}
+
+const OpenDataLoaderModal = ({
+  visible,
+  hideModal,
+  onOk,
+  onVerify,
+  loading,
+}: IModalProps<OpenDataLoaderFormValues>) => {
+  const { t } = useTranslation();
+
+  const FormSchema = useMemo(
+    () =>
+      z.object({
+        llm_name: z.string().min(1, {
+          message: t('setting.modelNameMessage'),
+        }),
+        opendataloader_apiserver: z.string().min(1, {
+          message: t('setting.apiServerMessage'),
+        }),
+        opendataloader_api_key: z.string().optional(),
+      }),
+    [t],
+  );
+
+  const form = useForm<OpenDataLoaderFormValues>({
+    resolver: zodResolver(FormSchema),
+    defaultValues: {
+      opendataloader_apiserver: '',
+      opendataloader_api_key: '',
+    },
+  });
+
+  const handleOk = async (values: OpenDataLoaderFormValues) => {
+    const ret = await onOk?.(values as any);
+    if (ret) {
+      hideModal?.();
+    }
+  };
+
+  return (
+    <Dialog open={visible} onOpenChange={hideModal}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>
+            <LLMHeader name={LLMFactory.OpenDataLoader} />
+          </DialogTitle>
+        </DialogHeader>
+        <Form {...form}>
+          <form
+            onSubmit={form.handleSubmit(handleOk)}
+            className="space-y-6"
+            id="opendataloader-form"
+          >
+            <RAGFlowFormItem
+              name="llm_name"
+              label={t('setting.modelName')}
+              required
+            >
+              <Input placeholder="my-opendataloader" />
+            </RAGFlowFormItem>
+            <RAGFlowFormItem
+              name="opendataloader_apiserver"
+              label={t('setting.baseUrl')}
+              required
+            >
+              <Input placeholder="http://your-opendataloader-service:9383" />
+            </RAGFlowFormItem>
+            <RAGFlowFormItem
+              name="opendataloader_api_key"
+              label={t('setting.apiKey')}
+            >
+              <Input
+                type="password"
+                placeholder={t('setting.apiKeyPlaceholder')}
+              />
+            </RAGFlowFormItem>
+            {onVerify && (
+              <VerifyButton
+                onVerify={onVerify as (postBody: any) => Promise<VerifyResult>}
+              />
+            )}
+          </form>
+        </Form>
+        <DialogFooter className="flex justify-end space-x-2">
+          <Button type="button" variant="secondary" onClick={hideModal}>
+            {t('common.cancel')}
+          </Button>
+          <ButtonLoading
+            type="submit"
+            form="opendataloader-form"
+            loading={loading}
+          >
+            {t('common.add')}
+          </ButtonLoading>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+};
+
+export default memo(OpenDataLoaderModal);