zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-29 20:19:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: resolve model_id to model before judge parse method (#17493)

Browse Source
This commit is contained in:
Lynn
2026-07-29 09:34:52 +08:00
committed by GitHub
parent 75836cfc10
commit 8d1eff6ea2
11 changed files with 91 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
api/db/joint_services/tenant_model_service.py
View File

@@ -500,6 +500,26 @@ def ensure_somark_from_env(tenant_id: str) -> str | None:
)
def get_composite_model_name_by_id(model_id: str) -> str:
"""Convert a tenant_model.id to the composite model name string
``model_name@instance_name@provider_name``.
Raises LookupError if the model, instance, or provider is not found.
"""
exist, model_obj = TenantModelService.get_by_id(model_id)
if not exist:
raise LookupError(f"TenantModel id={model_id} not found.")
ok, instance_obj = TenantModelInstanceService.get_by_id(model_obj.instance_id)
if not ok:
raise LookupError(f"Instance id={model_obj.instance_id} not found for model id={model_id}.")
ok, provider_obj = TenantModelProviderService.get_by_id(model_obj.provider_id)
if not ok:
raise LookupError(f"Provider id={model_obj.provider_id} not found for model id={model_id}.")
return f"{model_obj.model_name}@{instance_obj.instance_name}@{provider_obj.provider_name}"
def ensure_mistral_ocr_from_env(tenant_id: str) -> str | None:
return _ensure_ocr_provider_from_env(
tenant_id,

10
rag/app/book.py
View File

@@ -21,6 +21,7 @@ from io import BytesIO
from deepdoc.parser.utils import get_text
from rag.app import naive
from rag.app.naive import by_plaintext, PARSERS
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from common.constants import MAXIMUM_PAGE_NUMBER
from common.parser_config_utils import normalize_layout_recognizer
from rag.nlp import bullets_category, is_english, remove_contents_table, hierarchical_merge, make_colon_as_title, naive_merge, random_choices, tokenize_table, tokenize_chunks, attach_media_context
@@ -91,7 +92,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
callback(0.8, "Finish parsing.")
elif re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

10
rag/app/laws.py
View File

@@ -27,6 +27,7 @@ from deepdoc.parser.utils import get_text
from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge
from rag.nlp import rag_tokenizer, Node
from deepdoc.parser import PdfParser, DocxParser, HtmlParser
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from rag.app.naive import by_plaintext, PARSERS
from common.parser_config_utils import normalize_layout_recognizer
@@ -183,7 +184,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
return tokenize_chunks(chunks, doc, eng, None, language=lang)
elif re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

10
rag/app/manual.py
View File

@@ -26,6 +26,7 @@ from common.token_utils import num_tokens_from_string
from deepdoc.parser import PdfParser, DocxParser
from deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper, vision_figure_parser_docx_wrapper
from docx import Document
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from rag.app.naive import by_plaintext, PARSERS
from common.parser_config_utils import normalize_layout_recognizer
@@ -146,7 +147,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
# is it English
eng = lang.lower() == "english" # pdf_parser.is_english
if re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

10
rag/app/naive.py
View File

@@ -35,6 +35,7 @@ from api.db.joint_services.tenant_model_service import (
ensure_mineru_from_env,
ensure_opendataloader_from_env,
ensure_paddleocr_from_env,
get_composite_model_name_by_id,
get_first_provider_model_name,
resolve_model_config,
get_tenant_default_model_by_type,
@@ -1038,7 +1039,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
return res
elif re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
opendataloader_llm_name = kwargs.pop("opendataloader_llm_name", None)
if layout_recognizer == "OpenDataLoader" and parser_model_name:
opendataloader_llm_name = parser_model_name

10
rag/app/one.py
View File

@@ -24,6 +24,7 @@ from rag.nlp import rag_tokenizer, tokenize
from deepdoc.parser import PdfParser, ExcelParser, HtmlParser
from deepdoc.parser.figure_parser import vision_figure_parser_docx_wrapper_naive
from rag.app.naive import by_plaintext, PARSERS
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from common.constants import MAXIMUM_PAGE_NUMBER, MAXIMUM_TASK_PAGE_NUMBER
from common.parser_config_utils import normalize_layout_recognizer
@@ -87,7 +88,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
callback(0.8, "Finish parsing.")
elif re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

10
rag/app/paper.py
View File

@@ -23,6 +23,7 @@ from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context
from deepdoc.parser import PdfParser
import numpy as np
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from rag.app.naive import by_plaintext, PARSERS
from common.parser_config_utils import normalize_layout_recognizer
@@ -138,7 +139,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
"""
parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
if re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

7
rag/app/picture.py
View File

@@ -25,7 +25,7 @@ import numpy as np
from PIL import Image
from api.db.services.llm_service import LLMBundle
from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, get_first_provider_model_name, resolve_model_config, ensure_paddleocr_from_env
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id, get_tenant_default_model_by_type, get_first_provider_model_name, resolve_model_config, ensure_paddleocr_from_env
from common.constants import LLMType
from common.parser_config_utils import normalize_layout_recognizer
from common.string_utils import clean_markdown_block
@@ -111,6 +111,11 @@ def _try_paddleocr_image(filename, binary, tenant_id, parser_config, callback):
if not layout_recognize:
return ""
if tenant_id and isinstance(layout_recognize, str):
try:
layout_recognize = get_composite_model_name_by_id(layout_recognize)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize)
if layout_recognizer != "PaddleOCR":
return ""

10
rag/app/presentation.py
View File

@@ -25,6 +25,7 @@ from pypdf import PdfReader as pdf2_read
from deepdoc.parser import PdfParser, PlainParser
from deepdoc.parser.ppt_parser import RAGFlowPptParser
from rag.app.naive import by_plaintext, PARSERS
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
from common.constants import MAXIMUM_PAGE_NUMBER
from common.parser_config_utils import normalize_layout_recognizer
from rag.nlp import rag_tokenizer
@@ -195,7 +196,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
logging.warning(error_msg)
raise NotImplementedError(error_msg)
elif re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
if tenant_id and isinstance(layout_recognize_raw, str):
try:
layout_recognize_raw = get_composite_model_name_by_id(layout_recognize_raw)
except LookupError:
pass
layout_recognizer, parser_model_name = normalize_layout_recognizer(layout_recognize_raw)
if isinstance(layout_recognizer, bool):
layout_recognizer = "DeepDOC" if layout_recognizer else "Plain Text"

1
test/unit_test/rag/app/test_picture_video.py
View File

@@ -41,6 +41,7 @@ def _load_picture_module(tokenized_texts):
tenant_model_service = ModuleType("api.db.joint_services.tenant_model_service")
tenant_model_service.get_tenant_default_model_by_type = lambda *args, **kwargs: {}
tenant_model_service.get_first_provider_model_name = lambda *args, **kwargs: None
tenant_model_service.get_composite_model_name_by_id = lambda model_id: model_id
tenant_model_service.resolve_model_config = lambda *args, **kwargs: {}
tenant_model_service.ensure_paddleocr_from_env = lambda *args, **kwargs: None

1
test/unit_test/rag/test_laws_docx_tables.py
View File

@@ -55,6 +55,7 @@ def docx_chunker():
_stub("deepdoc.parser.utils", get_text=lambda *a, **k: "")
_stub("rag.app.naive", by_plaintext=lambda *a, **k: ([], [], None), PARSERS={})
_stub("common.parser_config_utils", normalize_layout_recognizer=lambda x: (x, None))
_stub("api.db.joint_services.tenant_model_service", get_composite_model_name_by_id=lambda x: x)
module = import_module("rag.app.laws")
module = reload(module)
yield module.Docx