zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-09 21:04:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Remove redundant functions. (#16671)

Browse Source 
### Summary

Remove redundant functions.
This commit is contained in:
Kevin Hu
2026-07-06 19:02:25 +08:00
committed by GitHub
parent bfb9641128
commit 52f985f43e
5 changed files with 31 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
api/apps/restful_apis/document_api.py
View File

@@ -78,46 +78,19 @@ from common.ssrf_guard import assert_url_is_safe
from rag.nlp import search
def _parser_config_compilation_template_group_ids(parser_config) -> list[str]:
"""Read template-group ids from a doc's parser_config.
The doc now references compilation template groups via a list. A
legacy single string id is still accepted. Old
``compilation_template_ids`` data is
intentionally ignored per the migration spec.
"""
def _normalize(raw) -> list[str]:
if isinstance(raw, str):
raw = [raw]
if not isinstance(raw, list):
return []
ids: list[str] = []
seen: set[str] = set()
for gid in raw:
if not isinstance(gid, str):
continue
gid = gid.strip()
if gid and gid not in seen:
seen.add(gid)
ids.append(gid)
return ids
if not isinstance(parser_config, dict):
return []
if "compilation_template_group_id" in parser_config:
return _normalize(parser_config.get("compilation_template_group_id"))
ext = parser_config.get("ext")
if isinstance(ext, dict):
return _normalize(ext.get("compilation_template_group_id"))
return []
def _compilation_template_group_id_changed(old_config, new_config) -> bool:
from rag.svr.task_executor_refactor.chunk_post_processor import (
_parser_config_compilation_template_group_ids,
)
return _parser_config_compilation_template_group_ids(old_config) != _parser_config_compilation_template_group_ids(new_config)
def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool:
from rag.svr.task_executor_refactor.chunk_post_processor import (
_parser_config_compilation_template_group_ids,
)
if not isinstance(parser_config, dict):
return False
if "compilation_template_group_id" not in parser_config and not (isinstance(parser_config.get("ext"), dict) and "compilation_template_group_id" in parser_config["ext"]):

3
api/db/services/compilation_template_service.py
View File

@@ -125,7 +125,6 @@ class CompilationTemplateService(CommonService):
def list_saved(cls, tenant_id: str, keywords: str = "", kind: str = "", orderby: str = "create_time", desc: bool = True) -> list[dict]:
query = cls.model.select().where(
cls.model.tenant_id == tenant_id,
not cls.model.is_builtin,
cls.model.status == StatusEnum.VALID.value,
)
if keywords:
@@ -151,7 +150,6 @@ class CompilationTemplateService(CommonService):
template = cls.model.get_or_none(
cls.model.id == template_id,
cls.model.tenant_id == tenant_id,
not cls.model.is_builtin,
cls.model.status == StatusEnum.VALID.value,
)
return cls._to_saved_dict(template) if template else None
@@ -162,7 +160,6 @@ class CompilationTemplateService(CommonService):
query = cls.model.select(fn.COUNT(cls.model.id)).where(
cls.model.tenant_id == tenant_id,
cls.model.name == name,
not cls.model.is_builtin,
cls.model.status == StatusEnum.VALID.value,
)
if exclude_id:

8
rag/svr/task_executor_refactor/chunk_post_processor.py
View File

@@ -332,7 +332,6 @@ from common.exceptions import TaskCanceledException # noqa: E402
from common.misc_utils import thread_pool_exec # noqa: E402
from common.token_utils import num_tokens_from_string # noqa: E402
from rag.nlp import search # noqa: E402
from api.apps.restful_apis.chunk_api import _compilation_template_kind # noqa: E402
from api.db.services.document_service import DocumentService # noqa: E402
from api.db.services.compilation_template_service import ( # noqa: E402
CompilationTemplateService,
@@ -375,9 +374,6 @@ STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S = 120.0
# ----- parser_config helpers -----------------------------------------
# Duplicated from ``task_handler`` so this module stays free of a
# reverse import (task_handler → this module via dispatch; the other
# direction would be circular).
def _parser_config_compilation_template_group_ids(parser_config) -> list[str]:
@@ -919,6 +915,8 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
generate synthesis output (wiki pages, essence paragraphs, etc.).
Compile_kwd and REFINE prompt are read from the template config.
"""
from api.apps.restful_apis.chunk_api import _compilation_template_kind
ctx = handler._task_context
template_ids = _parser_config_compilation_template_ids(ctx.parser_config, ctx.tenant_id)
if not template_ids:
@@ -1225,7 +1223,7 @@ async def run_document_post_chunking_if_last(
async def _maybe_run_raptor():
raptor_cfg = (ctx.parser_config or {}).get("raptor") or {}
if not raptor_cfg.get("use_raptor"):
if not raptor_cfg.get("do_raptor"):
return
try:
ok_doc, doc_obj = DocumentService.get_by_id(task_doc_id)

46
rag/svr/task_executor_refactor/dataset_wiki_generator.py
View File

@@ -34,9 +34,8 @@ Design notes:
module decoupled from ``TaskHandler``'s streaming chunk loader.
* The eligibility loop resolves each doc's
``parser_config.compilation_template_group_id`` to a template list
via ``CompilationTemplateGroupService.resolve_template_ids``. That
helper is duplicated as a small private function here so the module
stays free of a task_handler import (which would be circular).
via the shared parser-config helper and
``CompilationTemplateGroupService.resolve_template_ids``.
* The persistence helpers (``persist_wiki_pages_to_es`` etc.) are
exposed at module level for testing but are only called from
:func:`run_wiki` in production.
@@ -89,38 +88,25 @@ WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE = "Auto-update via run_wiki (action={action}
# ----- helpers -------------------------------------------------------
def _parser_config_compilation_template_group_id(parser_config) -> str:
"""Read the single template-group id from a doc's ``parser_config``.
Duplicated from ``task_handler`` so this module doesn't import from
it (avoids a circular import — ``task_handler`` imports this module
from its ``task_type == "artifact"`` dispatch branch).
"""
if not isinstance(parser_config, dict):
return ""
gid = parser_config.get("compilation_template_group_id")
if isinstance(gid, str) and gid.strip():
return gid.strip()
ext = parser_config.get("ext")
if isinstance(ext, dict):
gid = ext.get("compilation_template_group_id")
if isinstance(gid, str) and gid.strip():
return gid.strip()
return ""
def _parser_config_compilation_template_ids(parser_config, tenant_id: str) -> list[str]:
"""Resolve a doc's ``parser_config`` to its compile-template ids by
looking up the configured group. Returns ``[]`` if the doc has no
group set or the group cannot be resolved."""
"""Resolve a doc's ``parser_config`` to compile-template ids by
looking up configured groups. Returns ``[]`` if no group resolves."""
from rag.svr.task_executor_refactor.chunk_post_processor import (
_parser_config_compilation_template_group_ids,
)
from api.db.services.compilation_template_group_service import (
CompilationTemplateGroupService,
)
group_id = _parser_config_compilation_template_group_id(parser_config)
if not group_id:
return []
return CompilationTemplateGroupService.resolve_template_ids(group_id, tenant_id)
template_ids: list[str] = []
seen: set[str] = set()
for group_id in _parser_config_compilation_template_group_ids(parser_config):
for template_id in CompilationTemplateGroupService.resolve_template_ids(group_id, tenant_id):
if template_id in seen:
continue
seen.add(template_id)
template_ids.append(template_id)
return template_ids
# ----- persistence ---------------------------------------------------

41
rag/svr/task_executor_refactor/task_handler.py
View File

@@ -63,48 +63,15 @@ from rag.prompts.generator import run_toc_from_text
from common import settings
def _parser_config_compilation_template_group_ids(parser_config) -> list[str]:
"""Read template-group ids from a doc's parser_config.
Templates were previously referenced as a list
(``compilation_template_ids``); after the template-group refactor
a doc instead points at one or more groups, and the orchestrator
resolves each group's child templates at runtime. Old
``compilation_template_ids`` data is intentionally ignored per
the migration spec.
"""
def _normalize(raw) -> list[str]:
if isinstance(raw, str):
raw = [raw]
if not isinstance(raw, list):
return []
ids: list[str] = []
seen: set[str] = set()
for gid in raw:
if not isinstance(gid, str):
continue
gid = gid.strip()
if gid and gid not in seen:
seen.add(gid)
ids.append(gid)
return ids
if not isinstance(parser_config, dict):
return []
if "compilation_template_group_id" in parser_config:
return _normalize(parser_config.get("compilation_template_group_id"))
ext = parser_config.get("ext")
if isinstance(ext, dict):
return _normalize(ext.get("compilation_template_group_id"))
return []
def _parser_config_compilation_template_ids(parser_config, tenant_id: str) -> list[str]:
"""Resolve a doc's parser_config to compile-template ids by
looking up configured groups. Returns ``[]`` if the doc has no
group set or no group can be resolved.
"""
from rag.svr.task_executor_refactor.chunk_post_processor import (
_parser_config_compilation_template_group_ids,
)
template_ids: list[str] = []
seen: set[str] = set()
for group_id in _parser_config_compilation_template_group_ids(parser_config):