diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py index 37eeb905b..4bf149d9f 100644 --- a/api/apps/restful_apis/document_api.py +++ b/api/apps/restful_apis/document_api.py @@ -78,46 +78,19 @@ from common.ssrf_guard import assert_url_is_safe from rag.nlp import search -def _parser_config_compilation_template_group_ids(parser_config) -> list[str]: - """Read template-group ids from a doc's parser_config. - - The doc now references compilation template groups via a list. A - legacy single string id is still accepted. Old - ``compilation_template_ids`` data is - intentionally ignored per the migration spec. - """ - - def _normalize(raw) -> list[str]: - if isinstance(raw, str): - raw = [raw] - if not isinstance(raw, list): - return [] - ids: list[str] = [] - seen: set[str] = set() - for gid in raw: - if not isinstance(gid, str): - continue - gid = gid.strip() - if gid and gid not in seen: - seen.add(gid) - ids.append(gid) - return ids - - if not isinstance(parser_config, dict): - return [] - if "compilation_template_group_id" in parser_config: - return _normalize(parser_config.get("compilation_template_group_id")) - ext = parser_config.get("ext") - if isinstance(ext, dict): - return _normalize(ext.get("compilation_template_group_id")) - return [] - - def _compilation_template_group_id_changed(old_config, new_config) -> bool: + from rag.svr.task_executor_refactor.chunk_post_processor import ( + _parser_config_compilation_template_group_ids, + ) + return _parser_config_compilation_template_group_ids(old_config) != _parser_config_compilation_template_group_ids(new_config) def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool: + from rag.svr.task_executor_refactor.chunk_post_processor import ( + _parser_config_compilation_template_group_ids, + ) + if not isinstance(parser_config, dict): return False if "compilation_template_group_id" not in parser_config and not (isinstance(parser_config.get("ext"), dict) and "compilation_template_group_id" in parser_config["ext"]): diff --git a/api/db/services/compilation_template_service.py b/api/db/services/compilation_template_service.py index fd969f5ce..bb31d280a 100644 --- a/api/db/services/compilation_template_service.py +++ b/api/db/services/compilation_template_service.py @@ -125,7 +125,6 @@ class CompilationTemplateService(CommonService): def list_saved(cls, tenant_id: str, keywords: str = "", kind: str = "", orderby: str = "create_time", desc: bool = True) -> list[dict]: query = cls.model.select().where( cls.model.tenant_id == tenant_id, - not cls.model.is_builtin, cls.model.status == StatusEnum.VALID.value, ) if keywords: @@ -151,7 +150,6 @@ class CompilationTemplateService(CommonService): template = cls.model.get_or_none( cls.model.id == template_id, cls.model.tenant_id == tenant_id, - not cls.model.is_builtin, cls.model.status == StatusEnum.VALID.value, ) return cls._to_saved_dict(template) if template else None @@ -162,7 +160,6 @@ class CompilationTemplateService(CommonService): query = cls.model.select(fn.COUNT(cls.model.id)).where( cls.model.tenant_id == tenant_id, cls.model.name == name, - not cls.model.is_builtin, cls.model.status == StatusEnum.VALID.value, ) if exclude_id: diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py index c071fc8ee..b2738162d 100644 --- a/rag/svr/task_executor_refactor/chunk_post_processor.py +++ b/rag/svr/task_executor_refactor/chunk_post_processor.py @@ -332,7 +332,6 @@ from common.exceptions import TaskCanceledException # noqa: E402 from common.misc_utils import thread_pool_exec # noqa: E402 from common.token_utils import num_tokens_from_string # noqa: E402 from rag.nlp import search # noqa: E402 -from api.apps.restful_apis.chunk_api import _compilation_template_kind # noqa: E402 from api.db.services.document_service import DocumentService # noqa: E402 from api.db.services.compilation_template_service import ( # noqa: E402 CompilationTemplateService, @@ -375,9 +374,6 @@ STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S = 120.0 # ----- parser_config helpers ----------------------------------------- -# Duplicated from ``task_handler`` so this module stays free of a -# reverse import (task_handler → this module via dispatch; the other -# direction would be circular). def _parser_config_compilation_template_group_ids(parser_config) -> list[str]: @@ -919,6 +915,8 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) -> generate synthesis output (wiki pages, essence paragraphs, etc.). Compile_kwd and REFINE prompt are read from the template config. """ + from api.apps.restful_apis.chunk_api import _compilation_template_kind + ctx = handler._task_context template_ids = _parser_config_compilation_template_ids(ctx.parser_config, ctx.tenant_id) if not template_ids: @@ -1225,7 +1223,7 @@ async def run_document_post_chunking_if_last( async def _maybe_run_raptor(): raptor_cfg = (ctx.parser_config or {}).get("raptor") or {} - if not raptor_cfg.get("use_raptor"): + if not raptor_cfg.get("do_raptor"): return try: ok_doc, doc_obj = DocumentService.get_by_id(task_doc_id) diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py index a34fe84d0..304dd2455 100644 --- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py +++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py @@ -34,9 +34,8 @@ Design notes: module decoupled from ``TaskHandler``'s streaming chunk loader. * The eligibility loop resolves each doc's ``parser_config.compilation_template_group_id`` to a template list - via ``CompilationTemplateGroupService.resolve_template_ids``. That - helper is duplicated as a small private function here so the module - stays free of a task_handler import (which would be circular). + via the shared parser-config helper and + ``CompilationTemplateGroupService.resolve_template_ids``. * The persistence helpers (``persist_wiki_pages_to_es`` etc.) are exposed at module level for testing but are only called from :func:`run_wiki` in production. @@ -89,38 +88,25 @@ WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE = "Auto-update via run_wiki (action={action} # ----- helpers ------------------------------------------------------- -def _parser_config_compilation_template_group_id(parser_config) -> str: - """Read the single template-group id from a doc's ``parser_config``. - - Duplicated from ``task_handler`` so this module doesn't import from - it (avoids a circular import — ``task_handler`` imports this module - from its ``task_type == "artifact"`` dispatch branch). - """ - if not isinstance(parser_config, dict): - return "" - gid = parser_config.get("compilation_template_group_id") - if isinstance(gid, str) and gid.strip(): - return gid.strip() - ext = parser_config.get("ext") - if isinstance(ext, dict): - gid = ext.get("compilation_template_group_id") - if isinstance(gid, str) and gid.strip(): - return gid.strip() - return "" - - def _parser_config_compilation_template_ids(parser_config, tenant_id: str) -> list[str]: - """Resolve a doc's ``parser_config`` to its compile-template ids by - looking up the configured group. Returns ``[]`` if the doc has no - group set or the group cannot be resolved.""" + """Resolve a doc's ``parser_config`` to compile-template ids by + looking up configured groups. Returns ``[]`` if no group resolves.""" + from rag.svr.task_executor_refactor.chunk_post_processor import ( + _parser_config_compilation_template_group_ids, + ) from api.db.services.compilation_template_group_service import ( CompilationTemplateGroupService, ) - group_id = _parser_config_compilation_template_group_id(parser_config) - if not group_id: - return [] - return CompilationTemplateGroupService.resolve_template_ids(group_id, tenant_id) + template_ids: list[str] = [] + seen: set[str] = set() + for group_id in _parser_config_compilation_template_group_ids(parser_config): + for template_id in CompilationTemplateGroupService.resolve_template_ids(group_id, tenant_id): + if template_id in seen: + continue + seen.add(template_id) + template_ids.append(template_id) + return template_ids # ----- persistence --------------------------------------------------- diff --git a/rag/svr/task_executor_refactor/task_handler.py b/rag/svr/task_executor_refactor/task_handler.py index 73f160f09..bc261099c 100644 --- a/rag/svr/task_executor_refactor/task_handler.py +++ b/rag/svr/task_executor_refactor/task_handler.py @@ -63,48 +63,15 @@ from rag.prompts.generator import run_toc_from_text from common import settings -def _parser_config_compilation_template_group_ids(parser_config) -> list[str]: - """Read template-group ids from a doc's parser_config. - - Templates were previously referenced as a list - (``compilation_template_ids``); after the template-group refactor - a doc instead points at one or more groups, and the orchestrator - resolves each group's child templates at runtime. Old - ``compilation_template_ids`` data is intentionally ignored per - the migration spec. - """ - - def _normalize(raw) -> list[str]: - if isinstance(raw, str): - raw = [raw] - if not isinstance(raw, list): - return [] - ids: list[str] = [] - seen: set[str] = set() - for gid in raw: - if not isinstance(gid, str): - continue - gid = gid.strip() - if gid and gid not in seen: - seen.add(gid) - ids.append(gid) - return ids - - if not isinstance(parser_config, dict): - return [] - if "compilation_template_group_id" in parser_config: - return _normalize(parser_config.get("compilation_template_group_id")) - ext = parser_config.get("ext") - if isinstance(ext, dict): - return _normalize(ext.get("compilation_template_group_id")) - return [] - - def _parser_config_compilation_template_ids(parser_config, tenant_id: str) -> list[str]: """Resolve a doc's parser_config to compile-template ids by looking up configured groups. Returns ``[]`` if the doc has no group set or no group can be resolved. """ + from rag.svr.task_executor_refactor.chunk_post_processor import ( + _parser_config_compilation_template_group_ids, + ) + template_ids: list[str] = [] seen: set[str] = set() for group_id in _parser_config_compilation_template_group_ids(parser_config):