Feat: Compilation benefit naive rag. (#17555)

### Summary

Compilation benifit naive rag

---------

Co-authored-by: Yingfeng Zhang <yingfeng.zhang@gmail.com>

api/apps/restful_apis/chunk_api.py

@@ -637,7 +637,7 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):
             config = template.get("config") if isinstance(template.get("config"), dict) else {}
             raw_kind = (config.get("kind") if isinstance(config, dict) else "") or template.get("kind") or ""
             kind_norm = _compilation_template_kind(raw_kind)
-            if kind_norm == "artifacts":
+            if kind_norm == "wiki":
                 continue
             seen_configured_ids.add(template_id)
             configured_ids.append(template_id)

api/apps/restful_apis/dataset_api.py

@@ -792,16 +792,21 @@ def delete_dataset_structure(tenant_id, dataset_id):
 @login_required
 @add_tenant_id_to_kwargs
 async def get_wiki_alteration(tenant_id, dataset_id):
-    """Return document drift for the dataset Artifact wiki.
+    """Return document drift for a compiled dataset product.
 
-    GET /api/v1/datasets/<dataset_id>/artifacts/alteration
+    GET /api/v1/datasets/<dataset_id>/artifacts/alteration?kind=<kind>
+    ``kind`` (default ``wiki``) is one of: wiki | graph | mindmap | timeline |
+    tree (tree covers both ``tree`` and ``page_index``).
     Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}
     """
+    kind = (request.args.get("kind") or "wiki").strip().lower()
+    if kind not in {"wiki", "graph", "mindmap", "timeline", "tree"}:
+        return get_error_data_result(message=f"Unsupported kind: {kind!r}. Expected one of: wiki, graph, mindmap, timeline, tree.")
     try:
-        success, result = await dataset_api_service.get_wiki_alteration(
-            dataset_id,
-            tenant_id,
-        )
+        if kind == "wiki":
+            success, result = await dataset_api_service.get_wiki_alteration(dataset_id, tenant_id)
+        else:
+            success, result = await dataset_api_service.get_structure_alteration(dataset_id, tenant_id, kind)
         if success:
             return get_result(data=result)
         return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)

api/apps/restful_apis/utils/compilation_template_validation.py

@@ -56,7 +56,7 @@ def validate_template_payload(req: dict, require_all: bool = True) -> str:
                     return f"{section.capitalize()} field description is too long."
                 if len(str((field or {}).get("rule") or "")) > 1024:
                     return f"{section.capitalize()} field rule is too long."
-        if config.get("kind") == "artifacts" or req.get("kind") == "artifacts":
+        if config.get("kind") == "wiki" or req.get("kind") == "wiki":
             for field in (config.get("claim") or {}).get("fields") or []:
                 if not str((field or {}).get("statement") or "").strip():
                     return "Claim statement is required."

api/apps/services/dataset_api_service.py

@@ -50,13 +50,13 @@ _STRUCTURE_INDEX_TYPE_TO_KIND = {
 }
 _STRUCTURE_INDEX_TYPES = frozenset(_STRUCTURE_INDEX_TYPE_TO_KIND)
 
-_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap", "artifact", "skill"} | set(_STRUCTURE_INDEX_TYPES)
+_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap", "wiki", "skill"} | set(_STRUCTURE_INDEX_TYPES)
 
 _INDEX_TYPE_TO_TASK_TYPE = {
     "graph": "structure_graph",
     "raptor": "raptor",
     "mindmap": "structure_mindmap",
-    "artifact": "artifact",
+    "wiki": "wiki",
     "skill": "skill",
     # Structure merge types carry their own task_type (== index_type) so the
     # executor can resolve which kind to merge from the task body.
@@ -67,7 +67,7 @@ _INDEX_TYPE_TO_TASK_ID_FIELD = {
     "graph": "graphrag_task_id",
     "raptor": "raptor_task_id",
     "mindmap": "mindmap_task_id",
-    "artifact": "wiki_task_id",
+    "wiki": "wiki_task_id",
     "skill": "skill_task_id",
     **{t: f"{t}_task_id" for t in _STRUCTURE_INDEX_TYPES},
 }
@@ -76,7 +76,7 @@ _INDEX_TYPE_TO_DISPLAY_NAME = {
     "graph": "Graph",
     "raptor": "RAPTOR",
     "mindmap": "Mindmap",
-    "artifact": "Artifact",
+    "wiki": "Wiki",
     "skill": "Skill",
     "structure_graph": "Structure Graph",
     "structure_mindmap": "Structure Mindmap",
@@ -1614,40 +1614,53 @@ def _extract_pipeline_compiler_group_ids(dsl) -> list[str]:
     return group_ids
 
 
-def _template_is_wiki(template: dict | None) -> bool:
+def _normalize_template_kind(raw) -> str:
+    """Lowercase a template's raw kind WITHOUT folding.
+
+    Unlike :func:`_compilation_template_kind` (which folds ``page_index`` /
+    ``knowledge_graph`` into ``timeline``), this keeps kinds distinct so the
+    alteration API can tell ``graph`` / ``timeline`` / ``tree`` apart.
+    """
+    if not isinstance(raw, str):
+        return ""
+    return raw.strip().lower().replace("-", "_")
+
+
+def _template_matches_kind(template: dict | None, accepted: set[str]) -> bool:
     if not isinstance(template, dict):
         return False
     config = template.get("config") if isinstance(template.get("config"), dict) else {}
     raw_kind = config.get("kind") or template.get("kind") or ""
-    return _compilation_template_kind(raw_kind) == "artifacts"
+    return _normalize_template_kind(raw_kind) in accepted
 
 
-def _group_has_wiki_template(group_id: str, tenant_id: str, group_cache: dict[str, bool]) -> bool:
+def _group_has_template_of_kind(group_id: str, tenant_id: str, accepted: set[str], group_cache: dict[str, bool]) -> bool:
     if group_id in group_cache:
         return group_cache[group_id]
     from api.db.services.compilation_template_group_service import CompilationTemplateGroupService
 
     group = CompilationTemplateGroupService.get_saved(group_id, tenant_id)
-    has_wiki = any(_template_is_wiki(template) for template in (group or {}).get("templates") or [])
-    group_cache[group_id] = has_wiki
-    return has_wiki
+    matched = any(_template_matches_kind(template, accepted) for template in (group or {}).get("templates") or [])
+    group_cache[group_id] = matched
+    return matched
 
 
-def _parser_config_has_wiki_template(parser_config, tenant_id: str, template_cache: dict[str, bool]) -> bool:
+def _parser_config_has_template_of_kind(parser_config, tenant_id: str, accepted: set[str], template_cache: dict[str, bool]) -> bool:
     from api.db.services.compilation_template_service import CompilationTemplateService
     from rag.svr.task_executor_refactor.chunk_post_processor import _parser_config_compilation_template_ids
 
     for template_id in _parser_config_compilation_template_ids(parser_config, tenant_id):
         if template_id not in template_cache:
-            template_cache[template_id] = _template_is_wiki(CompilationTemplateService.get_saved(template_id, tenant_id))
+            template_cache[template_id] = _template_matches_kind(CompilationTemplateService.get_saved(template_id, tenant_id), accepted)
         if template_cache[template_id]:
             return True
     return False
 
 
-def _pipeline_has_wiki_compiler(
+def _pipeline_has_compiler_of_kind(
     pipeline_id: str,
     tenant_id: str,
+    accepted: set[str],
     pipeline_cache: dict[str, bool],
     group_cache: dict[str, bool],
 ) -> bool:
@@ -1665,9 +1678,9 @@ def _pipeline_has_wiki_compiler(
         return False
 
     group_ids = _extract_pipeline_compiler_group_ids(getattr(canvas, "dsl", None))
-    has_wiki = any(_group_has_wiki_template(group_id, tenant_id, group_cache) for group_id in group_ids)
-    pipeline_cache[pipeline_id] = has_wiki
-    return has_wiki
+    matched = any(_group_has_template_of_kind(group_id, tenant_id, accepted, group_cache) for group_id in group_ids)
+    pipeline_cache[pipeline_id] = matched
+    return matched
 
 
 def _skill_index_or_none(tenant_id: str, kb_id: str):
@@ -1841,48 +1854,63 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
     # fields only) to enumerate the distinct template ids whose top-level kind
     # matches the request; ``build_bucket`` below then reads each template's rows.
     meta_fields = ["id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]
-    kind_template_ids: list[str] = []
-    seen_tid: set[str] = set()
-    has_templateless = False
-    offset = 0
     page_size = 1000
-    pages = 0
-    while True:
-        pages += 1
-        try:
-            res = await thread_pool_exec(
-                settings.docStoreConn.search,
-                meta_fields,
-                [],
-                {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": ["dataset"]},
-                [],
-                OrderByExpr(),
-                offset,
-                page_size,
-                index_nm,
-                [dataset_id],
-            )
-            meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}
-        except Exception:
-            logging.exception("get_dataset_structure: docStore discovery failed for kb=%s", dataset_id)
-            return True, empty
-        if not meta_rows:
-            break
-        for row in meta_rows.values():
-            tid = _row_template_id(row)
-            stamped_kind = (row.get("compilation_template_kind_kwd") or "").strip()
-            row_kind = stamped_kind or _template_meta(tid) or ""
-            if _resolve_dataset_structure_kind(row_kind) != resolved_kind:
-                continue
-            if tid:
-                if tid not in seen_tid:
-                    seen_tid.add(tid)
-                    kind_template_ids.append(tid)
-            else:
-                has_templateless = True
-        if len(meta_rows) < page_size:
-            break
-        offset += page_size
+
+    async def _discover_scope_templates(scope_kwd: str) -> tuple[list[str], bool, int]:
+        scope_template_ids: list[str] = []
+        seen_scope_tid: set[str] = set()
+        scope_has_templateless = False
+        offset = 0
+        pages = 0
+        while True:
+            pages += 1
+            try:
+                res = await thread_pool_exec(
+                    settings.docStoreConn.search,
+                    meta_fields,
+                    [],
+                    {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": [scope_kwd]},
+                    [],
+                    OrderByExpr(),
+                    offset,
+                    page_size,
+                    index_nm,
+                    [dataset_id],
+                )
+                meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}
+            except Exception:
+                logging.exception("get_dataset_structure: docStore discovery failed for kb=%s scope=%s", dataset_id, scope_kwd)
+                return [], False, pages
+            if not meta_rows:
+                break
+            for row in meta_rows.values():
+                tid = _row_template_id(row)
+                stamped_kind = row.get("compilation_template_kind_kwd") or ""
+                if isinstance(stamped_kind, list):
+                    stamped_kind = stamped_kind[0].strip()
+                row_kind = stamped_kind or _template_meta(tid) or ""
+                if _resolve_dataset_structure_kind(row_kind) != resolved_kind:
+                    continue
+                if tid:
+                    if tid not in seen_scope_tid:
+                        seen_scope_tid.add(tid)
+                        scope_template_ids.append(tid)
+                else:
+                    scope_has_templateless = True
+            if len(meta_rows) < page_size:
+                break
+            offset += page_size
+        return scope_template_ids, scope_has_templateless, pages
+
+    dataset_template_ids, has_templateless, dataset_pages = await _discover_scope_templates("dataset")
+    kind_template_ids: list[str] = list(dataset_template_ids)
+    template_scope_by_id: dict[str, str] = {tid: "dataset" for tid in dataset_template_ids}
+
+    doc_template_ids, _, doc_pages = await _discover_scope_templates("doc")
+    for tid in doc_template_ids:
+        if tid not in template_scope_by_id:
+            template_scope_by_id[tid] = "doc"
+            kind_template_ids.append(tid)
 
     # Detect datasets that have ONLY the legacy dataset_graph blob (no
     # entity/relation rows yet) so the fallback path below handles them.
@@ -1907,9 +1935,11 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
             pass
 
     logging.debug(
-        "get_dataset_structure: discovered %d template(s) in %d page(s) for kb=%s kind=%s",
-        len(kind_template_ids),
-        pages,
+        "get_dataset_structure: discovered %d dataset template(s) and %d doc template(s) in %d/%d page(s) for kb=%s kind=%s",
+        len(dataset_template_ids),
+        len(doc_template_ids),
+        dataset_pages,
+        doc_pages,
         dataset_id,
         resolved_kind,
     )
@@ -1927,17 +1957,20 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
 
         def _scope_for_template(row: dict):
             tid = _row_template_id(row) or ""
-            stamped = (row.get("compilation_template_kind_kwd") or "").strip()
+            stamped = row.get("compilation_template_kind_kwd") or ""
+            if isinstance(stamped, list):
+                stamped = stamped[0].strip()
+            scope_kwd = template_scope_by_id.get(tid, "dataset") if tid else "dataset"
             meta = _bucket_meta_for(tid, stamped) if tid else {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind}
             if tid:
-                return meta, {"compilation_template_ids": [tid], "scope_kwd": ["dataset"]}
-            return meta, {"compilation_template_kind_kwd": [stamped], "scope_kwd": ["dataset"]}
+                return meta, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]}
+            return meta, {"compilation_template_kind_kwd": [stamped], "scope_kwd": [scope_kwd]}
 
         bucket_meta, kw_entities, kw_relations = await sgc.keyword_subgraph(
             index_nm,
             dataset_id,
             embd_mdl,
-            {"compilation_template_ids": kind_template_ids, "knowledge_graph_kwd": ["entity"], "scope_kwd": ["dataset"]},
+            {"compilation_template_ids": kind_template_ids, "knowledge_graph_kwd": ["entity"], "scope_kwd": ["dataset", "doc"]},
             keywords,
             _scope_for_template,
             log_ctx=f"kb={dataset_id}",
@@ -1956,11 +1989,18 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
     # ── normal mode: per-template subgraph sampling from raw KB-wide rows. ──
     templates_out: list[dict] = []
     for tid in kind_template_ids:
+        scope_kwd = template_scope_by_id.get(tid, "dataset")
         try:
-            entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["dataset"]})
+            entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]})
         except Exception:
             logging.exception("get_dataset_structure: bucket build failed for kb=%s template=%s", dataset_id, tid)
             continue
+        if scope_kwd == "dataset" and not entities and not relations:
+            try:
+                entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["doc"]})
+            except Exception:
+                logging.exception("get_dataset_structure: doc fallback bucket build failed for kb=%s template=%s", dataset_id, tid)
+                continue
         if resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:
             entities = sgc.filter_entities_with_relations(entities, relations)
             if not entities:
@@ -1993,7 +2033,10 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
             legacy_rows = {}
         legacy_bucket = {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind, "entities": [], "relations": []}
         for row in legacy_rows.values():
-            if _resolve_dataset_structure_kind((row.get("compilation_template_kind_kwd") or "").strip()) != resolved_kind:
+            stamped_kind = row.get("compilation_template_kind_kwd") or ""
+            if isinstance(stamped_kind, list):
+                stamped_kind = stamped_kind[0].strip()
+            if _resolve_dataset_structure_kind((stamped_kind or "").strip()) != resolved_kind:
                 continue
             try:
                 graph = json.loads(row.get("content_with_weight") or "{}")
@@ -2012,14 +2055,39 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
     return True, {"kind": kind, "templates": templates_out}
 
 
-async def get_wiki_alteration(dataset_id: str, tenant_id: str):
-    """Return doc-level drift between current dataset docs and compiled wiki provenance."""
-    if not KnowledgebaseService.accessible(dataset_id, tenant_id):
-        return False, "no authorization"
-    ok, kb = KnowledgebaseService.get_by_id(dataset_id)
-    if not ok:
-        return False, "Invalid Dataset ID"
+# ── artifacts/alteration: per-``kind`` provenance & eligibility mapping ──
+#
+# Alteration is a pure set operation shared by every kind:
+#   removed        = involved (product provenance) − current (dataset docs)
+#   newly_uploaded = eligible (should be compiled) − involved
+# Only two inputs vary by kind: how ``involved`` provenance is gathered, and
+# which template kinds make a doc ``eligible``.
 
+# Non-folded template kinds that make a doc eligible for each API kind.
+_ALTERATION_ELIGIBLE_TEMPLATE_KINDS = {
+    "wiki": {"artifacts"},
+    "graph": {"knowledge_graph"},
+    "mindmap": {"mind_map"},
+    "timeline": {"timeline"},
+    "tree": {"tree", "page_index"},
+}
+
+# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` on
+# ``scope_kwd="dataset"`` rows, discovered by their stamped top-level kind.
+_ALTERATION_KIND_TO_MERGED_ROW_KIND = {
+    "graph": "knowledge_graph",
+    "mindmap": "mind_map",
+    "timeline": "timeline",
+}
+
+# Per-document structure kinds keep one product row per document; provenance is
+# the row's own ``doc_id`` (no dataset merge, no ``source_doc_ids``). ``tree`` and
+# ``page_index`` write distinct ``compile_kwd`` values.
+_ALTERATION_TREE_COMPILE_KWDS = ["tree", "page_index"]
+
+
+async def _current_dataset_docs(dataset_id: str):
+    """Return ``(docs, current_doc_id_set)`` for the dataset."""
     docs, _ = await thread_pool_exec(
         DocumentService.get_by_kb_id,
         kb_id=dataset_id,
@@ -2032,81 +2100,147 @@ async def get_wiki_alteration(dataset_id: str, tenant_id: str):
         types=[],
         suffix=[],
     )
-    current_doc_ids = {str(doc.get("id")) for doc in docs or [] if doc.get("id")}
+    docs = docs or []
+    current_doc_ids = {str(doc.get("id")) for doc in docs if doc.get("id")}
+    return docs, current_doc_ids
 
-    wiki_involved_doc_ids: set[str] = set()
-    pack = _wiki_index_or_none(kb.tenant_id, dataset_id)
-    if pack is not None:
-        index_nm, _ = pack
-        from common.doc_store.doc_store_base import OrderByExpr
 
-        select_fields = ["id", "source_doc_ids"]
-        offset = 0
-        page_size = 1000
-        while True:
-            try:
-                res = await thread_pool_exec(
-                    settings.docStoreConn.search,
-                    select_fields=select_fields,
-                    highlight_fields=[],
-                    condition={"compile_kwd": [WIKI_PAGE_COMPILE_KWD]},
-                    match_expressions=[],
-                    order_by=OrderByExpr(),
-                    offset=offset,
-                    limit=page_size,
-                    index_names=index_nm,
-                    knowledgebase_ids=[dataset_id],
-                )
-                rows = settings.docStoreConn.get_fields(res, select_fields) or {}
-            except Exception:
-                logging.exception("get_wiki_alteration: docStore search failed for kb=%s", dataset_id)
-                rows = {}
+def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc_ids: set) -> dict:
+    """Build the drift response dict shared by every ``kind``."""
+    removed_doc_ids = sorted(involved_doc_ids - current_doc_ids)
+    newly_uploaded_doc_ids = sorted(eligible_doc_ids - involved_doc_ids)
+    return {
+        "removed": len(removed_doc_ids),
+        "newly_uploaded": len(newly_uploaded_doc_ids),
+        "removed_doc_ids": removed_doc_ids,
+        "newly_uploaded_doc_ids": newly_uploaded_doc_ids,
+        "involved_doc_ids": sorted(involved_doc_ids),
+        "eligible_doc_ids": sorted(eligible_doc_ids),
+    }
 
-            if not rows:
-                break
-            for row in rows.values():
-                source_doc_ids = row.get("source_doc_ids")
-                if isinstance(source_doc_ids, str):
-                    source_doc_ids = [source_doc_ids]
-                if not isinstance(source_doc_ids, list):
-                    continue
-                wiki_involved_doc_ids.update(str(doc_id) for doc_id in source_doc_ids if doc_id)
-
-            offset += page_size
-            total = settings.docStoreConn.get_total(res)
-            if not total or offset >= int(total):
-                break
 
+def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:
+    """Doc ids whose parser_config or pipeline carries a template of ``kind``."""
+    accepted = _ALTERATION_ELIGIBLE_TEMPLATE_KINDS.get(kind) or set()
     template_cache: dict[str, bool] = {}
     group_cache: dict[str, bool] = {}
     pipeline_cache: dict[str, bool] = {}
-    eligible_wiki_doc_ids: set[str] = set()
+    eligible: set[str] = set()
     for doc in docs or []:
         doc_id = str(doc.get("id") or "")
         if not doc_id:
             continue
         parser_config = doc.get("parser_config") or {}
-        if _parser_config_has_wiki_template(parser_config, kb.tenant_id, template_cache):
-            eligible_wiki_doc_ids.add(doc_id)
+        if _parser_config_has_template_of_kind(parser_config, tenant_id, accepted, template_cache):
+            eligible.add(doc_id)
             continue
-        if _pipeline_has_wiki_compiler(
-            doc.get("pipeline_id") or "",
-            kb.tenant_id,
-            pipeline_cache,
-            group_cache,
-        ):
-            eligible_wiki_doc_ids.add(doc_id)
+        if _pipeline_has_compiler_of_kind(doc.get("pipeline_id") or "", tenant_id, accepted, pipeline_cache, group_cache):
+            eligible.add(doc_id)
+    return eligible
 
-    removed_doc_ids = sorted(wiki_involved_doc_ids - current_doc_ids)
-    newly_uploaded_doc_ids = sorted(eligible_wiki_doc_ids - wiki_involved_doc_ids)
-    return True, {
-        "removed": len(removed_doc_ids),
-        "newly_uploaded": len(newly_uploaded_doc_ids),
-        "removed_doc_ids": removed_doc_ids,
-        "newly_uploaded_doc_ids": newly_uploaded_doc_ids,
-        "involved_doc_ids": sorted(wiki_involved_doc_ids),
-        "eligible_doc_ids": sorted(eligible_wiki_doc_ids),
-    }
+
+async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str, from_list: bool) -> set:
+    """Page a docStore search, folding each row's ``field`` into a doc-id set.
+
+    ``from_list`` reads ``field`` as ``source_doc_ids`` (a list of provenance doc
+    ids); otherwise ``field`` is the row's own ``doc_id`` scalar.
+    """
+    from common.doc_store.doc_store_base import OrderByExpr
+
+    involved: set[str] = set()
+    select_fields = ["id", field]
+    offset = 0
+    page_size = 1000
+    while True:
+        try:
+            res = await thread_pool_exec(
+                settings.docStoreConn.search,
+                select_fields=select_fields,
+                highlight_fields=[],
+                condition=condition,
+                match_expressions=[],
+                order_by=OrderByExpr(),
+                offset=offset,
+                limit=page_size,
+                index_names=index_nm,
+                knowledgebase_ids=[dataset_id],
+            )
+            rows = settings.docStoreConn.get_fields(res, select_fields) or {}
+        except Exception:
+            logging.exception("alteration: docStore search failed for kb=%s cond=%s", dataset_id, condition)
+            rows = {}
+
+        if not rows:
+            break
+        for row in rows.values():
+            value = row.get(field)
+            if from_list:
+                if isinstance(value, str):
+                    value = [value]
+                if isinstance(value, list):
+                    involved.update(str(d) for d in value if d)
+            elif value:
+                involved.add(str(value))
+
+        offset += page_size
+        total = settings.docStoreConn.get_total(res)
+        if not total or offset >= int(total):
+            break
+    return involved
+
+
+async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> set:
+    """Gather the doc ids baked into the compiled product for ``kind``."""
+    if kind == "wiki":
+        return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": [WIKI_PAGE_COMPILE_KWD]}, "source_doc_ids", from_list=True)
+    if kind in _ALTERATION_KIND_TO_MERGED_ROW_KIND:
+        condition = {
+            "knowledge_graph_kwd": ["entity", "relation"],
+            "scope_kwd": ["dataset"],
+            "compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],
+        }
+        return await _involved_doc_ids_paged(index_nm, dataset_id, condition, "source_doc_ids", from_list=True)
+    if kind == "tree":
+        return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": list(_ALTERATION_TREE_COMPILE_KWDS)}, "doc_id", from_list=False)
+    return set()
+
+
+async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
+    """Shared driver: doc-level drift between the ``kind`` product and the dataset."""
+    if not KnowledgebaseService.accessible(dataset_id, tenant_id):
+        return False, "no authorization"
+    ok, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if not ok:
+        return False, "Invalid Dataset ID"
+
+    docs, current_doc_ids = await _current_dataset_docs(dataset_id)
+
+    involved_doc_ids: set[str] = set()
+    pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
+    if pack is not None:
+        index_nm, _ = pack
+        involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)
+
+    eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)
+    return True, _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)
+
+
+async def get_wiki_alteration(dataset_id: str, tenant_id: str):
+    """Return doc-level drift between current dataset docs and compiled wiki provenance."""
+    return await _get_alteration(dataset_id, tenant_id, "wiki")
+
+
+async def get_structure_alteration(dataset_id: str, tenant_id: str, kind: str):
+    """Return doc-level drift for a non-wiki structure ``kind``.
+
+    ``kind`` is one of ``graph`` / ``mindmap`` / ``timeline`` (dataset-merged
+    products, provenance from ``source_doc_ids``) or ``tree`` (per-document
+    products covering ``tree`` + ``page_index``, provenance from ``doc_id``).
+    """
+    kind = (kind or "").strip().lower()
+    if kind not in _ALTERATION_KIND_TO_MERGED_ROW_KIND and kind != "tree":
+        return False, f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, tree."
+    return await _get_alteration(dataset_id, tenant_id, kind)
 
 
 async def list_wiki_pages(