Feat: Compilation benefit naive rag. (#17555)

### Summary

Compilation benifit naive rag

---------

Co-authored-by: Yingfeng Zhang <yingfeng.zhang@gmail.com>

rag/advanced_rag/harness/orchestrator/agentic.py

@@ -310,7 +310,7 @@ async def _add_template_group_compilations(comps: set[str], parser_config: dict,
                 comps.add("page_index")
             elif kind in {"mindmap", "mind_map"}:
                 comps.add("mindmap")
-            elif kind == "artifacts":
+            elif kind == "wiki":
                 comps.add("wiki")
 
 

rag/advanced_rag/harness/orchestrator/direct.py

@@ -11,9 +11,9 @@ async def direct_search(state: dict, tools) -> dict:
     """Single hybrid search → merge into kbinfos."""
     question = state.get("question", "")
     keywords = state.get("keywords", "")
-    _LOG.info("[Direct search] Looking up the knowledge base for: \"%s\" (keywords: %s)", question, keywords)
+    _LOG.info('[Direct search] Looking up the knowledge base for: "%s" (keywords: %s)', question, keywords)
 
-    result = await hybrid_search(tools, query=question, keywords=keywords)
+    result = await hybrid_search(tools, query=question, keywords=keywords, use_compiled=True)
     _merge_kbinfos(tools, result)
 
     if not _has_chunks(tools):

rag/advanced_rag/harness/tools/navigation.py

@@ -543,13 +543,22 @@ async def dataset_navigation_by_tree(tools, topic: str, keywords: str = "", doc_
 # its way to a small subgraph: seed entities by the question, hop out over
 # relations to the 2nd-degree neighbours, then answer from that subgraph.
 
-_KG_SEEDS = 3  # entities matched directly to the question
-_KG_HOPS = 1  # relation hops out from the seeds (1 => "2nd degree")
-_KG_NEIGHBORS = 10  # neighbour entity rows resolved per hop
+# Scope of the compiled KG rows we search. Without a doc_scope we read the
+# dataset-merged rows (one graph per dataset); with a doc_scope we read the
+# per-document rows and filter by doc_id. Kept local rather than imported from
+# the task-executor layer so the harness has no dependency on it.
+_SCOPE_KWD_DATASET = "dataset"
+_SCOPE_KWD_DOC = "doc"
+
+_KG_SEEDS = 2  # top-N entities matched directly to the question
+_KG_SEED_POOL = 64  # KNN candidate pool before the mention_count_int re-sort
+_KG_SEED_SIM = 0.8  # dense-similarity floor for seed entities
+_KG_HOPS = 2  # relation hops out from the seeds (1 => "2nd degree")
+_KG_NEIGHBORS = 128  # cap on neighbour entity rows resolved per hop
 _KG_REL_LIMIT = 32  # relations fetched per endpoint filter
 
 
-async def _kg_scopes(tools, doc_scope: list[str] | None):
+async def _kg_scopes(tools, doc_scope: list[str] | None = None):
     """Resolve the (kb_id, tenant_id, doc_ids|None) groups to search.
 
     With a ``doc_scope`` the graph is limited to those docs (grouped by their
@@ -567,29 +576,59 @@ async def _kg_scopes(tools, doc_scope: list[str] | None):
     return [(kb.id, kb.tenant_id, None) for kb in getattr(tools, "kbs", []) or []]
 
 
-async def _kg_search(tools, kb_id: str, tenant_id: str, doc_ids, kind: str, text: str = "", top_n: int = 8, extra: dict | None = None) -> list[dict]:
-    """Search the compiled KG rows of one KB and return the raw field maps."""
+async def _kg_search(
+    tools,
+    kb_id: str,
+    tenant_id: str,
+    doc_ids,
+    kind: str,
+    text: str = "",
+    top_n: int = 8,
+    extra: dict | None = None,
+    scope_kwd: str | None = None,
+    order_desc: str | None = None,
+    pool: int | None = None,
+    similarity: float = 0.6,
+) -> list[dict]:
+    """Search the compiled KG rows of one KB and return the raw field maps.
+
+    ``scope_kwd`` narrows to dataset-merged (``"dataset"``) or per-doc (``"doc"``)
+    rows. ``order_desc`` sorts the hits by that field descending; when combined
+    with a dense ``text`` match, ``pool`` sets the KNN candidate count so the
+    re-sort ranks a wider pool than the ``top_n`` finally kept. ``similarity`` is
+    the dense-match floor.
+    """
     from common import settings
     from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
     from common.misc_utils import thread_pool_exec
     from rag.nlp import search
 
     condition: dict = {"knowledge_graph_kwd": [kind]}
+    if scope_kwd:
+        condition["scope_kwd"] = [scope_kwd]
     if doc_ids:
         condition["doc_id"] = list(doc_ids)
     if extra:
         condition.update(extra)
 
-    fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "from_entity_kwd", "to_entity_kwd"]
+    fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "name_kwd", "mention_count_int", "from_entity_kwd", "to_entity_kwd"]
     exprs = []
     if text:
+        knn_topn = pool or top_n
         if getattr(tools, "embed_mdl", None):
             try:
-                exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, top_n, 0.1))
+                exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, knn_topn, similarity))
             except Exception:
                 _LOG.exception("[Graph exploration] vector build failed; using keyword match")
         if not exprs:
-            exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, top_n))
+            exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, knn_topn))
+
+    order_by = OrderByExpr()
+    if order_desc:
+        try:
+            order_by.desc(order_desc)
+        except Exception:
+            order_by = OrderByExpr()
 
     try:
         res = await thread_pool_exec(
@@ -598,7 +637,7 @@ async def _kg_search(tools, kb_id: str, tenant_id: str, doc_ids, kind: str, text
             [],
             condition,
             exprs,
-            OrderByExpr(),
+            order_by,
             0,
             top_n,
             search.index_name(tenant_id),
@@ -651,6 +690,25 @@ def _kg_parse_relation(row: dict) -> dict | None:
     }
 
 
+def _endpoint_terms(names) -> list[str]:
+    """Case variants for matching relation endpoints.
+
+    ``dataset_structure_merger`` lowercases ``from_entity_kwd``/``to_entity_kwd``
+    on merged rows while entity names keep their original case, so hop queries
+    must try both forms. Accepts a single name or an iterable and returns the
+    sorted union of each name's original and lowercased form.
+    """
+    if isinstance(names, str):
+        names = [names]
+    terms: set[str] = set()
+    for n in names or []:
+        n = (n or "").strip()
+        if n:
+            terms.add(n)
+            terms.add(n.lower())
+    return sorted(terms)
+
+
 def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_names: list[str]) -> dict:
     """Group the source_chunk_ids of the relevant entities AND relations by doc.
 
@@ -684,23 +742,31 @@ def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_
 async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[str] | None = None) -> dict:
     """Explore the compiled knowledge graph to answer ``query``.
 
-    Searches seed entities for the question, hops out over their relations to the
-    2nd-degree neighbours, asks the chat model to answer from that subgraph, then
-    pulls the source passages behind the entities/relations it found relevant and
-    narrows them with ``keywords``. Falls back to a plain hybrid search when the
-    KB has no compiled graph or no source text is behind the relevant nodes.
+    Seeds the top-``_KG_SEEDS`` entities for the question (dense match above
+    ``_KG_SEED_SIM``, ranked by ``mention_count_int``), hops ``_KG_HOPS`` out over
+    their relations, then asks the chat model whether that subgraph answers the
+    question. When it does, the answer is returned directly; when it doesn't, the
+    source passages behind the entities/relations the model found relevant are
+    returned as evidence (narrowed by ``keywords``) so the caller can continue.
 
-    :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}``
+    Without ``doc_scope`` the dataset-merged graph is searched
+    (``scope_kwd="dataset"``); with it, the per-document rows of those docs
+    (``scope_kwd="doc"``, filtered by ``doc_id``).
+
+    :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}`` — exactly one
+        of ``answer`` / ``chunks`` is populated.
     """
-    from rag.advanced_rag.harness.tools.search import _narrow_by_keywords, hybrid_search
+    from rag.advanced_rag.harness.tools.search import _narrow_by_keywords
 
+    _empty = {"answer": "", "chunks": [], "doc_aggs": []}
     _LOG.info(f'[Graph exploration] Exploring the knowledge graph for "{query}" (keywords: {keywords})')
 
     scopes = await _kg_scopes(tools, doc_scope)
     if not scopes:
-        _LOG.info("[Graph exploration] No knowledge base in scope — falling back to a normal search.")
-        return await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)
+        _LOG.info("[Graph exploration] No knowledge base in scope.")
+        return _empty
 
+    scope_kwd = _SCOPE_KWD_DOC if doc_scope else _SCOPE_KWD_DATASET
     text = f"{query} {keywords}".strip()
     entities: list[dict] = []
     relations: list[dict] = []
@@ -718,52 +784,79 @@ async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[s
         return added
 
     for kb_id, tenant_id, doc_ids in scopes:
-        seed_rows = await _kg_search(tools, kb_id, tenant_id, doc_ids, "entity", text=text, top_n=_KG_SEEDS)
+        # (1) Seeds: condition C — dense match (>= _KG_SEED_SIM) over the scoped
+        # entity rows, ranked by mention_count_int desc, top _KG_SEEDS.
+        seed_rows = await _kg_search(
+            tools,
+            kb_id,
+            tenant_id,
+            doc_ids,
+            "entity",
+            text=text,
+            top_n=_KG_SEEDS,
+            scope_kwd=scope_kwd,
+            order_desc="mention_count_int",
+            pool=_KG_SEED_POOL,
+            similarity=_KG_SEED_SIM,
+        )
         seeds = [e for e in (_kg_parse_entity(r) for r in seed_rows.values()) if e]
         frontier = _add_entities(seeds, kb_id)
         _LOG.info("[Graph exploration] Seeded %d entity(ies): %s", len(frontier), ", ".join(frontier) or "none")
 
+        # (2) Expand _KG_HOPS out, collecting relations and neighbour entities.
         for _hop in range(_KG_HOPS):
             if not frontier:
                 break
-            # Relations touching the current frontier (as source OR target).
+            terms = _endpoint_terms(frontier)  # case variants — merged rows lowercase endpoints
             rel_rows: dict = {}
-            rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, extra={"from_entity_kwd": frontier}))
-            rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, extra={"to_entity_kwd": frontier}))
+            rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"from_entity_kwd": terms}))
+            rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"to_entity_kwd": terms}))
             hop_relations = [r for r in (_kg_parse_relation(x) for x in rel_rows.values()) if r]
             relations.extend(hop_relations)
 
-            neighbour_names = {n for r in hop_relations for n in (r["from"], r["to"]) if n.lower() not in ent_names}
-            if not neighbour_names:
+            seen_lower = {k.split(":", 1)[1] for k in ent_names if k.startswith(f"{kb_id}:")}
+            neigh_names = {n.strip() for r in hop_relations for n in (r["from"], r["to"]) if n and n.strip()}
+            neigh_lower_set = {n.lower() for n in neigh_names} - seen_lower
+            if not neigh_lower_set:
                 break
-            neigh_rows = await _kg_search(tools, kb_id, tenant_id, doc_ids, "entity", text=" ".join(neighbour_names), top_n=_KG_NEIGHBORS)
-            wanted = {n.lower() for n in neighbour_names}
-            neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e and e["name"].lower() in wanted]
+            neigh_filtered = {n for n in neigh_names if n.lower() in neigh_lower_set}
+            neigh_rows = await _kg_search(
+                tools,
+                kb_id,
+                tenant_id,
+                doc_ids,
+                "entity",
+                top_n=min(max(len(neigh_filtered), 1), _KG_NEIGHBORS),
+                scope_kwd=scope_kwd,
+                extra={"name_kwd": _endpoint_terms(neigh_filtered)},
+            )
+            neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e]
             frontier = _add_entities(neighbours, kb_id)
             _LOG.info("[Graph exploration] Hop %d reached %d neighbour entity(ies).", _hop + 1, len(frontier))
 
     if not entities and not relations:
-        _LOG.info("[Graph exploration] No compiled knowledge graph here — falling back to a normal search.")
-        return await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)
+        _LOG.info("[Graph exploration] No compiled knowledge graph in scope.")
+        return _empty
 
     _LOG.info("[Graph exploration] Built a subgraph of %d entity(ies) and %d relation(s).", len(entities), len(relations))
 
+    # (3) Does the subgraph answer the question?
     answer, relevant = await _ask_structure(tools, query, entities, relations, "knowledge graph", "Graph exploration")
 
+    # (4a) Sufficient — return the answer, no chunks.
+    if answer:
+        _LOG.info("[Graph exploration] The subgraph answered the question directly.")
+        return {"answer": answer, "chunks": [], "doc_aggs": []}
+
+    # (4b) Insufficient — return the source passages behind the relevant nodes.
     evidence = _collect_evidence_ids(entities, relations, relevant)
     chunks: list[dict] = []
     for doc_id, ids in evidence.items():
         if doc_id and ids:
             chunks.extend(await _load_chunks_by_ids(tools, doc_id, ids))
 
-    if not chunks:
-        _LOG.info("[Graph exploration] No source text behind those nodes — falling back to a normal search.")
-        fallback = await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)
-        fallback["answer"] = answer
-        return fallback
-
     before = len(chunks)
     chunks = _narrow_by_keywords(chunks, keywords)
-    _LOG.info("[Graph exploration] Pulled %d source passage(s) behind those nodes, kept %d after keyword filtering.", before, len(chunks))
+    _LOG.info("[Graph exploration] Insufficient; returning %d evidence passage(s) (%d before keyword filtering).", len(chunks), before)
 
-    return {"answer": answer, "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}
+    return {"answer": "", "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}

rag/advanced_rag/harness/tools/search.py

@@ -4,6 +4,7 @@ import logging
 import re
 import hashlib
 from common import settings
+from .navigation import _kg_scopes
 
 _LOG = logging.getLogger(__name__)
 
@@ -179,7 +180,7 @@ def _normalize(kbinfos: dict, tenant_ids: list[str] | str | None) -> dict:
     return kbinfos
 
 
-async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "") -> dict:
+async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "", use_compiled: bool = False) -> dict:
     if not tools.kb_ids and not kb_ids:
         return {"chunks": [], "doc_aggs": []}
     target_ids = kb_ids or tools.kb_ids
@@ -220,6 +221,9 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_
         length = len(kbinfos["chunks"])
         kbinfos["chunks"] = _narrow_by_keywords(kbinfos.get("chunks", []), keywords)
         _LOG.info(f"[Hybrid search] Kept {len(kbinfos['chunks'])} of {length} passage(s) that actually mention the keywords.")
+    if use_compiled and kbinfos.get("chunks"):
+        _LOG.info("[Hybrid search] Compiled expansion enabled — enriching with page_index/tree/KG navigation.")
+        await _expand_with_compiled(tools, query, keywords, kbinfos)
     if cache is not None:
         cache[cache_key] = kbinfos
     return kbinfos
@@ -277,6 +281,473 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:
     return kbinfos
 
 
+# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───
+
+
+async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict) -> None:
+    """Zero-LLM compiled-product expansion: page_index → tree → KG.
+
+    For each bound KB, searches compiled entity rows matching the query,
+    hops 1-hop via relations to find neighbour entities, then appends
+    their source passages to ``kbinfos["chunks"]``.
+    """
+    before = len(kbinfos.get("chunks", []))
+    seen_ids = {c.get("chunk_id") or c.get("id") for c in kbinfos.get("chunks", [])}
+
+    scopes = await _kg_scopes(tools)
+    if not scopes:
+        return
+
+    for kb_id, tenant_id, doc_ids in scopes:
+        # 1-hop entity-graph expansion per template kind.
+        # Each template writes entity/relation rows tagged with
+        # ``compilation_template_kind_kwd`` — search them independently.
+        for label, template_kind in (
+            ("knowledge_graph", "knowledge_graph"),
+            ("mind_map", "mind_map"),
+            ("timeline", "timeline"),
+            ("page_index", "page_index"),
+        ):
+            chunks = await _expand_compiled_strategy(
+                tools,
+                kb_id,
+                tenant_id,
+                doc_ids,
+                query,
+                seen_ids,
+                template_kind=template_kind,
+                max_chunks=5,
+            )
+            if chunks:
+                kbinfos.setdefault("chunks", []).extend(chunks)
+                _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
+
+        # Tree structure graph (uses ``compile_kwd``, not template kind).
+        chunks = await _expand_compiled_strategy(
+            tools,
+            kb_id,
+            tenant_id,
+            doc_ids,
+            query,
+            seen_ids,
+            compile_kwd="tree",
+            max_chunks=5,
+        )
+        if chunks:
+            kbinfos.setdefault("chunks", []).extend(chunks)
+            _LOG.debug("[Compiled expand] tree: +%d chunks", len(chunks))
+
+        # Synthesis pages — standalone rendered articles from wiki / session
+        # graph / session essence templates.  Searched directly (no entity-graph nav).
+        for label, ckwd in (
+            ("wiki_page", "wiki_page"),
+            ("artifact_page", "artifact_page"),
+            ("essence", "essence"),
+        ):
+            chunks = await _expand_wiki_page_strategy(
+                tools,
+                kb_id,
+                tenant_id,
+                doc_ids,
+                query,
+                seen_ids,
+                compile_kwd=ckwd,
+                max_chunks=5,
+            )
+            if chunks:
+                kbinfos.setdefault("chunks", []).extend(chunks)
+                _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
+
+    # Re-sort so compiled-expansion chunks blend by similarity with regular ones.
+    chunks = kbinfos.get("chunks", [])
+    if chunks:
+        chunks.sort(key=lambda c: c.get("similarity", 0.0), reverse=True)
+
+    after = len(chunks)
+    _LOG.info("[Hybrid search] Compiled expansion added %d chunks.", after - before)
+
+
+async def _search_compiled_rows(
+    tools,
+    kb_id: str,
+    tenant_id: str,
+    doc_ids: list[str] | None,
+    kind: str,
+    *,
+    text: str = "",
+    top_n: int = 8,
+    extra: dict | None = None,
+    compile_kwd: str | None = None,
+    template_kind: str | None = None,
+) -> dict:
+    """Search compiled KG rows in one KB, returning raw field maps.
+
+    *compile_kwd* filters by the ``compile_kwd`` field (e.g. "tree" for tree
+    structure nodes).  *template_kind* filters by ``compilation_template_kind_kwd``
+    (e.g. "knowledge_graph", "mind_map").  Leave both ``None`` to scan all rows.
+    """
+    from common import settings
+    from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
+    from common.misc_utils import thread_pool_exec
+    from rag.nlp import search
+
+    condition: dict = {"knowledge_graph_kwd": [kind]}
+    if compile_kwd:
+        condition["compile_kwd"] = compile_kwd
+    if template_kind:
+        condition["compilation_template_kind_kwd"] = template_kind
+    if doc_ids:
+        condition["doc_id"] = list(doc_ids)
+    if extra:
+        condition.update(extra)
+
+    fields = [
+        "content_with_weight",
+        "source_chunk_ids",
+        "doc_id",
+        "docnm_kwd",
+        "from_entity_kwd",
+        "to_entity_kwd",
+        "name_kwd",
+    ]
+    exprs = []
+    if text:
+        embd_mdl = getattr(tools, "embed_mdl", None)
+        if embd_mdl:
+            try:
+                exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+            except Exception:
+                _LOG.exception("[Compiled expand] vector build failed; using keyword match")
+        if not exprs:
+            exprs.append(
+                MatchTextExpr(
+                    ["content_ltks", "content_sm_ltks"],
+                    text,
+                    top_n,
+                )
+            )
+
+    try:
+        res = await thread_pool_exec(
+            settings.docStoreConn.search,
+            fields,
+            [],
+            condition,
+            exprs,
+            OrderByExpr(),
+            0,
+            top_n,
+            search.index_name(tenant_id),
+            [kb_id],
+        )
+        return settings.docStoreConn.get_fields(res, fields) or {}
+    except Exception:
+        _LOG.exception("[Compiled expand] search failed (kind=%s compile_kwd=%s)", kind, compile_kwd)
+        return {}
+
+
+async def _load_chunks_for_doc(tools, doc_id: str, chunk_ids: list[str]) -> list[dict]:
+    """Load chunks by their IDs from the doc store."""
+    if not chunk_ids:
+        return []
+    from common import settings
+    from common.doc_store.doc_store_base import OrderByExpr
+    from common.misc_utils import thread_pool_exec
+    from rag.nlp import search
+
+    resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)
+    if not resolved:
+        return []
+    kb_id, tenant_id = resolved
+
+    fields = ["content_with_weight", "docnm_kwd", "doc_id", "id"]
+    try:
+        res = await thread_pool_exec(
+            settings.docStoreConn.search,
+            fields,
+            [],
+            {"id": list(chunk_ids)},
+            [],
+            OrderByExpr(),
+            0,
+            len(chunk_ids),
+            search.index_name(tenant_id),
+            [kb_id],
+        )
+        rows = settings.docStoreConn.get_fields(res, fields)
+        if not rows:
+            return []
+        return [{**v, "chunk_id": k} for k, v in rows.items()]
+    except Exception:
+        _LOG.exception("[Compiled expand] failed to load chunks for doc_id=%s", doc_id)
+        return []
+
+
+async def _expand_compiled_strategy(
+    tools,
+    kb_id: str,
+    tenant_id: str,
+    doc_ids: list[str] | None,
+    query: str,
+    seen_ids: set[str],
+    *,
+    compile_kwd: str | None = None,
+    template_kind: str | None = None,
+    max_chunks: int = 5,
+) -> list[dict]:
+    """Generic 1-hop compiled expansion: entity search → relation nav → chunk load.
+
+    1. Embedding-match seed entities (filtered by *compile_kwd* or *template_kind*).
+    2. Fetch relations adjacent to seed entities (forward + backward).
+    3. Collect neighbour entity names (1-hop away).
+    4. Look up neighbour entities to get ``source_chunk_ids``.
+    5. Load actual chunks, deduplicate, respect *max_chunks*.
+
+    *compile_kwd* is used for structure graphs (e.g. "tree").
+    *template_kind* is used for entity extraction rows (e.g. "knowledge_graph").
+    """
+    import json
+
+    # -- 1. Seed entities --
+    seed_rows = await _search_compiled_rows(
+        tools,
+        kb_id,
+        tenant_id,
+        doc_ids,
+        "entity",
+        text=query,
+        top_n=5,
+        compile_kwd=compile_kwd,
+        template_kind=template_kind,
+    )
+    if not seed_rows:
+        return []
+
+    seed_names: set[str] = set()
+    for r in seed_rows.values():
+        try:
+            payload = json.loads(r.get("content_with_weight") or "{}")
+        except Exception:
+            continue
+        name = (payload.get("name") or payload.get("title") or "").strip()
+        if name:
+            seed_names.add(name)
+    if not seed_names:
+        return []
+
+    # -- 2. Adjacent relations (outgoing + incoming) --
+    # Provide both original and lowercased names — dataset_structure_merger
+    # lowercases merged-row endpoints while per-doc rows keep original case.
+    seed_list = sorted({n.lower() for n in seed_names} | seed_names)
+    fwd = await _search_compiled_rows(
+        tools,
+        kb_id,
+        tenant_id,
+        doc_ids,
+        "relation",
+        top_n=50,
+        compile_kwd=compile_kwd,
+        template_kind=template_kind,
+        extra={"from_entity_kwd": seed_list},
+    )
+    bwd = await _search_compiled_rows(
+        tools,
+        kb_id,
+        tenant_id,
+        doc_ids,
+        "relation",
+        top_n=50,
+        compile_kwd=compile_kwd,
+        template_kind=template_kind,
+        extra={"to_entity_kwd": seed_list},
+    )
+    all_rels = {**fwd, **bwd}
+
+    # -- 3. Neighbour names (1-hop, exclude seeds) --
+    seed_lower = {n.lower() for n in seed_names}
+    neighbour_names: set[str] = set()
+    for r in all_rels.values():
+        frm = (r.get("from_entity_kwd") or "").strip()
+        frm_lower = frm.lower()
+        to = (r.get("to_entity_kwd") or "").strip()
+        to_lower = to.lower()
+        if frm_lower in seed_lower and to and to_lower not in seed_lower:
+            neighbour_names.add(to)
+        if to_lower in seed_lower and frm and frm_lower not in seed_lower:
+            neighbour_names.add(frm)
+    if not neighbour_names:
+        return []
+
+    # -- 4. Neighbour entity source_chunk_ids --
+    # Provide both original and lowercased — same as seed_list above.
+    neigh_list = sorted({n.lower() for n in neighbour_names} | neighbour_names)
+    if len(neigh_list) > 100:
+        neigh_list = neigh_list[:100]  # reasonable cap for name_kwd search
+    neigh_rows = await _search_compiled_rows(
+        tools,
+        kb_id,
+        tenant_id,
+        doc_ids,
+        "entity",
+        top_n=len(neigh_list),
+        compile_kwd=compile_kwd,
+        template_kind=template_kind,
+        extra={"name_kwd": neigh_list},
+    )
+
+    # Group chunk IDs by doc
+    by_doc: dict[str, set[str]] = {}
+    for r in neigh_rows.values():
+        doc_id = r.get("doc_id") or ""
+        for cid in r.get("source_chunk_ids") or []:
+            if cid and cid not in seen_ids:
+                by_doc.setdefault(doc_id, set()).add(cid)
+
+    # -- 5. Load and return --
+    new_chunks: list[dict] = []
+    for doc_id, cids in by_doc.items():
+        if len(new_chunks) >= max_chunks:
+            break
+        limit = max_chunks - len(new_chunks)
+        chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
+        for c in chunks:
+            cid = c.get("chunk_id") or c.get("id")
+            if cid and cid not in seen_ids:
+                seen_ids.add(cid)
+                new_chunks.append(c)
+
+    return new_chunks
+
+
+async def _search_synthesis_pages(
+    tools,
+    kb_id: str,
+    tenant_id: str,
+    doc_ids: list[str] | None,
+    text: str,
+    *,
+    compile_kwd: str = "wiki_page",
+    top_n: int = 8,
+) -> dict:
+    """Search synthesis-compiled page rows (no knowledge_graph_kwd filter).
+
+    Synthesis pages are standalone articles (wiki_page, artifact_page,
+    essence, etc.) with ``content_with_weight``, keyword index, and
+    vector.  They do NOT carry the ``knowledge_graph_kwd`` field (unlike
+    entity/relation rows from extraction).
+    """
+    from common import settings
+    from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
+    from common.misc_utils import thread_pool_exec
+    from rag.nlp import search
+
+    condition: dict = {"compile_kwd": compile_kwd, "available_int": 1}
+    if doc_ids:
+        condition["source_doc_ids"] = list(doc_ids)
+
+    fields = [
+        "content_with_weight",
+        "summary_with_weight",
+        "source_chunk_ids",
+        "doc_id",
+        "title_kwd",
+        "topic_kwd",
+    ]
+
+    exprs = []
+    if text:
+        embd_mdl = getattr(tools, "embed_mdl", None)
+        if embd_mdl:
+            try:
+                exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+            except Exception:
+                _LOG.exception("[Wiki expand] vector build failed; using keyword match")
+        if not exprs:
+            exprs.append(
+                MatchTextExpr(
+                    ["content_ltks", "content_sm_ltks"],
+                    text,
+                    top_n,
+                )
+            )
+
+    try:
+        res = await thread_pool_exec(
+            settings.docStoreConn.search,
+            fields,
+            [],
+            condition,
+            exprs,
+            OrderByExpr(),
+            0,
+            top_n,
+            search.index_name(tenant_id),
+            [kb_id],
+        )
+        return settings.docStoreConn.get_fields(res, fields) or {}
+    except Exception:
+        _LOG.exception("[Wiki expand] search failed for kb=%s", kb_id)
+        return {}
+
+
+async def _expand_wiki_page_strategy(
+    tools,
+    kb_id: str,
+    tenant_id: str,
+    doc_ids: list[str] | None,
+    query: str,
+    seen_ids: set[str],
+    *,
+    compile_kwd: str = "wiki_page",
+    max_chunks: int = 5,
+) -> list[dict]:
+    """Expand synthesis-compiled pages: semantic search → load source chunks.
+
+    Unlike ``_expand_compiled_strategy`` (which does 1-hop entity-graph
+    navigation), synthesis pages are standalone rendered articles — we search
+    them directly and load the referenced source chunks as context.
+
+    *compile_kwd* selects which synthesis type: "wiki_page" (Wiki template),
+    "artifact_page" (Session Graph synthesis), "essence" (Session Essence).
+    """
+    # -- 1. Search synthesis pages --
+    wiki_rows = await _search_synthesis_pages(
+        tools,
+        kb_id,
+        tenant_id,
+        doc_ids,
+        query,
+        compile_kwd=compile_kwd,
+        top_n=5,
+    )
+    if not wiki_rows:
+        return []
+
+    # -- 2. Collect source_chunk_ids from matching pages --
+    by_doc: dict[str, set[str]] = {}
+    for r in wiki_rows.values():
+        doc_id = r.get("doc_id") or ""
+        for cid in r.get("source_chunk_ids") or []:
+            if cid and cid not in seen_ids:
+                by_doc.setdefault(doc_id, set()).add(cid)
+
+    # -- 3. Load chunks, assign high similarity for priority ranking --
+    new_chunks: list[dict] = []
+    for doc_id, cids in by_doc.items():
+        if len(new_chunks) >= max_chunks:
+            break
+        limit = max_chunks - len(new_chunks)
+        chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
+        for c in chunks:
+            cid = c.get("chunk_id") or c.get("id")
+            if cid and cid not in seen_ids:
+                seen_ids.add(cid)
+                c.setdefault("similarity", 0.9)  # wiki pages rank high
+                new_chunks.append(c)
+
+    return new_chunks
+
+
 async def web_search(tools, query: str, keywords: str = "") -> dict:
     if not tools.has_web():
         return {"chunks": [], "doc_aggs": []}

rag/advanced_rag/knowlege_compile/runner.py

@@ -135,7 +135,7 @@ def load_active_templates(template_ids, tenant_id: str) -> list[tuple[str, dict]
             logging.warning("document_structure_compile: template %s config is invalid", template_id)
             continue
         kind = _compilation_template_kind(parser_cfg.get("kind"))
-        if not kind or kind == "artifacts":
+        if not kind or kind == "wiki":
             continue
         active_templates.append((template_id, parser_cfg))
     return active_templates

rag/svr/task_executor.py

@@ -136,7 +136,7 @@ TASK_TYPE_TO_PIPELINE_TASK_TYPE = {
     "graphrag": PipelineTaskType.GRAPH_RAG,
     "mindmap": PipelineTaskType.MINDMAP,
     "memory": PipelineTaskType.MEMORY,
-    "artifact": PipelineTaskType.ARTIFACT,
+    "wiki": PipelineTaskType.ARTIFACT,
     "skill": PipelineTaskType.SKILL,
     "structure_graph": PipelineTaskType.STRUCTURE_GRAPH,
     "structure_mindmap": PipelineTaskType.STRUCTURE_MINDMAP,
@@ -152,7 +152,7 @@ _KB_FANOUT_TASK_TYPES = [
     "graphrag",
     "raptor",
     "mindmap",
-    "artifact",
+    "wiki",
     "skill",
     "structure_graph",
     "structure_mindmap",

rag/svr/task_executor_refactor/dataset_wiki_generator.py

@@ -1028,7 +1028,7 @@ async def run_wiki(
             template = CompilationTemplateService.get_saved(template_id, ctx.tenant_id)
             config = template.get("config") if template else {}
             kind = _compilation_template_kind(config.get("kind") if isinstance(config, dict) else "")
-            if kind == "artifacts":
+            if kind == "wiki":
                 eligible.append((d, template_id))
                 break
     if not eligible:

rag/svr/task_executor_refactor/task_handler.py

@@ -255,7 +255,7 @@ class TaskHandler:
                 await self._run_graphrag(embedding_model)
             elif task_type == "mindmap":
                 ctx.progress_cb(1, "place holder")
-            elif task_type == "artifact":
+            elif task_type == "wiki":
                 from rag.svr.task_executor_refactor.dataset_wiki_generator import (
                     run_wiki,
                 )