Generate navigation, navigation search (#18096)

### Summary
2 API

POST /api/v1/datasets/{dataset_id}/navigation

GET
/api/v1/datasets/{dataset_id}/navigation/search?q={query}&mode={mode}&top_k={topk}


2 cli

uv run --no-sync python3 admin/client/ragflow_cli.py -h 127.0.0.1 -p
9380 -t user

ragflow> GENERATE NAVIGATION OF DATASET 'frames tree';

ragflow> NAVIGATION SEARCH 'Christie introduced blockchain-based digital
passports' IN DATASET 'frames tree' MODE 'all' topk 20;

(mode: chunk, nav_cluster, nav_doc, navigation_tree, all)

api/apps/restful_apis/dataset_api.py

@@ -16,10 +16,11 @@
 import logging
 
 from peewee import OperationalError
-from quart import request, make_response
-from common.constants import RetCode
-from api.apps import login_required, current_user
-from api.utils.api_utils import get_error_argument_result, get_error_data_result, get_json_result, get_result, add_tenant_id_to_kwargs
+from quart import make_response, request
+
+from api.apps import current_user, login_required
+from api.apps.services import dataset_api_service
+from api.utils.api_utils import add_tenant_id_to_kwargs, get_error_argument_result, get_error_data_result, get_json_result, get_result
 from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_size
 from api.utils.validation_utils import (
     CreateDatasetReq,
@@ -31,7 +32,7 @@ from api.utils.validation_utils import (
     validate_and_parse_json_request,
     validate_and_parse_request_args,
 )
-from api.apps.services import dataset_api_service
+from common.constants import RetCode
 
 
 @manager.route("/datasets/tags/aggregation", methods=["GET"])  # noqa: F821
@@ -980,6 +981,51 @@ async def list_dataset_nav(tenant_id, dataset_id):
         return get_error_data_result(message="Internal server error")
 
 
+@manager.route("/datasets/<dataset_id>/navigation/search", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def search_dataset_nav(tenant_id, dataset_id):
+    """Unified navigation search across different knowledge layers.
+
+    GET /api/v1/datasets/<dataset_id>/navigation/search?q=<query>&mode=<mode>&top_k=20
+
+    Modes:
+      - nav_doc:         navigation tree document leaves (default)
+      - nav_cluster:     navigation tree cluster nodes
+      - navigation_tree: tree-structured BFS beam descent
+      - chunk:           raw document chunks (deduplicated by doc_id)
+      - all:             union of all modes above
+
+    Success: {"code": 0, "data": {"mode": <mode>, "total": <n>,
+        "items": [{"doc_id": str, "score": float}, ...]}}
+    """
+    q = (request.args.get("q") or "").strip()
+    if not q:
+        return get_result(data={"mode": request.args.get("mode", "nav_doc"), "total": 0, "items": []})
+    mode = (request.args.get("mode") or "nav_doc").strip()
+    top_k_raw = request.args.get("top_k")
+    top_k = None
+    if top_k_raw:
+        try:
+            top_k = max(1, int(top_k_raw))
+        except (ValueError, TypeError):
+            return get_error_data_result(message="top_k must be a positive integer")
+    try:
+        success, result = await dataset_api_service.search_dataset_layers(
+            dataset_id,
+            tenant_id,
+            q,
+            mode,
+            top_k=top_k,
+        )
+        if success:
+            return get_result(data=result)
+        return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
 @manager.route("/datasets/<dataset_id>/navigation/<path:name>/children", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
@@ -1048,6 +1094,38 @@ async def delete_dataset_nav_node(tenant_id, dataset_id, name):
         return get_error_data_result(message="Internal server error")
 
 
+@manager.route("/datasets/<dataset_id>/navigation", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def generate_dataset_nav(tenant_id, dataset_id):
+    """Create the entire navigation tree from all dataset documents.
+
+    Deletes any existing navigation tree first, then rebuilds it from
+    scratch. When ``documents`` is provided, only those docs are used;
+    otherwise all docs in the dataset are auto-discovered.
+
+    POST /api/v1/datasets/<dataset_id>/navigation
+    Body (optional): {"documents": [{"doc_id": "...", "summary": "...",
+           "doc_title": "... (optional)", "source_type": "... (optional)"}, ...]}
+    Success: {"code": 0, "data": {"deleted": <n>, "upserted": <n>}}
+    """
+    try:
+        req = await request.json or {}
+        documents = req.get("documents")
+
+        success, result = await dataset_api_service.generate_nav(
+            dataset_id,
+            tenant_id,
+            documents,
+        )
+        if success:
+            return get_result(data=result)
+        return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
 @manager.route("/datasets/<dataset_id>/skills/<path:skill_kwd>", methods=["DELETE"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs

api/apps/services/dataset_api_service.py

@@ -13,25 +13,24 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 import json
+import logging
 import os
 import re
 
-from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_id, get_composite_model_name_by_ids
-from common.constants import PAGERANK_FLD, LLMType
-from common import settings
 from api.db.db_models import File
+from api.db.joint_services.tenant_model_service import get_composite_model_name_by_ids, resolve_model_config, resolve_model_id
+from api.db.services.connector_service import Connector2KbService
 from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService, validate_dataset_embedding_models
-from api.db.services.connector_service import Connector2KbService
 from api.db.services.task_service import GRAPH_RAPTOR_FAKE_DOC_ID, TaskService
 from api.db.services.tenant_model_service import TenantModelService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
-from common.constants import FileSource, StatusEnum
 from api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_keys, verify_embedding_availability
+from common import settings
+from common.constants import PAGERANK_FLD, FileSource, LLMType, StatusEnum
 from common.misc_utils import thread_pool_exec, thread_pool_exec_long_time
 from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD
 
@@ -382,8 +381,7 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):
             from rag.nlp import search
 
             settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD}, search.index_name(kb.tenant_id), kb.id)
-    if "parse_type" in req:
-        del req["parse_type"]
+    req.pop("parse_type", None)
 
     if not KnowledgebaseService.update_by_id(kb.id, req):
         return False, "Update dataset error.(Database error)"
@@ -547,8 +545,8 @@ def delete_knowledge_graph(dataset_id: str, tenant_id: str):
     if not KnowledgebaseService.accessible(dataset_id, tenant_id):
         return False, "no authorization"
     _, kb = KnowledgebaseService.get_by_id(dataset_id)
-    from rag.nlp import search
     from rag.graphrag.phase_markers import clear_phase_markers
+    from rag.nlp import search
 
     settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation", "community_report"]}, search.index_name(kb.tenant_id), dataset_id)
     # Wiping the graph invalidates any phase-completion markers used to
@@ -934,8 +932,8 @@ def delete_index(dataset_id: str, tenant_id: str, index_type: str, wipe: bool =
         TaskService.delete_by_id(task_id)
 
     if wipe and index_type == "graph":
-        from rag.nlp import search
         from rag.graphrag.phase_markers import clear_phase_markers
+        from rag.nlp import search
 
         settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation", "community_report"]}, search.index_name(kb.tenant_id), dataset_id)
         # Wiping the graph invalidates any phase-completion markers used to
@@ -1173,12 +1171,11 @@ def check_embedding(dataset_id: str, tenant_id: str, req: dict):
     import random
 
     import numpy as np
-    from common.constants import RetCode
-    from common.doc_store.doc_store_base import OrderByExpr
-    from rag.nlp import search
 
     from api.db.services.llm_service import LLMBundle
-    from common.constants import LLMType
+    from common.constants import LLMType, RetCode
+    from common.doc_store.doc_store_base import OrderByExpr
+    from rag.nlp import search
 
     def _guess_vec_field(src: dict):
         for k in src or {}:
@@ -1843,10 +1840,10 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
         return True, empty
     index_nm, _ = pack
 
-    from common.doc_store.doc_store_base import OrderByExpr
+    from api.apps.services import structure_graph_common as sgc
     from api.db.services.compilation_template_service import CompilationTemplateService
     from api.db.services.tenant_llm_service import TenantLLMService
-    from api.apps.services import structure_graph_common as sgc
+    from common.doc_store.doc_store_base import OrderByExpr
 
     keywords = (keywords or "").strip()
     _, active_doc_ids = await _current_dataset_docs(dataset_id)
@@ -3125,7 +3122,10 @@ def _nav_item(row: dict) -> dict:
     return {
         "name": row.get("name") or "",
         "description": payload.get("description") or "",
-        # doc_id count under this node: the cluster's tally, or 1 for a leaf.
+        "keywords": list(payload.get("keywords") or []),
+        "entities": list(payload.get("entities") or []),
+        "graph_content": payload.get("graph_content") or "",
+        # doc_id count under this node: the cluster`s tally, or 1 for a leaf.
         "doc_count": int(row.get("doc_count_int") or 0) if is_cluster else 1,
         "type": "cluster" if is_cluster else "doc",
         "doc_id": None if is_cluster else (row.get("doc_id") or row.get("name")),
@@ -3348,6 +3348,450 @@ async def delete_nav_node(dataset_id: str, tenant_id: str, name: str):
             logging.exception("delete_nav_node: lock release failed for kb=%s", dataset_id)
 
 
+async def generate_nav(
+    dataset_id: str,
+    tenant_id: str,
+    documents: list[dict] | None = None,
+):
+    """Create the entire navigation tree.
+
+    Deletes any existing navigation tree first, then rebuilds it from
+    scratch. When ``documents`` is provided, only those doc→summary pairs
+    are used; otherwise all documents in the dataset are auto-discovered
+    and inserted into the tree.
+
+    ``documents`` is a list of ``{"doc_id": str, "summary": str,
+    "doc_title": str (optional), "source_type": str (optional)}``.
+
+    Returns ``(True, {"deleted": <n>, "upserted": <n>})`` on success.
+    """
+    if not KnowledgebaseService.accessible(dataset_id, tenant_id):
+        return False, "no authorization"
+
+    _, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if kb is None:
+        return False, "Dataset not found."
+
+    # Resolve models.
+    from api.db.joint_services.tenant_model_service import (
+        get_tenant_default_model_by_type,
+    )
+    from api.db.services.llm_service import LLMBundle
+    from common.constants import LLMType
+    from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc
+
+    if kb.embd_id:
+        embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+    else:
+        embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)
+    embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
+
+    chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
+    chat_mdl = LLMBundle(kb.tenant_id, chat_model_config)
+
+    # Step 0: auto-discover documents when not explicitly provided.
+    if not documents:
+        try:
+            # Query all documents in the dataset directly without
+            # File / File2Document JOINs so that every document
+            # participates in the navigation tree (including docs
+            # created via API that have no file record).
+            from api.db.db_utils import DB
+            from api.db.services.doc_metadata_service import DocMetadataService
+
+            with DB.connection_context():
+                doc_rows = list(
+                    DocumentService.model.select(
+                        DocumentService.model.id,
+                        DocumentService.model.name,
+                    )
+                    .where(
+                        DocumentService.model.kb_id == dataset_id,
+                    )
+                    .dicts()
+                )
+            doc_ids = [str(row["id"]) for row in doc_rows]
+            metadata_map = DocMetadataService.get_metadata_for_documents(doc_ids, dataset_id) if doc_ids else {}
+            all_docs = []
+            for row in doc_rows:
+                did = str(row["id"])
+                all_docs.append(
+                    {
+                        "id": did,
+                        "name": (row.get("name") or ""),
+                        "meta_fields": metadata_map.get(did, {}),
+                    }
+                )
+
+            # Prefer RAPTOR-generated summaries stored in the knowledge
+            # graph (compile_kwd="tree", knowledge_graph_kwd="graph")
+            # so that rebuilt nav descriptions match the original
+            # tree-compilation output.  Fall back to meta_fields.title
+            # or filename when the graph is absent.
+            raptor_summaries: dict[str, str] = {}
+            pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
+            if pack is not None:
+                try:
+                    index_nm, _ = pack
+                    from common.doc_store.doc_store_base import OrderByExpr
+
+                    graph_res = settings.docStoreConn.search(
+                        select_fields=["doc_id", "content_with_weight"],
+                        highlight_fields=[],
+                        condition={"compile_kwd": ["tree"], "knowledge_graph_kwd": ["graph"]},
+                        match_expressions=[],
+                        order_by=OrderByExpr(),
+                        offset=0,
+                        limit=10000,
+                        index_names=index_nm,
+                        knowledgebase_ids=[dataset_id],
+                    )
+                    graph_map = settings.docStoreConn.get_fields(graph_res, ["doc_id", "content_with_weight"])
+                    for row in (graph_map or {}).values():
+                        gid = str(row.get("doc_id") or "")
+                        if not gid:
+                            continue
+                        try:
+                            graph = json.loads(row.get("content_with_weight") or "{}")
+                        except Exception:
+                            continue
+                        entities = graph.get("entities") or []
+                        relations = graph.get("relations") or []
+                        child_names = {r.get("to") for r in relations if isinstance(r, dict)}
+
+                        # Build both:
+                        #   - root_summary: first line of root desc (short,
+                        #     for the display title / description field)
+                        #   - graph_text: structured text from ALL entities
+                        #     and relations (for embedding, keyword extraction,
+                        #     entity extraction, and stored as graph_content).
+                        # entity name -> description
+                        name_desc: dict[str, str] = {}
+                        for ent in entities:
+                            if not isinstance(ent, dict):
+                                continue
+                            nm = (ent.get("name") or "").strip()
+                            if nm:
+                                name_desc[nm] = (ent.get("description") or "").strip()
+
+                        # root entity = entity whose name never appears as a
+                        # relation target (not a child of anyone).
+                        root_name = ""
+                        root_summary = ""
+                        for ent in entities:
+                            if isinstance(ent, dict) and ent.get("name") not in child_names:
+                                root_name = (ent.get("name") or "").strip()
+                                root_summary = name_desc.get(root_name, "")
+                                root_summary = root_summary.splitlines()[0].strip() if root_summary else root_name
+                                break
+
+                        # Build the full graph text
+                        graph_parts: list[str] = []
+                        if root_name and name_desc.get(root_name):
+                            graph_parts.append(root_name)
+                            graph_parts.append(name_desc[root_name])
+
+                        child_names_set = {n for n in name_desc if n in child_names}
+                        if child_names_set:
+                            graph_parts.append("")
+                            for cname in sorted(child_names_set):
+                                cdesc = name_desc.get(cname, "")
+                                line = f"- {cname}"
+                                if cdesc:
+                                    line += f": {cdesc.splitlines()[0].strip()}"
+                                graph_parts.append(line)
+
+                        graph_text = "\n".join(graph_parts) if graph_parts else ""
+
+                        if root_summary:
+                            raptor_summaries[gid] = {
+                                "title": root_summary,
+                                "graph_text": graph_text or root_summary,
+                            }
+                except Exception:
+                    logging.exception("generate_nav: failed to read RAPTOR graph summaries for kb=%s", dataset_id)
+
+            documents = []
+            for d in all_docs:
+                doc_id = str(d.get("id", ""))
+                if not doc_id:
+                    continue
+                if doc_id in raptor_summaries:
+                    summary = raptor_summaries[doc_id]
+                else:
+                    meta = d.get("meta_fields") or {}
+                    summary = (meta.get("title") or "").strip() or d.get("name", "")
+                documents.append(
+                    {
+                        "doc_id": doc_id,
+                        "summary": summary,
+                    }
+                )
+
+        except Exception:
+            logging.exception("generate_nav: failed to auto-discover docs for kb=%s", dataset_id)
+            return False, "Failed to auto-discover documents."
+
+    if not documents:
+        return False, "No documents found in dataset."
+
+    # Step 1: delete the entire existing navigation tree so we start clean.
+    deleted = 0
+    pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
+    if pack is not None:
+        index_nm, _ = pack
+        try:
+            deleted = await thread_pool_exec(
+                settings.docStoreConn.delete,
+                {"compile_kwd": [_NAV_COMPILE_KWD]},
+                index_nm,
+                dataset_id,
+            )
+            deleted = int(deleted or 0)
+        except Exception:
+            logging.exception("generate_nav: failed to clear existing nav for kb=%s", dataset_id)
+            return False, "Failed to clear existing navigation tree."
+
+    # Step 2: rebuild the tree from the provided doc→summary pairs.
+    upserted = 0
+    for doc in documents or []:
+        doc_id = (doc.get("doc_id") or "").strip()
+        summary = doc.get("summary")
+        # summary can be a plain string or a RAPTOR tree dict.
+        if isinstance(summary, str):
+            summary = summary.strip()
+        if not doc_id or not summary:
+            continue
+        try:
+            await upsert_dataset_nav_doc(
+                tenant_id=tenant_id,
+                kb_id=dataset_id,
+                doc_id=doc_id,
+                summary_or_tree=summary,
+                embd_mdl=embd_mdl,
+                chat_mdl=chat_mdl,
+            )
+            upserted += 1
+        except Exception:
+            logging.exception("generate_nav: failed for doc=%s kb=%s", doc_id, dataset_id)
+            return True, {"deleted": deleted, "upserted": upserted, "failed_doc_id": doc_id}
+
+    return True, {"deleted": deleted, "upserted": upserted}
+
+
+# ---------------------------------------------------------------------------
+# Unified Explore Search
+# ---------------------------------------------------------------------------
+
+_LAYERS_HANDLERS: dict[str, str] = {
+    "nav_doc": "_search_layers_nav_docs",
+    "nav_cluster": "_search_layers_nav_clusters",
+    "navigation_tree": "_search_layers_navigation_tree",
+    "chunk": "_search_layers_chunks",
+    "all": "_search_layers_all",
+}
+
+
+async def search_dataset_layers(
+    dataset_id: str,
+    tenant_id: str,
+    query: str,
+    mode: str,
+    *,
+    top_k: int | None = None,
+) -> tuple[bool, dict]:
+    """Unified search across different knowledge layers of a dataset.
+
+    Args:
+        mode: One of ``"chunk"``, ``"nav_doc"``, ``"nav_cluster"``, ``"navigation_tree"``, ``"all"``.
+            - chunk: raw document chunks (via the main retrieval pipeline)
+            - nav_doc: navigation tree document leaves
+            - nav_cluster: navigation tree cluster nodes
+            - navigation_tree: tree-structured BFS beam descent
+            - all: union of all modes, deduplicated by doc_id with best score
+
+        Items are shaped as ``{"doc_id": str, "score": float}``.
+    """
+    from rag.advanced_rag.knowlege_compile.dataset_nav import search_dataset_nav
+
+    if not KnowledgebaseService.accessible(dataset_id, tenant_id):
+        return False, "no authorization"
+    if mode not in _LAYERS_HANDLERS:
+        return False, f"unknown mode: {mode}, expected one of {list(_LAYERS_HANDLERS.keys())}"
+    _, kb = KnowledgebaseService.get_by_id(dataset_id)
+
+    try:
+        from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type
+        from api.db.services.llm_service import LLMBundle
+
+        if kb.embd_id:
+            embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+        else:
+            embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)
+        embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
+    except Exception as e:
+        logging.warning(
+            "search_dataset_layers: failed to create LLMBundle(EMBEDDING) for tenant=%s: %s: %s",
+            kb.tenant_id,
+            type(e).__name__,
+            e,
+        )
+        logging.exception("Full traceback for LLMBundle(EMBEDDING) failure")
+        embd_mdl = None
+
+    if mode == "nav_doc":
+        return await _search_layers_nav_docs(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav)
+    elif mode == "nav_cluster":
+        return await _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav)
+    elif mode == "navigation_tree":
+        return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav)
+    elif mode == "chunk":
+        return await _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb)
+    elif mode == "all":
+        return await _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb, search_dataset_nav)
+    else:
+        return False, f"unknown mode: {mode}"
+
+
+async def _search_layers_nav_docs(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn):
+    items = await _nav_search_result(
+        tenant_id,
+        dataset_id,
+        query,
+        top_k,
+        embd_mdl,
+        search_fn,
+        type_kwd="nav_doc",
+    )
+    return True, {"mode": "nav_doc", "total": len(items), "items": items}
+
+
+async def _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn):
+    items = await _nav_search_result(
+        tenant_id,
+        dataset_id,
+        query,
+        top_k,
+        embd_mdl,
+        search_fn,
+        type_kwd="nav_cluster",
+    )
+    return True, {"mode": "nav_cluster", "total": len(items), "items": items}
+
+
+async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn):
+    from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent
+
+    items = await search_nav_tree_descent(
+        tenant_id,
+        dataset_id,
+        query,
+        embd_mdl,
+        top_k=top_k,
+    )
+    return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+
+
+async def _nav_search_result(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, **kwargs):
+    results = await search_fn(
+        tenant_id,
+        dataset_id,
+        query,
+        embd_mdl=embd_mdl,
+        top_k=top_k,
+        **kwargs,
+    )
+    items: list[dict] = []
+    for r in results:
+        doc_id = (r.get("doc_id") or "").strip() if isinstance(r.get("doc_id"), str) else (r.get("doc_ids") or [None])[0]
+        items.append(
+            {
+                "doc_id": str(doc_id) if doc_id else "",
+                "score": round(float(r.get("score", 0.0)), 4),
+            }
+        )
+    return items
+
+
+async def _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb):
+    from common import settings
+
+    tenant_ids = [tenant_id]
+
+    kwargs = {}
+    if top_k is not None:
+        kwargs["top"] = top_k
+
+    fetch_k = max(top_k, 10) * 3 if top_k is not None else 1024
+    try:
+        ranks = await settings.retriever.retrieval(
+            query,
+            embd_mdl,
+            tenant_ids,
+            [dataset_id],
+            1,
+            fetch_k,
+            0.0,
+            0.3,
+            **kwargs,
+        )
+    except Exception:
+        return False, "chunk retrieval failed"
+
+    doc_scores: dict[str, float] = {}
+    for c in ranks.get("chunks", []):
+        doc_id = (c.get("doc_id") or "").strip()
+        score = float(c.get("similarity") or c.get("score") or 0.0)
+        if doc_id and score > doc_scores.get(doc_id, -1.0):
+            doc_scores[doc_id] = score
+
+    items = sorted(
+        ({"doc_id": d, "score": round(s, 4)} for d, s in doc_scores.items()),
+        key=lambda x: x["score"],
+        reverse=True,
+    )
+    if top_k is not None and top_k > 0:
+        items = items[:top_k]
+
+    return True, {"mode": "chunk", "total": len(items), "items": items}
+
+
+async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb, search_fn):
+    """Run all modes and return the union of doc_ids, with best score per doc."""
+    import asyncio as _asyncio
+
+    result_lists = await _asyncio.gather(
+        _search_layers_nav_docs(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn),
+        _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn),
+        _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn),
+        _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb),
+        return_exceptions=True,
+    )
+
+    doc_scores: dict[str, float] = {}
+    for result in result_lists:
+        if isinstance(result, Exception):
+            continue
+        ok, data = result
+        if not ok:
+            continue
+        for item in data.get("items", []):
+            doc_id = item.get("doc_id", "")
+            score = float(item.get("score", 0.0))
+            if doc_id and score > doc_scores.get(doc_id, -1.0):
+                doc_scores[doc_id] = score
+
+    items = sorted(
+        ({"doc_id": d, "score": round(s, 4)} for d, s in doc_scores.items()),
+        key=lambda x: x["score"],
+        reverse=True,
+    )
+    if top_k is not None and top_k > 0:
+        items = items[:top_k]
+
+    return True, {"mode": "all", "total": len(items), "items": items}
+
+
 async def update_wiki_page(
     dataset_id: str,
     tenant_id: str,
@@ -3390,11 +3834,11 @@ async def update_wiki_page(
         return True, None
     index_nm, _ = pack
 
-    from rag.advanced_rag.knowlege_compile.wiki import (
-        _wiki_transform_links,
-        _wiki_extract_summary,
-    )
     from api.db.services.file_commit_service import FileCommitService
+    from rag.advanced_rag.knowlege_compile.wiki import (
+        _wiki_extract_summary,
+        _wiki_transform_links,
+    )
 
     full_slug = f"{page_type}/{slug}" if "/" not in slug else slug