Feat: More APIs for compiler list and deletion of compilation result. (#17425)

### Summary

 More APIs for comliler list and deletion of compilation result.

api/apps/restful_apis/agent_api.py

@@ -671,6 +671,12 @@ def prompts():
     )
 
 
+# Synthetic ``canvas_category`` the frontend passes to list compilation template
+# groups through the merged /agents endpoint. Also the ``type`` discriminator
+# stamped on group items in the merged response.
+_COMPILATION_TEMPLATE_GROUP_CATEGORY = "compilation_template_group"
+
+
 @manager.route("/agents", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
@@ -702,6 +708,75 @@ def list_agents(tenant_id):
     else:
         effective_owner_ids = list(authorized_owner_ids)
 
+    # Groups-only: an explicit ``compilation_template_group`` category returns
+    # just the caller's template groups (no agents) via list_saved, so the
+    # frontend can render a dedicated tab. list_saved paginates in Python.
+    if canvas_category == _COMPILATION_TEMPLATE_GROUP_CATEGORY:
+        from api.db.services.compilation_template_group_service import CompilationTemplateGroupService
+
+        try:
+            groups = CompilationTemplateGroupService.list_saved(tenant_id, keywords, "", order_by, desc)
+        except Exception:
+            logging.exception("list_agents: compilation template group list failed for tenant=%s", tenant_id)
+            groups = []
+        for group in groups:
+            group["type"] = _COMPILATION_TEMPLATE_GROUP_CATEGORY
+        total = len(groups)
+        if page_number and items_per_page:
+            start = (page_number - 1) * items_per_page
+            groups = groups[start : start + items_per_page]
+        return get_json_result(data={"canvas": groups, "total": total})
+
+    # Merge mode: with no ``canvas_category`` (and no agent-only filters), list
+    # the caller's compilation template groups alongside agents, interleaved by
+    # ``update_time``. ``canvas_type`` / ``tags`` are agent-only concepts, so
+    # their presence keeps the response agent-only.
+    merge_groups = not canvas_category and not canvas_type and not tags
+    if merge_groups:
+        from api.db.services.compilation_template_group_service import CompilationTemplateGroupService
+
+        # Fetch every matching agent (page_number=0 disables SQL pagination) so
+        # the two sources can be globally ordered before we page in Python.
+        agents, _ = UserCanvasService.get_by_tenant_ids(
+            effective_owner_ids,
+            tenant_id,
+            0,
+            0,
+            order_by,
+            desc,
+            keywords,
+            None,
+            tags,
+            canvas_type,
+        )
+        # Groups are owner-only (no team sharing), so they're scoped to the
+        # caller. Keyword filters the group name; scope is left unfiltered.
+        try:
+            groups = CompilationTemplateGroupService.list_saved(tenant_id, keywords, "", order_by, desc)
+        except Exception:
+            logging.exception("list_agents: compilation template group merge failed for tenant=%s", tenant_id)
+            groups = []
+
+        items: list[dict] = []
+        for agent in agents:
+            agent["type"] = "agent"
+            items.append(agent)
+        for group in groups:
+            group["type"] = _COMPILATION_TEMPLATE_GROUP_CATEGORY
+            group["title"] = group["name"]
+            items.append(group)
+
+        # Interleave by update_time (the requested merge key); items missing the
+        # field sort as oldest.
+        items.sort(key=lambda item: item.get("update_time") or 0, reverse=desc)
+
+        total = len(items)
+        if page_number and items_per_page:
+            start = (page_number - 1) * items_per_page
+            items = items[start : start + items_per_page]
+
+        return get_json_result(data={"canvas": items, "total": total})
+
     canvas, total = UserCanvasService.get_by_tenant_ids(
         effective_owner_ids,
         tenant_id,

api/apps/restful_apis/dataset_api.py

@@ -747,6 +747,47 @@ async def get_dataset_structure(tenant_id, dataset_id):
         return get_error_data_result(message="Internal server error")
 
 
+@manager.route("/datasets/<dataset_id>/artifacts_structure", methods=["DELETE"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def delete_dataset_structure(tenant_id, dataset_id):
+    """Delete the dataset-scope (KB-wide) structure graph for one kind.
+
+    DELETE /api/v1/datasets/<dataset_id>/artifacts_structure?kind=<kind>
+    Optional query param: wipe=false cancels the task without deleting stored rows.
+    """
+    kind = request.args.get("kind", "")
+    if isinstance(kind, str):
+        kind = kind.strip()
+    if not kind:
+        return get_error_data_result(
+            message="`kind` is required (one of: graph, mindmap, timeline, session_essence, session_graph).",
+            code=RetCode.ARGUMENT_ERROR,
+        )
+    if dataset_api_service._resolve_dataset_structure_kind(kind) is None:
+        return get_error_data_result(
+            message=f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph.",
+            code=RetCode.ARGUMENT_ERROR,
+        )
+    wipe_arg = (request.args.get("wipe", "true") or "true").strip().lower()
+    wipe = wipe_arg not in ("false", "0", "no", "off")
+    try:
+        success, result = dataset_api_service.delete_dataset_structure(
+            dataset_id,
+            tenant_id,
+            kind,
+            wipe=wipe,
+        )
+        if success:
+            return get_result(data=result)
+        if result == "No authorization.":
+            return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
+        return get_error_data_result(message=result)
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
 @manager.route("/datasets/<dataset_id>/artifacts/alteration", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs

api/apps/services/dataset_api_service.py

@@ -1721,6 +1721,13 @@ _DATASET_STRUCTURE_KIND_ALIASES = {
     "session_essence": "session_essence",
     "session_graph": "session_graph",
 }
+_DATASET_STRUCTURE_KIND_TO_INDEX_TYPE = {
+    "knowledge_graph": "structure_graph",
+    "mind_map": "structure_mindmap",
+    "timeline": "timeline",
+    "session_essence": "session_essence",
+    "session_graph": "session_graph",
+}
 
 
 def _resolve_dataset_structure_kind(kind) -> str | None:
@@ -1730,6 +1737,15 @@ def _resolve_dataset_structure_kind(kind) -> str | None:
     return _DATASET_STRUCTURE_KIND_ALIASES.get(kind.strip().lower().replace("-", "_"))
 
 
+def delete_dataset_structure(dataset_id: str, tenant_id: str, kind: str, wipe: bool = True):
+    """Delete the merged KB-wide structure rows for one artifacts_structure kind."""
+    resolved_kind = _resolve_dataset_structure_kind(kind)
+    if not resolved_kind:
+        return False, f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph."
+    index_type = _DATASET_STRUCTURE_KIND_TO_INDEX_TYPE[resolved_kind]
+    return delete_index(dataset_id, tenant_id, index_type, wipe=wipe)
+
+
 async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywords: str = ""):
     """Load the dataset-scope (KB-wide) structure graph for one ``kind``.
 

rag/advanced_rag/harness/tools/__init__.py

@@ -19,12 +19,14 @@ from rag.advanced_rag.harness.tools.navigation import catalog_navigate, dataset_
 # catalog_navigate covers both the tree/TOC outline and the page index.
 register_tool(
     "catalog_navigate",
-    _navigate_schema("catalog_navigate", "Answer from the document's compiled catalog (table of contents / page index)"),
+    _navigate_schema("catalog_navigate", "Get question-related chunks from the document's catalog (table of contents / page index)"),
     catalog_navigate,
     requires_compilation=True,
     compilation_type=("toc", "page_index"),
 )
-register_tool("mindmap_navigate", _navigate_schema("mindmap_navigate", "Navigate by mindmap"), mindmap_navigate, requires_compilation=True, compilation_type="mindmap")
+register_tool(
+    "mindmap_navigate", _navigate_schema("mindmap_navigate", "Get question-related chunks from the document's  mindmap"), mindmap_navigate, requires_compilation=True, compilation_type="mindmap"
+)
 register_tool(
     "dataset_navigate",
     _navigate_schema("dataset_navigate", "Find the most relevant documents via the dataset map, then search within them"),

rag/nlp/search.py

@@ -131,7 +131,7 @@ class Dealer:
             condition["must_not"] = req["must_not"]
         return condition
 
-    async def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=None, highlight: bool | list | None = None, rank_feature: dict | None = None):
+    async def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=None, highlight: bool | list | None = None, rank_feature: dict | None = None, min_match: bool = True):
         if highlight is None:
             highlight = False
 
@@ -189,7 +189,7 @@ class Dealer:
                 highlightFields = []
             elif isinstance(highlight, list):
                 highlightFields = highlight
-            matchText, keywords = self.qryr.question(qst, min_match=0.3)
+            matchText, keywords = self.qryr.question(qst, min_match=(0.3 if min_match else 0))
             if emb_mdl is None:
                 matchExprs = [matchText]
                 res = await thread_pool_exec(self.dataStore.search, src, highlightFields, filters, matchExprs, orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature)
@@ -220,7 +220,7 @@ class Dealer:
                         res = await thread_pool_exec(self.dataStore.search, src, [], filters, [], orderBy, offset, limit, idx_names, kb_ids)
                         total = self.dataStore.get_total(res)
                     else:
-                        matchText, _ = self.qryr.question(qst, min_match=0.1)
+                        matchText, _ = self.qryr.question(qst, min_match=(0.1 if min_match else 0))
                         matchDense.extra_options["similarity"] = 0.17
                         res = await thread_pool_exec(
                             self.dataStore.search, src, highlightFields, filters, [matchText, matchDense, fusionExpr], orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature
@@ -593,7 +593,8 @@ class Dealer:
             tenant_ids = tenant_ids.split(",")
 
         idx_names = [index_name(tid) for tid in tenant_ids]
-        sres = await self.search(req, idx_names, kb_ids, embd_mdl, highlight, rank_feature=rank_feature)
+        min_match = vector_similarity_weight < 0.8
+        sres = await self.search(req, idx_names, kb_ids, embd_mdl, highlight, rank_feature=rank_feature, min_match=min_match)
         # Temporary retrieval-side guard: prune chunks whose parent document no
         # longer exists before reranking and returning results.
         sres = await self._prune_deleted_chunks(sres)

rag/utils/es_conn.py

@@ -30,6 +30,7 @@ from common.constants import PAGERANK_FLD, TAG_FLD
 ATTEMPT_TIME = 2
 MAX_RESULT_WINDOW = 10000
 SEARCH_AFTER_BATCH_SIZE = 1000
+KNN_QUERY_STRING_FILTER_WEIGHT_THRESHOLD = 0.8
 
 # Single-document atomic pagerank_fea adjust (chunk feedback). Clamps using params.min_w / max_w;
 # removes field at zero for rank_feature compatibility.
@@ -58,6 +59,43 @@ if (nw <= 0.0) {
 """
 
 
+def _is_query_string_clause(clause: dict) -> bool:
+    return isinstance(clause, dict) and "query_string" in clause
+
+
+def _remove_query_string_must_clauses(must_clauses):
+    if isinstance(must_clauses, list):
+        return [copy.deepcopy(clause) for clause in must_clauses if not _is_query_string_clause(clause)]
+    if _is_query_string_clause(must_clauses):
+        return []
+    return [copy.deepcopy(must_clauses)] if must_clauses else []
+
+
+def _build_knn_filter_query(bool_query, vector_similarity_weight: float):
+    if bool_query is None:
+        return None
+
+    query = bool_query.to_dict()
+    if vector_similarity_weight <= KNN_QUERY_STRING_FILTER_WEIGHT_THRESHOLD:
+        return query
+
+    query = copy.deepcopy(query)
+    bool_part = query.get("bool")
+    if not isinstance(bool_part, dict):
+        return query
+
+    if "must" in bool_part:
+        must_clauses = _remove_query_string_must_clauses(bool_part["must"])
+        if must_clauses:
+            bool_part["must"] = must_clauses
+        else:
+            bool_part.pop("must", None)
+
+    if not any(bool_part.get(key) for key in ("must", "filter", "must_not", "should")):
+        return None
+    return query
+
+
 @singleton
 class ESConnection(ESConnectionBase):
     """
@@ -217,7 +255,7 @@ class ESConnection(ESConnectionBase):
                     m.topn,
                     m.topn * 2,
                     query_vector=list(m.embedding_data),
-                    filter=bool_query.to_dict(),
+                    filter=bool_query.to_dict(),  # filter=_build_knn_filter_query(bool_query, vector_similarity_weight),
                     similarity=similarity,
                 )
 

rag/utils/infinity_conn.py

@@ -24,8 +24,11 @@ import pandas as pd
 from common.constants import PAGERANK_FLD, TAG_FLD
 from common.doc_store.doc_store_base import MatchExpr, MatchTextExpr, MatchDenseExpr, FusionExpr, OrderByExpr
 from common.doc_store.infinity_conn_base import InfinityConnectionBase
+from common.float_utils import get_float
 
 
+DENSE_FILTER_FULLTEXT_WEIGHT_THRESHOLD = 0.8
+DEFAULT_VECTOR_SIMILARITY_WEIGHT = 0.5
 _JSON_LIST_FIELDS = frozenset(
     (
         "source_chunk_ids",
@@ -40,6 +43,27 @@ _JSON_LIST_FIELDS = frozenset(
 )
 
 
+def _vector_similarity_weight(match_expressions: list[MatchExpr]) -> float:
+    vector_similarity_weight = DEFAULT_VECTOR_SIMILARITY_WEIGHT
+    for matchExpr in match_expressions:
+        if not isinstance(matchExpr, FusionExpr) or matchExpr.method != "weighted_sum":
+            continue
+        fusion_params = matchExpr.fusion_params or {}
+        weights = fusion_params.get("weights")
+        if not weights:
+            continue
+        weight_parts = str(weights).split(",")
+        if len(weight_parts) > 1:
+            vector_similarity_weight = get_float(weight_parts[1])
+    return vector_similarity_weight
+
+
+def _build_dense_filter(filter_cond: str | None, filter_fulltext: str | None, vector_similarity_weight: float) -> str:
+    if vector_similarity_weight > DENSE_FILTER_FULLTEXT_WEIGHT_THRESHOLD:
+        return filter_cond or ""
+    return filter_fulltext or filter_cond or ""
+
+
 @singleton
 class InfinityConnection(InfinityConnectionBase):
     """
@@ -191,6 +215,7 @@ class InfinityConnection(InfinityConnectionBase):
                     self.logger.error(f"No valid tables found for indexNames {index_names} and knowledgebaseIds {knowledgebase_ids}")
                     return pd.DataFrame(), 0
 
+            # vector_similarity_weight = _vector_similarity_weight(match_expressions)
             for matchExpr in match_expressions:
                 if isinstance(matchExpr, MatchTextExpr):
                     if filter_cond and "filter" not in matchExpr.extra_options:
@@ -223,6 +248,9 @@ class InfinityConnection(InfinityConnectionBase):
                 elif isinstance(matchExpr, MatchDenseExpr):
                     if filter_fulltext and "filter" not in matchExpr.extra_options:
                         matchExpr.extra_options.update({"filter": filter_fulltext})
+                    # dense_filter = _build_dense_filter(filter_cond, filter_fulltext, vector_similarity_weight)
+                    # if dense_filter and "filter" not in matchExpr.extra_options:
+                    #    matchExpr.extra_options.update({"filter": dense_filter})
                     for k, v in matchExpr.extra_options.items():
                         if not isinstance(v, str):
                             matchExpr.extra_options[k] = str(v)