From f9e47ae0b90a645ec9c751611b3ccf7b5192e3b0 Mon Sep 17 00:00:00 2001 From: Kevin Hu Date: Mon, 27 Jul 2026 20:11:18 +0800 Subject: [PATCH] Feat: More APIs for compiler list and deletion of compilation result. (#17425) ### Summary More APIs for comliler list and deletion of compilation result. --- api/apps/restful_apis/agent_api.py | 75 ++++++++++++++++++++++ api/apps/restful_apis/dataset_api.py | 41 ++++++++++++ api/apps/services/dataset_api_service.py | 16 +++++ rag/advanced_rag/harness/tools/__init__.py | 6 +- rag/nlp/search.py | 9 +-- rag/utils/es_conn.py | 40 +++++++++++- rag/utils/infinity_conn.py | 28 ++++++++ 7 files changed, 208 insertions(+), 7 deletions(-) diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py index ded7a5b37c..40ee2c6ee0 100644 --- a/api/apps/restful_apis/agent_api.py +++ b/api/apps/restful_apis/agent_api.py @@ -671,6 +671,12 @@ def prompts(): ) +# Synthetic ``canvas_category`` the frontend passes to list compilation template +# groups through the merged /agents endpoint. Also the ``type`` discriminator +# stamped on group items in the merged response. +_COMPILATION_TEMPLATE_GROUP_CATEGORY = "compilation_template_group" + + @manager.route("/agents", methods=["GET"]) # noqa: F821 @login_required @add_tenant_id_to_kwargs @@ -702,6 +708,75 @@ def list_agents(tenant_id): else: effective_owner_ids = list(authorized_owner_ids) + # Groups-only: an explicit ``compilation_template_group`` category returns + # just the caller's template groups (no agents) via list_saved, so the + # frontend can render a dedicated tab. list_saved paginates in Python. + if canvas_category == _COMPILATION_TEMPLATE_GROUP_CATEGORY: + from api.db.services.compilation_template_group_service import CompilationTemplateGroupService + + try: + groups = CompilationTemplateGroupService.list_saved(tenant_id, keywords, "", order_by, desc) + except Exception: + logging.exception("list_agents: compilation template group list failed for tenant=%s", tenant_id) + groups = [] + for group in groups: + group["type"] = _COMPILATION_TEMPLATE_GROUP_CATEGORY + total = len(groups) + if page_number and items_per_page: + start = (page_number - 1) * items_per_page + groups = groups[start : start + items_per_page] + return get_json_result(data={"canvas": groups, "total": total}) + + # Merge mode: with no ``canvas_category`` (and no agent-only filters), list + # the caller's compilation template groups alongside agents, interleaved by + # ``update_time``. ``canvas_type`` / ``tags`` are agent-only concepts, so + # their presence keeps the response agent-only. + merge_groups = not canvas_category and not canvas_type and not tags + if merge_groups: + from api.db.services.compilation_template_group_service import CompilationTemplateGroupService + + # Fetch every matching agent (page_number=0 disables SQL pagination) so + # the two sources can be globally ordered before we page in Python. + agents, _ = UserCanvasService.get_by_tenant_ids( + effective_owner_ids, + tenant_id, + 0, + 0, + order_by, + desc, + keywords, + None, + tags, + canvas_type, + ) + # Groups are owner-only (no team sharing), so they're scoped to the + # caller. Keyword filters the group name; scope is left unfiltered. + try: + groups = CompilationTemplateGroupService.list_saved(tenant_id, keywords, "", order_by, desc) + except Exception: + logging.exception("list_agents: compilation template group merge failed for tenant=%s", tenant_id) + groups = [] + + items: list[dict] = [] + for agent in agents: + agent["type"] = "agent" + items.append(agent) + for group in groups: + group["type"] = _COMPILATION_TEMPLATE_GROUP_CATEGORY + group["title"] = group["name"] + items.append(group) + + # Interleave by update_time (the requested merge key); items missing the + # field sort as oldest. + items.sort(key=lambda item: item.get("update_time") or 0, reverse=desc) + + total = len(items) + if page_number and items_per_page: + start = (page_number - 1) * items_per_page + items = items[start : start + items_per_page] + + return get_json_result(data={"canvas": items, "total": total}) + canvas, total = UserCanvasService.get_by_tenant_ids( effective_owner_ids, tenant_id, diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py index 7c8764742e..7027796596 100644 --- a/api/apps/restful_apis/dataset_api.py +++ b/api/apps/restful_apis/dataset_api.py @@ -747,6 +747,47 @@ async def get_dataset_structure(tenant_id, dataset_id): return get_error_data_result(message="Internal server error") +@manager.route("/datasets//artifacts_structure", methods=["DELETE"]) # noqa: F821 +@login_required +@add_tenant_id_to_kwargs +def delete_dataset_structure(tenant_id, dataset_id): + """Delete the dataset-scope (KB-wide) structure graph for one kind. + + DELETE /api/v1/datasets//artifacts_structure?kind= + Optional query param: wipe=false cancels the task without deleting stored rows. + """ + kind = request.args.get("kind", "") + if isinstance(kind, str): + kind = kind.strip() + if not kind: + return get_error_data_result( + message="`kind` is required (one of: graph, mindmap, timeline, session_essence, session_graph).", + code=RetCode.ARGUMENT_ERROR, + ) + if dataset_api_service._resolve_dataset_structure_kind(kind) is None: + return get_error_data_result( + message=f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph.", + code=RetCode.ARGUMENT_ERROR, + ) + wipe_arg = (request.args.get("wipe", "true") or "true").strip().lower() + wipe = wipe_arg not in ("false", "0", "no", "off") + try: + success, result = dataset_api_service.delete_dataset_structure( + dataset_id, + tenant_id, + kind, + wipe=wipe, + ) + if success: + return get_result(data=result) + if result == "No authorization.": + return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR) + return get_error_data_result(message=result) + except Exception as e: + logging.exception(e) + return get_error_data_result(message="Internal server error") + + @manager.route("/datasets//artifacts/alteration", methods=["GET"]) # noqa: F821 @login_required @add_tenant_id_to_kwargs diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py index caf002f94d..76be8088cf 100644 --- a/api/apps/services/dataset_api_service.py +++ b/api/apps/services/dataset_api_service.py @@ -1721,6 +1721,13 @@ _DATASET_STRUCTURE_KIND_ALIASES = { "session_essence": "session_essence", "session_graph": "session_graph", } +_DATASET_STRUCTURE_KIND_TO_INDEX_TYPE = { + "knowledge_graph": "structure_graph", + "mind_map": "structure_mindmap", + "timeline": "timeline", + "session_essence": "session_essence", + "session_graph": "session_graph", +} def _resolve_dataset_structure_kind(kind) -> str | None: @@ -1730,6 +1737,15 @@ def _resolve_dataset_structure_kind(kind) -> str | None: return _DATASET_STRUCTURE_KIND_ALIASES.get(kind.strip().lower().replace("-", "_")) +def delete_dataset_structure(dataset_id: str, tenant_id: str, kind: str, wipe: bool = True): + """Delete the merged KB-wide structure rows for one artifacts_structure kind.""" + resolved_kind = _resolve_dataset_structure_kind(kind) + if not resolved_kind: + return False, f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph." + index_type = _DATASET_STRUCTURE_KIND_TO_INDEX_TYPE[resolved_kind] + return delete_index(dataset_id, tenant_id, index_type, wipe=wipe) + + async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywords: str = ""): """Load the dataset-scope (KB-wide) structure graph for one ``kind``. diff --git a/rag/advanced_rag/harness/tools/__init__.py b/rag/advanced_rag/harness/tools/__init__.py index b8d5b1fa19..bddd239855 100644 --- a/rag/advanced_rag/harness/tools/__init__.py +++ b/rag/advanced_rag/harness/tools/__init__.py @@ -19,12 +19,14 @@ from rag.advanced_rag.harness.tools.navigation import catalog_navigate, dataset_ # catalog_navigate covers both the tree/TOC outline and the page index. register_tool( "catalog_navigate", - _navigate_schema("catalog_navigate", "Answer from the document's compiled catalog (table of contents / page index)"), + _navigate_schema("catalog_navigate", "Get question-related chunks from the document's catalog (table of contents / page index)"), catalog_navigate, requires_compilation=True, compilation_type=("toc", "page_index"), ) -register_tool("mindmap_navigate", _navigate_schema("mindmap_navigate", "Navigate by mindmap"), mindmap_navigate, requires_compilation=True, compilation_type="mindmap") +register_tool( + "mindmap_navigate", _navigate_schema("mindmap_navigate", "Get question-related chunks from the document's mindmap"), mindmap_navigate, requires_compilation=True, compilation_type="mindmap" +) register_tool( "dataset_navigate", _navigate_schema("dataset_navigate", "Find the most relevant documents via the dataset map, then search within them"), diff --git a/rag/nlp/search.py b/rag/nlp/search.py index 6f69f83e10..4c46b5f5fe 100644 --- a/rag/nlp/search.py +++ b/rag/nlp/search.py @@ -131,7 +131,7 @@ class Dealer: condition["must_not"] = req["must_not"] return condition - async def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=None, highlight: bool | list | None = None, rank_feature: dict | None = None): + async def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=None, highlight: bool | list | None = None, rank_feature: dict | None = None, min_match: bool = True): if highlight is None: highlight = False @@ -189,7 +189,7 @@ class Dealer: highlightFields = [] elif isinstance(highlight, list): highlightFields = highlight - matchText, keywords = self.qryr.question(qst, min_match=0.3) + matchText, keywords = self.qryr.question(qst, min_match=(0.3 if min_match else 0)) if emb_mdl is None: matchExprs = [matchText] res = await thread_pool_exec(self.dataStore.search, src, highlightFields, filters, matchExprs, orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature) @@ -220,7 +220,7 @@ class Dealer: res = await thread_pool_exec(self.dataStore.search, src, [], filters, [], orderBy, offset, limit, idx_names, kb_ids) total = self.dataStore.get_total(res) else: - matchText, _ = self.qryr.question(qst, min_match=0.1) + matchText, _ = self.qryr.question(qst, min_match=(0.1 if min_match else 0)) matchDense.extra_options["similarity"] = 0.17 res = await thread_pool_exec( self.dataStore.search, src, highlightFields, filters, [matchText, matchDense, fusionExpr], orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature @@ -593,7 +593,8 @@ class Dealer: tenant_ids = tenant_ids.split(",") idx_names = [index_name(tid) for tid in tenant_ids] - sres = await self.search(req, idx_names, kb_ids, embd_mdl, highlight, rank_feature=rank_feature) + min_match = vector_similarity_weight < 0.8 + sres = await self.search(req, idx_names, kb_ids, embd_mdl, highlight, rank_feature=rank_feature, min_match=min_match) # Temporary retrieval-side guard: prune chunks whose parent document no # longer exists before reranking and returning results. sres = await self._prune_deleted_chunks(sres) diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py index 99239c6de5..7dd6366880 100644 --- a/rag/utils/es_conn.py +++ b/rag/utils/es_conn.py @@ -30,6 +30,7 @@ from common.constants import PAGERANK_FLD, TAG_FLD ATTEMPT_TIME = 2 MAX_RESULT_WINDOW = 10000 SEARCH_AFTER_BATCH_SIZE = 1000 +KNN_QUERY_STRING_FILTER_WEIGHT_THRESHOLD = 0.8 # Single-document atomic pagerank_fea adjust (chunk feedback). Clamps using params.min_w / max_w; # removes field at zero for rank_feature compatibility. @@ -58,6 +59,43 @@ if (nw <= 0.0) { """ +def _is_query_string_clause(clause: dict) -> bool: + return isinstance(clause, dict) and "query_string" in clause + + +def _remove_query_string_must_clauses(must_clauses): + if isinstance(must_clauses, list): + return [copy.deepcopy(clause) for clause in must_clauses if not _is_query_string_clause(clause)] + if _is_query_string_clause(must_clauses): + return [] + return [copy.deepcopy(must_clauses)] if must_clauses else [] + + +def _build_knn_filter_query(bool_query, vector_similarity_weight: float): + if bool_query is None: + return None + + query = bool_query.to_dict() + if vector_similarity_weight <= KNN_QUERY_STRING_FILTER_WEIGHT_THRESHOLD: + return query + + query = copy.deepcopy(query) + bool_part = query.get("bool") + if not isinstance(bool_part, dict): + return query + + if "must" in bool_part: + must_clauses = _remove_query_string_must_clauses(bool_part["must"]) + if must_clauses: + bool_part["must"] = must_clauses + else: + bool_part.pop("must", None) + + if not any(bool_part.get(key) for key in ("must", "filter", "must_not", "should")): + return None + return query + + @singleton class ESConnection(ESConnectionBase): """ @@ -217,7 +255,7 @@ class ESConnection(ESConnectionBase): m.topn, m.topn * 2, query_vector=list(m.embedding_data), - filter=bool_query.to_dict(), + filter=bool_query.to_dict(), # filter=_build_knn_filter_query(bool_query, vector_similarity_weight), similarity=similarity, ) diff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.py index 2d257fac96..f69324e07f 100644 --- a/rag/utils/infinity_conn.py +++ b/rag/utils/infinity_conn.py @@ -24,8 +24,11 @@ import pandas as pd from common.constants import PAGERANK_FLD, TAG_FLD from common.doc_store.doc_store_base import MatchExpr, MatchTextExpr, MatchDenseExpr, FusionExpr, OrderByExpr from common.doc_store.infinity_conn_base import InfinityConnectionBase +from common.float_utils import get_float +DENSE_FILTER_FULLTEXT_WEIGHT_THRESHOLD = 0.8 +DEFAULT_VECTOR_SIMILARITY_WEIGHT = 0.5 _JSON_LIST_FIELDS = frozenset( ( "source_chunk_ids", @@ -40,6 +43,27 @@ _JSON_LIST_FIELDS = frozenset( ) +def _vector_similarity_weight(match_expressions: list[MatchExpr]) -> float: + vector_similarity_weight = DEFAULT_VECTOR_SIMILARITY_WEIGHT + for matchExpr in match_expressions: + if not isinstance(matchExpr, FusionExpr) or matchExpr.method != "weighted_sum": + continue + fusion_params = matchExpr.fusion_params or {} + weights = fusion_params.get("weights") + if not weights: + continue + weight_parts = str(weights).split(",") + if len(weight_parts) > 1: + vector_similarity_weight = get_float(weight_parts[1]) + return vector_similarity_weight + + +def _build_dense_filter(filter_cond: str | None, filter_fulltext: str | None, vector_similarity_weight: float) -> str: + if vector_similarity_weight > DENSE_FILTER_FULLTEXT_WEIGHT_THRESHOLD: + return filter_cond or "" + return filter_fulltext or filter_cond or "" + + @singleton class InfinityConnection(InfinityConnectionBase): """ @@ -191,6 +215,7 @@ class InfinityConnection(InfinityConnectionBase): self.logger.error(f"No valid tables found for indexNames {index_names} and knowledgebaseIds {knowledgebase_ids}") return pd.DataFrame(), 0 + # vector_similarity_weight = _vector_similarity_weight(match_expressions) for matchExpr in match_expressions: if isinstance(matchExpr, MatchTextExpr): if filter_cond and "filter" not in matchExpr.extra_options: @@ -223,6 +248,9 @@ class InfinityConnection(InfinityConnectionBase): elif isinstance(matchExpr, MatchDenseExpr): if filter_fulltext and "filter" not in matchExpr.extra_options: matchExpr.extra_options.update({"filter": filter_fulltext}) + # dense_filter = _build_dense_filter(filter_cond, filter_fulltext, vector_similarity_weight) + # if dense_filter and "filter" not in matchExpr.extra_options: + # matchExpr.extra_options.update({"filter": dense_filter}) for k, v in matchExpr.extra_options.items(): if not isinstance(v, str): matchExpr.extra_options[k] = str(v)