From 38435255f95f7189e07b61fe6cbdb7ef02cd95c5 Mon Sep 17 00:00:00 2001 From: buua436 Date: Wed, 5 Aug 2026 19:40:27 +0800 Subject: [PATCH] fix: filter wiki artifacts by keywords (#17871) --- api/apps/restful_apis/dataset_api.py | 8 +- api/apps/services/dataset_api_service.py | 128 ++++++++++++++++++----- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py index fd2429d520..54e68a3b35 100644 --- a/api/apps/restful_apis/dataset_api.py +++ b/api/apps/restful_apis/dataset_api.py @@ -602,7 +602,7 @@ async def has_any_wiki(tenant_id, dataset_id): async def list_wiki_pages(tenant_id, dataset_id): """List artifact pages for the dataset Artifact tab. - GET /api/v1/datasets//artifacts?page=1&page_size=200&page_type=entity&topic=topic + GET /api/v1/datasets//artifacts?page=1&page_size=200&page_type=entity&topic=topic&keywords=query Success: {"code": 0, "data": {"total": int, "items": [{slug, title, page_type}]}} """ try: @@ -612,6 +612,7 @@ async def list_wiki_pages(tenant_id, dataset_id): return get_error_argument_result(str(e)) page_type = (request.args.get("page_type") or "").strip() or None topic = (request.args.get("topic") or "").strip() or None + keywords = (request.args.get("keywords") or "").strip() try: success, result = await dataset_api_service.list_wiki_pages( @@ -621,6 +622,7 @@ async def list_wiki_pages(tenant_id, dataset_id): page_size=page_size, page_type=page_type, topic=topic, + keywords=keywords, ) if success: return get_result(data=result) @@ -636,7 +638,7 @@ async def list_wiki_pages(tenant_id, dataset_id): async def list_wiki_topics(tenant_id, dataset_id): """List wiki topics for the dataset Artifact tab. - GET /api/v1/datasets//artifacts/topics?page=1&page_size=200 + GET /api/v1/datasets//artifacts/topics?page=1&page_size=200&keywords=query Success: {"code": 0, "data": {"total": int, "items": [{topic, title, slug}]}} """ try: @@ -644,6 +646,7 @@ async def list_wiki_topics(tenant_id, dataset_id): page_size = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE)) except ValueError as e: return get_error_argument_result(str(e)) + keywords = (request.args.get("keywords") or "").strip() try: success, result = await dataset_api_service.list_wiki_topics( @@ -651,6 +654,7 @@ async def list_wiki_topics(tenant_id, dataset_id): tenant_id, page=page, page_size=page_size, + keywords=keywords, ) if success: return get_result(data=result) diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py index 966bb4d033..dc9c6f939a 100644 --- a/api/apps/services/dataset_api_service.py +++ b/api/apps/services/dataset_api_service.py @@ -2301,6 +2301,7 @@ async def list_wiki_pages( page_size: int = 200, page_type: str | None = None, topic: str | None = None, + keywords: str = "", ): """List artifact pages for the left-hand 2-column list. @@ -2324,6 +2325,7 @@ async def list_wiki_pages( offset = (page - 1) * page_size page_type = page_type.strip() if isinstance(page_type, str) else page_type topic = topic.strip() if isinstance(topic, str) else topic + keywords = (keywords or "").strip().casefold() condition: dict = {"compile_kwd": [WIKI_PAGE_COMPILE_KWD]} if page_type: @@ -2350,39 +2352,72 @@ async def list_wiki_pages( "outlinks_int", "summary_with_weight", ] + + def _to_item(row): + slug = _scalar(row.get("slug_kwd")) + if not slug: + return None + return { + "slug": slug, + "title": _scalar(row.get("title_kwd")) or slug, + "page_type": _scalar(row.get("page_type_kwd")) or "concept", + "topic": _scalar(row.get("topic_kwd")) or "", + "summary": row.get("summary_with_weight") or "", + } + try: - res = settings.docStoreConn.search( - select_fields=select_fields, - highlight_fields=[], - condition=condition, - match_expressions=[], - order_by=order_by, - offset=offset, - limit=page_size, - index_names=index_nm, - knowledgebase_ids=[dataset_id], - ) - field_map = settings.docStoreConn.get_fields(res, select_fields) + if not keywords: + res = settings.docStoreConn.search( + select_fields=select_fields, + highlight_fields=[], + condition=condition, + match_expressions=[], + order_by=order_by, + offset=offset, + limit=page_size, + index_names=index_nm, + knowledgebase_ids=[dataset_id], + ) + field_map = settings.docStoreConn.get_fields(res, select_fields) + items = [item for row in (field_map or {}).values() if (item := _to_item(row)) is not None] + total = settings.docStoreConn.get_total(res) + else: + # Wiki list search is intentionally metadata-based. These fields + # are available on existing rows and behave consistently across + # every document-store backend, unlike a full-text query over + # backend-specific token fields. + matched_items = [] + batch_size = 1000 + search_offset = 0 + while True: + res = settings.docStoreConn.search( + select_fields=select_fields, + highlight_fields=[], + condition=condition, + match_expressions=[], + order_by=order_by, + offset=search_offset, + limit=batch_size, + index_names=index_nm, + knowledgebase_ids=[dataset_id], + ) + field_map = settings.docStoreConn.get_fields(res, select_fields) + rows = list((field_map or {}).values()) + if not rows: + break + for row in rows: + item = _to_item(row) + if item is not None and any(keywords in str(item[field]).casefold() for field in ("title", "slug", "summary")): + matched_items.append(item) + if len(rows) < batch_size: + break + search_offset += batch_size + total = len(matched_items) + items = matched_items[offset : offset + page_size] except Exception: logging.exception("list_wiki_pages: docStore search failed for kb=%s", dataset_id) return True, {"total": 0, "items": []} - total = settings.docStoreConn.get_total(res) - items = [] - for row in (field_map or {}).values(): - slug = _scalar(row.get("slug_kwd")) - if not slug: - continue - items.append( - { - "slug": slug, - "title": _scalar(row.get("title_kwd")) or slug, - "page_type": _scalar(row.get("page_type_kwd")) or "concept", - "topic": _scalar(row.get("topic_kwd")) or "", - "summary": row.get("summary_with_weight") or "", - } - ) - return True, {"total": int(total or 0), "items": items} @@ -2391,6 +2426,7 @@ async def list_wiki_topics( tenant_id: str, page: int = 1, page_size: int = 200, + keywords: str = "", ): """List wiki topics for the dataset Artifact tab.""" if not KnowledgebaseService.accessible(dataset_id, tenant_id): @@ -2407,6 +2443,7 @@ async def list_wiki_topics( page = max(1, int(page or 1)) page_size = max(1, min(int(page_size or 200), 1000)) offset = (page - 1) * page_size + keywords = (keywords or "").strip().casefold() try: agg_res = settings.docStoreConn.search( @@ -2477,6 +2514,41 @@ async def list_wiki_topics( key=lambda x: (-x["page_count"], x["title"].lower()), ) + if keywords: + matching_topics = {item["topic"] for item in ranked if any(keywords in str(item[field]).casefold() for field in ("topic", "title", "slug"))} + + child_fields = ["topic_kwd", "title_kwd", "slug_kwd", "summary_with_weight"] + batch_size = 1000 + child_offset = 0 + try: + while True: + child_res = settings.docStoreConn.search( + select_fields=child_fields, + highlight_fields=[], + condition={"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "page_type_kwd": ["concept", "entity"]}, + match_expressions=[], + order_by=OrderByExpr(), + offset=child_offset, + limit=batch_size, + index_names=index_nm, + knowledgebase_ids=[dataset_id], + ) + child_map = settings.docStoreConn.get_fields(child_res, child_fields) + child_rows = list((child_map or {}).values()) + if not child_rows: + break + for row in child_rows: + topic_name = _scalar(row.get("topic_kwd")) + if topic_name and any(keywords in str(_scalar(row.get(field)) or "").casefold() for field in ("title_kwd", "slug_kwd", "summary_with_weight")): + matching_topics.add(topic_name) + if len(child_rows) < batch_size: + break + child_offset += batch_size + except Exception: + logging.exception("list_wiki_topics: child-page keyword lookup failed for kb=%s", dataset_id) + + ranked = [item for item in ranked if item["topic"] in matching_topics] + total = len(ranked) items = ranked[offset : offset + page_size] return True, {"total": total, "items": items}