From 8379165c12aef7da774e2748624377e0efa7cd49 Mon Sep 17 00:00:00 2001 From: buua436 Date: Thu, 6 Aug 2026 16:49:50 +0800 Subject: [PATCH] fix: record generated wiki page versions (#17931) --- api/apps/services/dataset_api_service.py | 32 +++++++- api/db/services/file_commit_service.py | 43 ++++++++++ .../knowlege_compile/wiki_incremental.py | 34 +++++++- .../dataset_wiki_generator.py | 82 ++++++++++++++++--- rag/utils/es_conn.py | 4 + web/src/constants/knowledge.ts | 2 +- web/src/hooks/use-knowledge-request.ts | 18 ++-- 7 files changed, 190 insertions(+), 25 deletions(-) diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py index dc9c6f939a..9c9f9961f2 100644 --- a/api/apps/services/dataset_api_service.py +++ b/api/apps/services/dataset_api_service.py @@ -3174,7 +3174,7 @@ async def update_wiki_page( content_before = "" try: res = settings.docStoreConn.search( - select_fields=["id", "content_with_weight"], + select_fields=["id", "md_with_weight", "content_with_weight"], highlight_fields=[], condition={ "compile_kwd": [WIKI_PAGE_COMPILE_KWD], @@ -3190,11 +3190,11 @@ async def update_wiki_page( ) field_map = settings.docStoreConn.get_fields( res, - ["id", "content_with_weight"], + ["id", "md_with_weight", "content_with_weight"], ) if field_map: row_id, row = next(iter(field_map.items())) - content_before = row.get("content_with_weight") or "" + content_before = row.get("md_with_weight") or row.get("content_with_weight") or "" except Exception: logging.exception( "update_wiki_page: lookup failed for kb=%s slug=%s", @@ -3215,6 +3215,7 @@ async def update_wiki_page( ok = settings.docStoreConn.update( {"id": row_id}, { + "md_with_weight": rendered, "content_with_weight": rendered, "summary_with_weight": summary, "outlinks_kwd": list(outlinks), @@ -3233,6 +3234,17 @@ async def update_wiki_page( if not ok: return True, None + refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None) + if callable(refresh_idx): + try: + await thread_pool_exec(refresh_idx, index_nm) + except Exception: + logging.exception( + "update_wiki_page: index refresh failed for kb=%s slug=%s", + dataset_id, + full_slug, + ) + # Record a file_commit row on every real change. ``record_page_edit`` # returns None for empty-diff saves, which we silently swallow. try: @@ -3824,4 +3836,18 @@ async def clear_wiki(dataset_id: str, tenant_id: str): ) deleted[kwd] = False + from api.db.services.file_commit_service import FileCommitService + + if all(result is not False for result in deleted.values()): + try: + deleted["file_commit_history"] = FileCommitService.delete_all_page_history(dataset_id) + except Exception: + logging.exception( + "clear_wiki: failed to delete page version history for kb=%s", + dataset_id, + ) + deleted["file_commit_history"] = False + else: + deleted["file_commit_history"] = False + return True, {"deleted": deleted} diff --git a/api/db/services/file_commit_service.py b/api/db/services/file_commit_service.py index 86117596f0..30a6df25ac 100644 --- a/api/db/services/file_commit_service.py +++ b/api/db/services/file_commit_service.py @@ -801,6 +801,49 @@ class FileCommitService(CommonService): return commit_id + @classmethod + @DB.connection_context() + def delete_page_history(cls, kb_id: str, page_type: str, slug: str) -> int: + """Delete all stored versions for one Wiki page. + + Wiki versions are represented by a ``FileCommit`` plus its single + ``FileCommitItem``. They are not workspace commits, so removing the + page must remove both rows instead of leaving an orphaned history + that can reappear when the same slug is generated again. + """ + file_id = _wiki_file_id(kb_id, slug) + commit_ids = [ + row.commit_id + for row in FileCommitItem.select(FileCommitItem.commit_id).where((FileCommitItem.file_id == file_id) & (FileCommitItem.slug_kwd == slug) & (FileCommitItem.page_type_kwd == page_type)) + ] + if not commit_ids: + return 0 + + with DB.atomic(): + FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute() + deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute() + return deleted + + @classmethod + @DB.connection_context() + def delete_all_page_history(cls, kb_id: str) -> int: + """Delete all Wiki page versions belonging to a knowledge base.""" + commit_ids = [ + row.commit_id + for row in ( + FileCommitItem.select(FileCommitItem.commit_id) + .join(FileCommit, on=(FileCommit.id == FileCommitItem.commit_id)) + .where((FileCommit.folder_id == kb_id) & FileCommitItem.slug_kwd.is_null(False) & FileCommitItem.page_type_kwd.is_null(False)) + ) + ] + if not commit_ids: + return 0 + + with DB.atomic(): + FileCommitItem.delete().where(FileCommitItem.commit_id.in_(commit_ids)).execute() + deleted = FileCommit.delete().where((FileCommit.folder_id == kb_id) & FileCommit.id.in_(commit_ids)).execute() + return deleted + @classmethod @DB.connection_context() def list_page_commits( diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py index b9f5f727a9..b8e79ce875 100644 --- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py +++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py @@ -1567,12 +1567,19 @@ async def _wiki_refine_page( return existing_page if mode == "delete": - await thread_pool_exec( + deleted_count = await thread_pool_exec( settings.docStoreConn.delete, {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": [page_id]}, search.index_name(tenant_id), kb_id, ) + if not isinstance(deleted_count, int) or deleted_count <= 0: + logging.warning("wiki: page deletion did not remove page=%s", page_id) + return existing_page + from api.db.services.file_commit_service import FileCommitService + + commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}" + FileCommitService.delete_page_history(kb_id, page_type_kwd, commit_slug) return None # WeKnora-style verbatim evidence: load the ACTUAL source-chunk text for @@ -1732,6 +1739,31 @@ async def _wiki_refine_page( kb_id, ) + # Keep generated pages in the same version history as manual edits. The + # incremental compiler is the normal Wiki path, so without this record the + # first generated page (and subsequent generated revisions) are invisible + # to the commits API. + from api.db.services.file_commit_service import FileCommitService + + content_before = "" + if existing_page: + content_before = existing_page.get("md_with_weight") or existing_page.get("content_with_weight") or "" + commit_slug = page_id if page_id.startswith(f"{page_type_kwd}/") else f"{page_type_kwd}/{page_id}" + try: + FileCommitService.record_page_edit( + tenant_id=tenant_id, + kb_id=kb_id, + page_type=page_type_kwd, + slug=commit_slug, + content_before=content_before, + content_after=content, + title="Regenerated by artifact compilation", + comments=f"Auto-update via incremental wiki compilation (action={mode.upper()})", + user_id=None, + ) + except Exception: + logging.exception("wiki: generated page version record failed for page=%s", page_id) + return page diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py index 3cf929c606..d46960905a 100644 --- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py +++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py @@ -402,9 +402,11 @@ async def _wiki_delete_deleted_doc_state( deleted = set(deleted_doc_ids) derived_kwds = list(WIKI_DERIVED_COMPILE_KWDS) - select_fields = ["id", "source_doc_ids"] + select_fields = ["id", "source_doc_ids", "compile_kwd", "slug_kwd", "page_type_kwd"] to_delete: list[str] = [] to_shrink: list[tuple[str, list[str]]] = [] + page_history_to_delete: list[tuple[str, str, str]] = [] + failed_delete_row_ids: set[str] = set() offset = 0 page_size = 1000 while True: @@ -440,21 +442,44 @@ async def _wiki_delete_deleted_doc_state( to_shrink.append((row_id, remaining)) else: to_delete.append(row_id) + if row.get("compile_kwd") == WIKI_PAGE_COMPILE_KWD: + slug = row.get("slug_kwd") + if isinstance(slug, str) and slug: + page_history_to_delete.append((row_id, slug, row.get("page_type_kwd") or "concept")) if len(field_map) < page_size: break offset += page_size # Drop rows with no surviving owner (delete by id in batches). for i in range(0, len(to_delete), page_size): + batch_ids = to_delete[i : i + page_size] try: - await thread_pool_exec( + deleted_count = await thread_pool_exec( settings.docStoreConn.delete, - {"id": to_delete[i : i + page_size]}, + {"id": batch_ids}, index, kb_id, ) + if not isinstance(deleted_count, int) or deleted_count != len(batch_ids): + failed_delete_row_ids.update(batch_ids) except Exception: logging.exception("wiki: failed to drop orphaned derived rows in kb=%s", kb_id) + failed_delete_row_ids.update(batch_ids) + + if page_history_to_delete: + from api.db.services.file_commit_service import FileCommitService + + for row_id, slug, page_type in page_history_to_delete: + if row_id in failed_delete_row_ids: + continue + try: + FileCommitService.delete_page_history(kb_id, page_type, slug) + except Exception: + logging.exception( + "wiki: failed to delete version history for removed page=%s kb=%s", + slug, + kb_id, + ) # Shrink rows still owned by surviving docs to just those docs. for row_id, remaining in to_shrink: @@ -733,15 +758,17 @@ async def persist_wiki_pages( # Capture the prior rendered content for every slug we're about to # overwrite, so the per-page commit row downstream has a real diff - # baseline. Single batch read by slug_kwd IN [...] — one round-trip - # regardless of page count. Failures here degrade gracefully. + # baseline. Prefer md_with_weight because that is the canonical field + # consumed by the Wiki page API, with content_with_weight as fallback. + # Single batch read by slug_kwd IN [...] — one round-trip regardless of + # page count. Failures here degrade gracefully. target_slugs: list[str] = [(p.get("slug") or "").strip() for p in pages if isinstance(p.get("slug"), str) and p.get("slug")] prior_by_slug: dict[str, str] = {} if target_slugs: try: res = await thread_pool_exec( settings.docStoreConn.search, - ["id", "slug_kwd", "content_with_weight"], + ["id", "slug_kwd", "md_with_weight", "content_with_weight"], [], {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)}, [], @@ -753,11 +780,11 @@ async def persist_wiki_pages( ) field_map = settings.docStoreConn.get_fields( res, - ["id", "slug_kwd", "content_with_weight"], + ["id", "slug_kwd", "md_with_weight", "content_with_weight"], ) for row in (field_map or {}).values(): s = row.get("slug_kwd") - c = row.get("content_with_weight") + c = row.get("md_with_weight") or row.get("content_with_weight") if isinstance(s, str) and isinstance(c, str): prior_by_slug[s] = c except Exception: @@ -838,6 +865,7 @@ async def persist_wiki_pages( "related_kb_pages_kwd": list(page.get("related_kb_pages") or []), "source_chunk_ids": list(page.get("source_chunk_ids") or []), "source_doc_ids": list(page.get("source_doc_ids") or []), + "md_with_weight": content_md, "content_with_weight": content_md, # Summary kept verbatim alongside the rendered body so the # viewer can render it as a distinct (smaller) block above @@ -862,7 +890,7 @@ async def persist_wiki_pages( return try: - await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id) + insert_errors = await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id) except Exception: logging.exception( "wiki_persist: bulk insert failed for kb=%s (rows=%d)", @@ -871,6 +899,38 @@ async def persist_wiki_pages( ) return + if insert_errors: + logging.warning( + "wiki_persist: page insert returned %d error(s) for kb=%s", + len(insert_errors), + ) + + # Verify the rows that are actually visible after the bulk operation. Some + # backends can report partial bulk failures without raising, so recording a + # version for every input page would create history for pages that were not + # persisted. + try: + persisted_res = await thread_pool_exec( + settings.docStoreConn.search, + ["id", "slug_kwd"], + [], + {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "id": [row["id"] for row in rows]}, + [], + OrderByExpr(), + 0, + len(rows), + index, + [ctx.kb_id], + ) + persisted_fields = settings.docStoreConn.get_fields(persisted_res, ["id", "slug_kwd"]) + persisted_page_slugs = {row.get("slug_kwd") for row in (persisted_fields or {}).values() if isinstance(row.get("slug_kwd"), str)} + except Exception: + logging.exception( + "wiki_persist: failed to verify persisted pages for kb=%s; skipping history records", + kb_id_str, + ) + return + if topics_by_name: try: await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name, topic_doc_ids) @@ -886,9 +946,7 @@ async def persist_wiki_pages( # compile. for page in pages: slug = (page.get("slug") or "").strip() - if not slug: - continue - if not prior_by_slug.get(slug, ""): + if not slug or slug not in persisted_page_slugs: continue content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or "" action = (page.get("action") or "CREATE").upper() diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py index 4c8bfc24a6..b06a2c3e5c 100644 --- a/rag/utils/es_conn.py +++ b/rag/utils/es_conn.py @@ -102,6 +102,10 @@ class ESConnection(ESConnectionBase): CRUD operations """ + def refresh_idx(self, index_name: str) -> bool: + self.es.indices.refresh(index=index_name) + return True + def _es_search_once(self, index_names: list[str], query: dict, track_total_hits: bool): return self.es.search(index=index_names, body=query, timeout="600s", track_total_hits=track_total_hits) diff --git a/web/src/constants/knowledge.ts b/web/src/constants/knowledge.ts index 55e4fb1407..b97f5ef579 100644 --- a/web/src/constants/knowledge.ts +++ b/web/src/constants/knowledge.ts @@ -120,7 +120,7 @@ export enum ParseType { export enum ProcessingType { knowledgeGraph = 'Graph', raptor = 'RAPTOR', - artifact = 'Artifact', + artifact = 'wiki', skill = 'Skill', mindmap = 'Mindmap', timeline = 'Timeline', diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts index d33797d88f..739ec631a6 100644 --- a/web/src/hooks/use-knowledge-request.ts +++ b/web/src/hooks/use-knowledge-request.ts @@ -727,13 +727,15 @@ export const useUpdateArtifactPage = () => { ); if (data.code === 0) { message.success(i18n.t(`message.updated`)); - queryClient.invalidateQueries({ - queryKey: ArtifactKeys.detail( - knowledgeBaseId, - params.pageType, - params.slug, - ), - }); + const detailKey = ArtifactKeys.detail( + knowledgeBaseId, + params.pageType, + params.slug, + ); + if (data.data) { + queryClient.setQueryData(detailKey, data.data); + } + await queryClient.invalidateQueries({ queryKey: detailKey }); } return data; }, @@ -972,7 +974,7 @@ export const useRunArtifactIndex = (kind: string) => { if (isGoDatasetBackend()) { throw new Error(i18n.t('message.compileNotSupported')); } - const { data } = await runIndex(knowledgeBaseId, 'artifact'); + const { data } = await runIndex(knowledgeBaseId, 'wiki'); if (data?.code === 0) { message.success(i18n.t('message.operated')); queryClient.invalidateQueries({