From 653b00b94c9bd5062f133539421f0fb5bf5f09b7 Mon Sep 17 00:00:00 2001 From: Octopus Date: Sat, 9 May 2026 10:33:54 +0800 Subject: [PATCH] fix(sync): scope document IDs per connector to prevent cross-KB collisions (#14378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #14360 ## Problem When the same blob storage bucket is connected to multiple knowledge bases (each through a different data source connector), the sync pipeline hashes only the blob path (`bucket_type:bucket_name:object_key`) to derive the document ID. Every connector pointing at the same bucket therefore produces **identical IDs** for the same object. The collision guard in `FileService.upload_document` then fires for the second knowledge base: ``` Existing document id collision with another knowledge base; skipping update. ``` This makes it impossible to index the same bucket into more than one KB simultaneously. ## Solution Include `connector_id` in the hash input so that each connector produces a distinct document ID even when the underlying blob path is identical: ```python # Before "id": hash128(doc.id), # After "id": hash128(f"{task['connector_id']}:{doc.id}"), ``` Because each KB connection uses its own connector (with a unique `connector_id`), documents are now namespaced per connector and no collision occurs. **Note:** This is a breaking change for existing synced data sources. After upgrading, a re-sync will create new documents with the updated ID format. Old documents (indexed under the previous format) will remain in the database but can be manually deleted or cleaned up via a re-sync with reindex enabled. ## Testing - Verified that the one-line change produces unique IDs for two connectors pointing at the same S3 path. - Existing unit test `test_upload_document_skips_cross_kb_document_id_collision` continues to pass — the collision guard in `FileService` is still valid for genuinely colliding IDs from other sources. --------- Co-authored-by: octo-patch --- api/db/services/connector_service.py | 2 +- rag/svr/sync_data_source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py index 40f0b7b5c..9f7b0e6de 100644 --- a/api/db/services/connector_service.py +++ b/api/db/services/connector_service.py @@ -100,7 +100,7 @@ class ConnectorService(CommonService): return 0, [] source_type = f"{conn.source}/{conn.id}" - retain_doc_ids = {hash128(file.id) for file in file_list} + retain_doc_ids = {hash128(f"{connector_id}:{file.id}") for file in file_list} existing_docs = DocumentService.list_doc_headers_by_kb_and_source_type( kb_id, source_type, diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py index b5801905d..9a60701e7 100644 --- a/rag/svr/sync_data_source.py +++ b/rag/svr/sync_data_source.py @@ -202,7 +202,7 @@ class SyncBase: docs = [] for doc in document_batch: d = { - "id": hash128(doc.id), + "id": hash128(f"{task['connector_id']}:{doc.id}"), "connector_id": task["connector_id"], "source": self.SOURCE_NAME, "semantic_identifier": doc.semantic_identifier,