From 3b7a6eaa6cbcf4cfe7ba905eae3e4ba464e9a731 Mon Sep 17 00:00:00 2001 From: Magicbook1108 Date: Wed, 29 Apr 2026 11:29:17 +0800 Subject: [PATCH] Feat: sync deleted files in Bitbucket (#14450) ### What problem does this PR solve? Feat: sync deleted files in Bitbucket ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- common/data_source/bitbucket/connector.py | 5 +---- rag/svr/sync_data_source.py | 8 +++++++- web/src/pages/user-setting/data-source/constant/index.tsx | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/common/data_source/bitbucket/connector.py b/common/data_source/bitbucket/connector.py index 4b0240fa5f..0557d2a503 100644 --- a/common/data_source/bitbucket/connector.py +++ b/common/data_source/bitbucket/connector.py @@ -355,10 +355,7 @@ if __name__ == "__main__": start_time = datetime.fromtimestamp(0, tz=timezone.utc) end_time = datetime.now(timezone.utc) - for doc_batch in bitbucket.retrieve_all_slim_docs_perm_sync( - start=start_time.timestamp(), - end=end_time.timestamp(), - ): + for doc_batch in bitbucket.retrieve_all_slim_docs_perm_sync(): for doc in doc_batch: print(doc) diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py index 2c6d72cc94..ac0d4d0cb7 100644 --- a/rag/svr/sync_data_source.py +++ b/rag/svr/sync_data_source.py @@ -1334,12 +1334,17 @@ class Bitbucket(SyncBase): "bitbucket_api_token": self.conf["credentials"].get("bitbucket_api_token"), } ) + file_list = None if task["reindex"] == "1" or not task["poll_range_start"]: start_time = datetime.fromtimestamp(0, tz=timezone.utc) _begin_info = "totally" else: start_time = task.get("poll_range_start") + if self.conf.get("sync_deleted_files"): + file_list = [] + for slim_batch in self.connector.retrieve_all_slim_docs_perm_sync(): + file_list.extend(slim_batch) _begin_info = f"from {start_time}" end_time = datetime.now(timezone.utc) @@ -1371,7 +1376,8 @@ class Bitbucket(SyncBase): yield batch self.log_connection("Bitbucket", f"workspace({self.conf.get('workspace')})", task) - + if file_list is not None: + return wrapper(), file_list return wrapper() diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx index 2bb4d267f7..774b5c3f91 100644 --- a/web/src/pages/user-setting/data-source/constant/index.tsx +++ b/web/src/pages/user-setting/data-source/constant/index.tsx @@ -58,7 +58,7 @@ export const DataSourceFeatureVisibilityMap = { [DataSourceKey.GITHUB]: { syncDeletedFiles: true, }, -[DataSourceKey.GOOGLE_DRIVE]: { + [DataSourceKey.GOOGLE_DRIVE]: { syncDeletedFiles: true, }, [DataSourceKey.CONFLUENCE]: { @@ -85,6 +85,9 @@ export const DataSourceFeatureVisibilityMap = { [DataSourceKey.JIRA]: { syncDeletedFiles: true, }, + [DataSourceKey.BITBUCKET]: { + syncDeletedFiles: true, + }, [DataSourceKey.AIRTABLE]: { syncDeletedFiles: true, },