mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-23 00:46:42 +08:00
feat(dropbox): support deleted-file sync (#14476)
### What problem does this PR solve? Partially addresses #14362 by adding deleted-file sync support for the Dropbox data source. Dropbox previously did not provide the slim current-file snapshot required by stale document reconciliation, and its sync runner returned only document batches. As a result, enabling deleted-file sync could not remove local documents that had been deleted from Dropbox. This PR: - Adds `retrieve_all_slim_docs_perm_sync()` to `DropboxConnector`. - Reuses Dropbox metadata traversal to collect current remote file IDs without downloading file contents. - Wires incremental Dropbox sync to return `(document_generator, file_list)` when `sync_deleted_files` is enabled. - Enables the deleted-file sync toggle for Dropbox in the data source settings UI. - Adds regression coverage for slim snapshots, nested folders, paginated listings, duplicate filenames, and full reindex behavior. Tests: - `uv run pytest test/unit_test/common/test_dropbox_connector.py -q` - `uv run pytest test/unit_test/rag/test_sync_data_source.py -q` - `uv run pytest test/unit_test/common/test_dropbox_connector.py test/unit_test/rag/test_sync_data_source.py -q` - `uv run ruff check common/data_source/dropbox_connector.py rag/svr/sync_data_source.py test/unit_test/common/test_dropbox_connector.py test/unit_test/rag/test_sync_data_source.py` - `./node_modules/.bin/eslint src/pages/user-setting/data-source/constant/index.tsx` ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -19,6 +19,7 @@ import os
|
||||
import sys
|
||||
import types
|
||||
import warnings
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -167,3 +168,95 @@ async def test_run_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
|
||||
{},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class _FakeDropboxConnector:
|
||||
instance = None
|
||||
|
||||
def __init__(self, batch_size):
|
||||
self.batch_size = batch_size
|
||||
self.credentials = None
|
||||
self.retrieve_all_slim_docs_perm_sync_called = False
|
||||
self.snapshot_called_before_poll = None
|
||||
self.poll_source_call = None
|
||||
self.load_from_state_called = False
|
||||
self.poll_source_called = False
|
||||
_FakeDropboxConnector.instance = self
|
||||
|
||||
def load_credentials(self, credentials):
|
||||
self.credentials = credentials
|
||||
|
||||
def retrieve_all_slim_docs_perm_sync(self, callback=None):
|
||||
del callback
|
||||
self.retrieve_all_slim_docs_perm_sync_called = True
|
||||
self.snapshot_called_before_poll = not self.poll_source_called
|
||||
yield [types.SimpleNamespace(id="dropbox:id-1")]
|
||||
yield [types.SimpleNamespace(id="dropbox:id-2")]
|
||||
|
||||
def poll_source(self, start, end):
|
||||
self.poll_source_called = True
|
||||
self.poll_source_call = (start, end)
|
||||
return iter((["poll-sync"],))
|
||||
|
||||
def load_from_state(self):
|
||||
self.load_from_state_called = True
|
||||
return iter((["full-sync"],))
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.p2
|
||||
async def test_dropbox_generate_returns_snapshot_when_sync_deleted_enabled(monkeypatch):
|
||||
monkeypatch.setattr(sync_data_source, "DropboxConnector", _FakeDropboxConnector)
|
||||
poll_start = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
task = {
|
||||
**_make_task(),
|
||||
"reindex": "0",
|
||||
"poll_range_start": poll_start,
|
||||
"skip_connection_log": True,
|
||||
}
|
||||
sync = sync_data_source.Dropbox(
|
||||
{
|
||||
"batch_size": 2,
|
||||
"sync_deleted_files": True,
|
||||
"credentials": {"dropbox_access_token": "token-1"},
|
||||
}
|
||||
)
|
||||
|
||||
document_generator, file_list = await sync._generate(task)
|
||||
connector = _FakeDropboxConnector.instance
|
||||
|
||||
assert list(document_generator) == [["poll-sync"]]
|
||||
assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
|
||||
assert connector.credentials == {"dropbox_access_token": "token-1"}
|
||||
assert connector.retrieve_all_slim_docs_perm_sync_called is True
|
||||
assert connector.snapshot_called_before_poll is True
|
||||
assert connector.poll_source_call[0] == poll_start.timestamp()
|
||||
assert connector.poll_source_call[1] >= poll_start.timestamp()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.p2
|
||||
async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):
|
||||
monkeypatch.setattr(sync_data_source, "DropboxConnector", _FakeDropboxConnector)
|
||||
task = {
|
||||
**_make_task(),
|
||||
"reindex": "1",
|
||||
"poll_range_start": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
"skip_connection_log": True,
|
||||
}
|
||||
sync = sync_data_source.Dropbox(
|
||||
{
|
||||
"batch_size": 2,
|
||||
"sync_deleted_files": True,
|
||||
"credentials": {"dropbox_access_token": "token-1"},
|
||||
}
|
||||
)
|
||||
|
||||
document_generator, file_list = await sync._generate(task)
|
||||
connector = _FakeDropboxConnector.instance
|
||||
|
||||
assert list(document_generator) == [["full-sync"]]
|
||||
assert file_list is None
|
||||
assert connector.load_from_state_called is True
|
||||
assert connector.retrieve_all_slim_docs_perm_sync_called is False
|
||||
assert connector.poll_source_called is False
|
||||
|
||||
Reference in New Issue
Block a user