feat(dropbox): support deleted-file sync (#14476)

### What problem does this PR solve?

Partially addresses #14362 by adding deleted-file sync support for the
Dropbox data source.

Dropbox previously did not provide the slim current-file snapshot
required by stale document reconciliation, and its sync runner returned
only document batches. As a result, enabling deleted-file sync could not
remove local documents that had been deleted from Dropbox.

This PR:
- Adds `retrieve_all_slim_docs_perm_sync()` to `DropboxConnector`.
- Reuses Dropbox metadata traversal to collect current remote file IDs
without downloading file contents.
- Wires incremental Dropbox sync to return `(document_generator,
file_list)` when `sync_deleted_files` is enabled.
- Enables the deleted-file sync toggle for Dropbox in the data source
settings UI.
- Adds regression coverage for slim snapshots, nested folders, paginated
listings, duplicate filenames, and full reindex behavior.

Tests:
- `uv run pytest test/unit_test/common/test_dropbox_connector.py -q`
- `uv run pytest test/unit_test/rag/test_sync_data_source.py -q`
- `uv run pytest test/unit_test/common/test_dropbox_connector.py
test/unit_test/rag/test_sync_data_source.py -q`
- `uv run ruff check common/data_source/dropbox_connector.py
rag/svr/sync_data_source.py
test/unit_test/common/test_dropbox_connector.py
test/unit_test/rag/test_sync_data_source.py`
- `./node_modules/.bin/eslint
src/pages/user-setting/data-source/constant/index.tsx`

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
bitloi
2026-04-29 08:05:11 -03:00
committed by GitHub
parent db1a73b255
commit 2bc8c6d35e
5 changed files with 321 additions and 50 deletions

View File

@@ -19,6 +19,7 @@ import os
import sys
import types
import warnings
from datetime import datetime, timezone
import pytest
@@ -167,3 +168,95 @@ async def test_run_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
{},
)
]
class _FakeDropboxConnector:
instance = None
def __init__(self, batch_size):
self.batch_size = batch_size
self.credentials = None
self.retrieve_all_slim_docs_perm_sync_called = False
self.snapshot_called_before_poll = None
self.poll_source_call = None
self.load_from_state_called = False
self.poll_source_called = False
_FakeDropboxConnector.instance = self
def load_credentials(self, credentials):
self.credentials = credentials
def retrieve_all_slim_docs_perm_sync(self, callback=None):
del callback
self.retrieve_all_slim_docs_perm_sync_called = True
self.snapshot_called_before_poll = not self.poll_source_called
yield [types.SimpleNamespace(id="dropbox:id-1")]
yield [types.SimpleNamespace(id="dropbox:id-2")]
def poll_source(self, start, end):
self.poll_source_called = True
self.poll_source_call = (start, end)
return iter((["poll-sync"],))
def load_from_state(self):
self.load_from_state_called = True
return iter((["full-sync"],))
@pytest.mark.anyio
@pytest.mark.p2
async def test_dropbox_generate_returns_snapshot_when_sync_deleted_enabled(monkeypatch):
monkeypatch.setattr(sync_data_source, "DropboxConnector", _FakeDropboxConnector)
poll_start = datetime(2026, 1, 1, tzinfo=timezone.utc)
task = {
**_make_task(),
"reindex": "0",
"poll_range_start": poll_start,
"skip_connection_log": True,
}
sync = sync_data_source.Dropbox(
{
"batch_size": 2,
"sync_deleted_files": True,
"credentials": {"dropbox_access_token": "token-1"},
}
)
document_generator, file_list = await sync._generate(task)
connector = _FakeDropboxConnector.instance
assert list(document_generator) == [["poll-sync"]]
assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
assert connector.credentials == {"dropbox_access_token": "token-1"}
assert connector.retrieve_all_slim_docs_perm_sync_called is True
assert connector.snapshot_called_before_poll is True
assert connector.poll_source_call[0] == poll_start.timestamp()
assert connector.poll_source_call[1] >= poll_start.timestamp()
@pytest.mark.anyio
@pytest.mark.p2
async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):
monkeypatch.setattr(sync_data_source, "DropboxConnector", _FakeDropboxConnector)
task = {
**_make_task(),
"reindex": "1",
"poll_range_start": datetime(2026, 1, 1, tzinfo=timezone.utc),
"skip_connection_log": True,
}
sync = sync_data_source.Dropbox(
{
"batch_size": 2,
"sync_deleted_files": True,
"credentials": {"dropbox_access_token": "token-1"},
}
)
document_generator, file_list = await sync._generate(task)
connector = _FakeDropboxConnector.instance
assert list(document_generator) == [["full-sync"]]
assert file_list is None
assert connector.load_from_state_called is True
assert connector.retrieve_all_slim_docs_perm_sync_called is False
assert connector.poll_source_called is False