Feat: full optimization on connector dashboard (#14979)

### What problem does this PR solve? This PR improves the connector dashboard task management experience and adds better visibility into connector execution logs. ### Overview: #### Before <img width="700" alt="image" src="https://github.com/user-attachments/assets/e4a8ed6f-2e18-4f0f-8528-41a514550052" /> #### Now: <img width="700" alt="Screenshot from 2026-05-18 16-31-30" src="https://github.com/user-attachments/assets/d4ca193b-847a-49ae-9e4f-5fbca60ea627" /> ### 1. Add a new logging page to the connector dashboard A new logging page has been added so users can view connector task execution logs directly from the connector dashboard. ### 2. Merge the Resume button into Confirm The separate **Resume** button has been removed. The **Confirm** button now represents different actions depending on the current task state: - **Save**: Save form changes and reschedule tasks. - **Stop**: Cancel currently scheduled or running tasks. - **Resume**: Create new scheduled tasks after the previous tasks have been stopped. - **Start**: Start tasks when no task has been started yet. ### 3. Separate syncing and pruning tasks Connector tasks are now separated into **syncing** and **pruning**. Pruning is controlled by the **Sync deleted files** option: - When **Sync deleted files** is disabled, only syncing tasks are shown. - When **Sync deleted files** is enabled, both syncing and pruning tasks are shown. **Now: Sync deleted files disabled** <img width="700" alt="Sync deleted files disabled" src="https://github.com/user-attachments/assets/dbd9232e-614a-407f-a0b1-c109e5fa567d" /> **Now: Sync deleted files enabled** <img width="700" alt="Sync deleted files enabled" src="https://github.com/user-attachments/assets/1f527f48-ccb3-4ee8-97ca-086891489296" /> ### 4. Update logs in backend <img width="700" alt="image" src="https://github.com/user-attachments/assets/10a95a3f-98c1-4e67-8afa-ddf6cda5b0b2" /> ### 5. Remove connector resume API - Removed: `POST /v1/connectors/<connector_id>/resume` - Replaced by: `PATCH /v1/connectors/<connector_id>` ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2026-08-15 05:04:27 +08:00 · 2026-05-19 10:07:11 +08:00
parent 41a9fc0030
commit b69a6a5d80
15 changed files with 706 additions and 557 deletions
--- a/test/testcases/restful_api/test_connector_routes_unit.py
+++ b/test/testcases/restful_api/test_connector_routes_unit.py
@@ -205,7 +205,7 @@ def _load_connector_app(monkeypatch):
            return True

        @staticmethod
-        def resume(*_args, **_kwargs):
+        def cancel_tasks(*_args, **_kwargs):
            return True

        @staticmethod
@@ -252,7 +252,11 @@ def _load_connector_app(monkeypatch):
        PERMISSION_ERROR=403,
        AUTHENTICATION_ERROR=109,
    )
-    constants_mod.TaskStatus = SimpleNamespace(SCHEDULE="schedule", CANCEL="cancel")
+    constants_mod.TaskStatus = SimpleNamespace(
+        UNSTART="unstart",
+        SCHEDULE="schedule",
+        CANCEL="cancel",
+    )
    monkeypatch.setitem(sys.modules, "common.constants", constants_mod)

    config_mod = ModuleType("common.data_source.config")
@@ -349,7 +353,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    records = {"conn-1": _FakeConnectorRecord({"id": "conn-1", "source": "drive"})}
    update_calls = []
    save_calls = []
-    resume_calls = []
+    cancel_calls = []
    delete_calls = []

    monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda cid, payload: update_calls.append((cid, payload)))
@@ -362,7 +366,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda cid: (True, records[cid]))
    monkeypatch.setattr(module.ConnectorService, "list", lambda tenant_id: [{"id": "listed", "tenant": tenant_id}])
    monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda cid, page, page_size: ([{"id": "log-1"}], 9))
-    monkeypatch.setattr(module.ConnectorService, "resume", lambda cid, status: resume_calls.append((cid, status)))
+    monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda cid: cancel_calls.append(cid))
    monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda cid: delete_calls.append(cid))
    monkeypatch.setattr(module, "get_uuid", lambda: "generated-id")

@@ -384,6 +388,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    assert save_calls[-1]["id"] == "generated-id"
    assert save_calls[-1]["tenant_id"] == "tenant-1"
    assert save_calls[-1]["input_type"] == module.InputType.POLL
+    assert save_calls[-1]["status"] == module.TaskStatus.UNSTART
    assert res["data"]["id"] == "generated-id"

    list_res = module.list_connector()
@@ -401,14 +406,6 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    logs_res = module.list_logs("conn-log")
    assert logs_res["data"] == {"total": 9, "logs": [{"id": "log-1"}]}

-    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": True}))
-    assert _run(module.resume("conn-r1"))["data"] is True
-
-    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": False}))
-    assert _run(module.resume("conn-r2"))["data"] is True
-    assert ("conn-r1", module.TaskStatus.SCHEDULE) in resume_calls
-    assert ("conn-r2", module.TaskStatus.CANCEL) in resume_calls
-
    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"kb_id": "kb-1"}))
    monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: "rebuild-failed")
    failed_rebuild = _run(module.rebuild("conn-rb"))
@@ -421,7 +418,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):

    rm_res = module.rm_connector("conn-rm")
    assert rm_res["data"] is True
-    assert ("conn-rm", module.TaskStatus.CANCEL) in resume_calls
+    assert cancel_calls == ["conn-rm"]
    assert delete_calls == ["conn-rm"]


@@ -434,14 +431,14 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
    monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
    monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
    monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
-    monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+    monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda *_args: touched.append("cancel_tasks"))
    monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
    monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
    monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))

    def _get_request_json():
        touched.append("get_request_json")
-        return _AwaitableValue({"resume": True, "config": {"x": 1}})
+        return _AwaitableValue({"config": {"x": 1}})

    monkeypatch.setattr(module, "get_request_json", _get_request_json)

@@ -449,7 +446,6 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
        _run(module.update_connector("conn-victim")),
        module.get_connector("conn-victim"),
        module.list_logs("conn-victim"),
-        _run(module.resume("conn-victim")),
        _run(module.rebuild("conn-victim")),
        module.rm_connector("conn-victim"),
        _run(module.test_connector("conn-victim")),
--- a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
+++ b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py
@@ -205,7 +205,7 @@ def _load_connector_app(monkeypatch):
            return True

        @staticmethod
-        def resume(*_args, **_kwargs):
+        def cancel_tasks(*_args, **_kwargs):
            return True

        @staticmethod
@@ -349,7 +349,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    records = {"conn-1": _FakeConnectorRecord({"id": "conn-1", "source": "drive"})}
    update_calls = []
    save_calls = []
-    resume_calls = []
+    cancel_calls = []
    delete_calls = []

    monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda cid, payload: update_calls.append((cid, payload)))
@@ -362,7 +362,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda cid: (True, records[cid]))
    monkeypatch.setattr(module.ConnectorService, "list", lambda tenant_id: [{"id": "listed", "tenant": tenant_id}])
    monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda cid, page, page_size: ([{"id": "log-1"}], 9))
-    monkeypatch.setattr(module.ConnectorService, "resume", lambda cid, status: resume_calls.append((cid, status)))
+    monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda cid: cancel_calls.append(cid))
    monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda cid: delete_calls.append(cid))
    monkeypatch.setattr(module, "get_uuid", lambda: "generated-id")

@@ -401,14 +401,6 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):
    logs_res = module.list_logs("conn-log")
    assert logs_res["data"] == {"total": 9, "logs": [{"id": "log-1"}]}

-    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": True}))
-    assert _run(module.resume("conn-r1"))["data"] is True
-
-    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"resume": False}))
-    assert _run(module.resume("conn-r2"))["data"] is True
-    assert ("conn-r1", module.TaskStatus.SCHEDULE) in resume_calls
-    assert ("conn-r2", module.TaskStatus.CANCEL) in resume_calls
-
    monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"kb_id": "kb-1"}))
    monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: "rebuild-failed")
    failed_rebuild = _run(module.rebuild("conn-rb"))
@@ -421,7 +413,7 @@ def test_connector_basic_routes_and_task_controls(monkeypatch):

    rm_res = module.rm_connector("conn-rm")
    assert rm_res["data"] is True
-    assert ("conn-rm", module.TaskStatus.CANCEL) in resume_calls
+    assert cancel_calls == ["conn-rm"]
    assert delete_calls == ["conn-rm"]


@@ -434,14 +426,14 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
    monkeypatch.setattr(module.ConnectorService, "accessible", lambda cid, uid: False)
    monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda *_args: touched.append("get_by_id"))
    monkeypatch.setattr(module.SyncLogsService, "list_sync_tasks", lambda *_args: touched.append("list_sync_tasks"))
-    monkeypatch.setattr(module.ConnectorService, "resume", lambda *_args: touched.append("resume"))
+    monkeypatch.setattr(module.ConnectorService, "cancel_tasks", lambda *_args: touched.append("cancel_tasks"))
    monkeypatch.setattr(module.ConnectorService, "delete_by_id", lambda *_args: touched.append("delete_by_id"))
    monkeypatch.setattr(module.ConnectorService, "update_by_id", lambda *_args: touched.append("update_by_id"))
    monkeypatch.setattr(module.ConnectorService, "rebuild", lambda *_args: touched.append("rebuild"))

    def _get_request_json():
        touched.append("get_request_json")
-        return _AwaitableValue({"resume": True, "config": {"x": 1}})
+        return _AwaitableValue({"config": {"x": 1}})

    monkeypatch.setattr(module, "get_request_json", _get_request_json)

@@ -449,7 +441,6 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):
        _run(module.update_connector("conn-victim")),
        module.get_connector("conn-victim"),
        module.list_logs("conn-victim"),
-        _run(module.resume("conn-victim")),
        _run(module.rebuild("conn-victim")),
        module.rm_connector("conn-victim"),
        _run(module.test_connector("conn-victim")),
--- a/test/unit_test/rag/test_sync_data_source.py
+++ b/test/unit_test/rag/test_sync_data_source.py
@@ -133,7 +133,53 @@ def _patch_common_dependencies(monkeypatch):

@pytest.mark.anyio
@pytest.mark.p2
-async def test_run_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
+async def test_run_task_logic_skips_empty_sync_batches(monkeypatch):
+    _patch_common_dependencies(monkeypatch)
+    monkeypatch.setattr(
+        sync_data_source.SyncLogsService,
+        "increase_docs",
+        lambda *_args, **_kwargs: pytest.fail("increase_docs should not be called for empty batches"),
+    )
+    monkeypatch.setattr(
+        sync_data_source.KnowledgebaseService,
+        "get_by_id",
+        lambda *_args, **_kwargs: pytest.fail("get_by_id should not be called for empty batches"),
+    )
+    monkeypatch.setattr(
+        sync_data_source.SyncLogsService,
+        "duplicate_and_parse",
+        lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not be called for empty batches"),
+    )
+
+    await _FakeSync(iter(([],)))._run_task_logic(_make_task())
+
+
+@pytest.mark.anyio
+@pytest.mark.p2
+async def test_run_task_logic_skips_multiple_empty_sync_batches(monkeypatch):
+    _patch_common_dependencies(monkeypatch)
+    monkeypatch.setattr(
+        sync_data_source.SyncLogsService,
+        "increase_docs",
+        lambda *_args, **_kwargs: pytest.fail("increase_docs should not be called for empty batches"),
+    )
+    monkeypatch.setattr(
+        sync_data_source.KnowledgebaseService,
+        "get_by_id",
+        lambda *_args, **_kwargs: pytest.fail("get_by_id should not be called for empty batches"),
+    )
+    monkeypatch.setattr(
+        sync_data_source.SyncLogsService,
+        "duplicate_and_parse",
+        lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not be called for empty batches"),
+    )
+
+    await _FakeSync(iter(([], [],)))._run_task_logic(_make_task())
+
+
+@pytest.mark.anyio
+@pytest.mark.p2
+async def test_run_prune_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
    cleanup_calls = []

    _patch_common_dependencies(monkeypatch)
@@ -148,7 +194,14 @@ async def test_run_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
        _fake_cleanup,
    )

-    await _FakeSync((iter(()), []))._run_task_logic(_make_task())
+    task = {**_make_task(), "task_type": sync_data_source.ConnectorTaskType.PRUNE}
+    sync = _FakeSync(iter(()))
+    sync.conf["sync_deleted_files"] = True
+    sync.connector = types.SimpleNamespace(
+        retrieve_all_slim_docs_perm_sync=lambda: iter(([],))
+    )
+
+    await sync._run_task_logic(task)

    assert cleanup_calls == [
        (
@@ -166,7 +219,7 @@ async def test_run_task_logic_cleans_up_for_empty_snapshot(monkeypatch):

@pytest.mark.anyio
@pytest.mark.p2
-async def test_run_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
+async def test_run_prune_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
    cleanup_calls = []

    _patch_common_dependencies(monkeypatch)
@@ -182,7 +235,14 @@ async def test_run_task_logic_cleans_up_for_non_empty_snapshot(monkeypatch):
    )

    file_list = [types.SimpleNamespace(id="doc-1")]
-    await _FakeSync((iter(()), file_list))._run_task_logic(_make_task())
+    task = {**_make_task(), "task_type": sync_data_source.ConnectorTaskType.PRUNE}
+    sync = _FakeSync(iter(()))
+    sync.conf["sync_deleted_files"] = True
+    sync.connector = types.SimpleNamespace(
+        retrieve_all_slim_docs_perm_sync=lambda: iter((file_list,))
+    )
+
+    await sync._run_task_logic(task)

    assert cleanup_calls == [
        (
@@ -285,12 +345,13 @@ async def test_rdbms_generate_keeps_deleted_file_snapshot_without_timestamp_colu
        }
    )

-    document_generator, file_list = await sync._generate(task)
+    document_generator = await sync._generate(task)
    connector = _FakeRDBMSConnector.instance

    assert connector is not None
    assert connector.load_from_state_called is True
    assert connector.load_from_cursor_range_called is False
+    file_list = sync._collect_prune_snapshot(task)
    assert connector.retrieve_all_slim_docs_perm_sync_called is True
    assert file_list is not None
    assert [doc.id for doc in file_list] == ["row-1"]
@@ -447,14 +508,15 @@ async def test_dropbox_generate_returns_snapshot_when_sync_deleted_enabled(monke
        }
    )

-    document_generator, file_list = await sync._generate(task)
+    document_generator = await sync._generate(task)
    connector = _FakeDropboxConnector.instance

    assert list(document_generator) == [["poll-sync"]]
+    file_list = sync._collect_prune_snapshot(task)
    assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
    assert connector.credentials == {"dropbox_access_token": "token-1"}
    assert connector.retrieve_all_slim_docs_perm_sync_called is True
-    assert connector.snapshot_called_before_poll is True
+    assert connector.snapshot_called_before_poll is False
    assert connector.poll_source_call[0] == poll_start.timestamp()
    assert connector.poll_source_call[1] >= poll_start.timestamp()

@@ -477,11 +539,12 @@ async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):
        }
    )

-    document_generator, file_list = await sync._generate(task)
+    document_generator = await sync._generate(task)
    connector = _FakeDropboxConnector.instance

    assert list(document_generator) == [["full-sync"]]
-    assert file_list is None
    assert connector.load_from_state_called is True
-    assert connector.retrieve_all_slim_docs_perm_sync_called is False
+    file_list = sync._collect_prune_snapshot(task)
+    assert [doc.id for doc in file_list] == ["dropbox:id-1", "dropbox:id-2"]
+    assert connector.retrieve_all_slim_docs_perm_sync_called is True
    assert connector.poll_source_called is False