Feat: full optimization on connector dashboard (#14979)

### What problem does this PR solve?

This PR improves the connector dashboard task management experience and
adds better visibility into connector execution logs.

### Overview:

#### Before
<img width="700" alt="image"
src="https://github.com/user-attachments/assets/e4a8ed6f-2e18-4f0f-8528-41a514550052"
/>

#### Now:
<img width="700" alt="Screenshot from 2026-05-18 16-31-30"
src="https://github.com/user-attachments/assets/d4ca193b-847a-49ae-9e4f-5fbca60ea627"
/>

### 1. Add a new logging page to the connector dashboard

A new logging page has been added so users can view connector task
execution logs directly from the connector dashboard.

### 2. Merge the Resume button into Confirm

The separate **Resume** button has been removed. The **Confirm** button
now represents different actions depending on the current task state:

- **Save**: Save form changes and reschedule tasks.
- **Stop**: Cancel currently scheduled or running tasks.
- **Resume**: Create new scheduled tasks after the previous tasks have
been stopped.
- **Start**: Start tasks when no task has been started yet.

### 3. Separate syncing and pruning tasks

Connector tasks are now separated into **syncing** and **pruning**.

Pruning is controlled by the **Sync deleted files** option:

- When **Sync deleted files** is disabled, only syncing tasks are shown.
- When **Sync deleted files** is enabled, both syncing and pruning tasks
are shown.

**Now: Sync deleted files disabled**

<img width="700" alt="Sync deleted files disabled"
src="https://github.com/user-attachments/assets/dbd9232e-614a-407f-a0b1-c109e5fa567d"
/>

**Now: Sync deleted files enabled**

<img width="700" alt="Sync deleted files enabled"
src="https://github.com/user-attachments/assets/1f527f48-ccb3-4ee8-97ca-086891489296"
/>

### 4. Update logs in backend

<img width="700" alt="image"
src="https://github.com/user-attachments/assets/10a95a3f-98c1-4e67-8afa-ddf6cda5b0b2"
/>

### 5. Remove connector resume API

- Removed: `POST /v1/connectors/<connector_id>/resume`
- Replaced by: `PATCH /v1/connectors/<connector_id>`


### Type of change

- [x] New Feature (non-breaking change which adds functionality)

api/apps/restful_apis/connector_api.py

@@ -53,17 +53,34 @@ async def update_connector(connector_id):
         return _connector_auth_error(connector_id, current_user.id)
 
     req = await get_request_json()
+    if isinstance(req, dict) and isinstance(req.get("data"), dict):
+        req = req["data"]
+
     e, conn = ConnectorService.get_by_id(connector_id)
     if not e:
         return get_data_error_result(message="Can't find this Connector!")
 
+    should_sleep = False
     if req:
-        conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
-        conn["id"] = connector_id
-        ConnectorService.update_by_id(connector_id, conn)
+        update_fields = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
+        if update_fields:
+            update_fields["id"] = connector_id
+            ConnectorService.update_by_id(connector_id, update_fields)
+            should_sleep = True
 
-    await asyncio.sleep(1)
+        if req.get("reschedule"):
+            ConnectorService.cancel_tasks(connector_id)
+            ConnectorService.schedule_tasks(connector_id)
+        elif req.get("status") in [TaskStatus.CANCEL, "CANCEL"]:
+            ConnectorService.cancel_tasks(connector_id)
+        elif req.get("status") in [TaskStatus.SCHEDULE, "SCHEDULE"]:
+            ConnectorService.schedule_tasks(connector_id)
+
+    if should_sleep:
+        await asyncio.sleep(1)
     e, conn = ConnectorService.get_by_id(connector_id)
+    if not e:
+        return get_data_error_result(message="Can't find this Connector!")
 
     return get_json_result(data=conn.to_dict())
 
@@ -83,9 +100,9 @@ async def create_connector():
             "input_type": InputType.POLL,
             "config": req["config"],
             "refresh_freq": int(req.get("refresh_freq", 5)),
-            "prune_freq": int(req.get("prune_freq", 720)),
+            "prune_freq": int(req.get("prune_freq", 5)),
             "timeout_secs": int(req.get("timeout_secs", 60 * 29)),
-            "status": TaskStatus.SCHEDULE,
+            "status": TaskStatus.UNSTART,
         }
         ConnectorService.save(**conn)
 
@@ -127,21 +144,6 @@ def list_logs(connector_id):
     return get_json_result(data={"total": total, "logs": arr})
 
 
-@manager.route("/connectors/<connector_id>/resume", methods=["POST"])  # noqa: F821
-@login_required
-async def resume(connector_id):
-    """Resume or cancel sync for an accessible connector."""
-    if not ConnectorService.accessible(connector_id, current_user.id):
-        return _connector_auth_error(connector_id, current_user.id)
-
-    req = await get_request_json()
-    if req.get("resume"):
-        ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
-    else:
-        ConnectorService.resume(connector_id, TaskStatus.CANCEL)
-    return get_json_result(data=True)
-
-
 @manager.route("/connectors/<connector_id>/rebuild", methods=["POST"])  # noqa: F821
 @login_required
 async def rebuild(connector_id):
@@ -166,7 +168,7 @@ def rm_connector(connector_id):
     if not ConnectorService.accessible(connector_id, current_user.id):
         return _connector_auth_error(connector_id, current_user.id)
 
-    ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    ConnectorService.cancel_tasks(connector_id)
     ConnectorService.delete_by_id(connector_id)
     return get_json_result(data=True)
 

api/db/db_models.py

@@ -1224,6 +1224,7 @@ class DateTimeTzField(CharField):
 class SyncLogs(DataBaseModel):
     id = CharField(max_length=32, primary_key=True)
     connector_id = CharField(max_length=32, index=True)
+    task_type = CharField(max_length=32, null=False, default="sync", index=True)
     status = CharField(max_length=128, null=False, help_text="Processing status", index=True)
     from_beginning = CharField(max_length=1, null=True, help_text="", default="0", index=False)
     new_docs_indexed = IntegerField(default=0, index=False)
@@ -1632,6 +1633,7 @@ def migrate_db():
     alter_db_add_column(migrator, "llm_factories", "rank", IntegerField(default=0, index=False))
     alter_db_add_column(migrator, "api_4_conversation", "name", CharField(max_length=255, null=True, help_text="conversation name", index=False))
     alter_db_add_column(migrator, "api_4_conversation", "exp_user_id", CharField(max_length=255, null=True, help_text="exp_user_id", index=True))
+    alter_db_add_column(migrator, "sync_logs", "task_type", CharField(max_length=32, null=False, default="sync", index=True))
     # Migrate system_settings.value from CharField to TextField for longer sandbox configs
     alter_db_column_type(migrator, "system_settings", "value", TextField(null=False, help_text="Configuration value (JSON, string, etc.)"))
     alter_db_add_column(migrator, "document", "content_hash", CharField(max_length=32, null=True, help_text="xxhash128 of document content for change detection", default="", index=True))

api/db/services/connector_service.py

@@ -28,7 +28,7 @@ from api.db.services.document_service import DocumentService
 from api.db.services.document_service import DocMetadataService
 from api.utils.common import hash128
 from common.misc_utils import get_uuid
-from common.constants import TaskStatus
+from common.constants import ConnectorTaskType, TaskStatus
 from common.settings import TIMEZONE
 from common.time_utils import current_timestamp, timestamp_to_date
 
@@ -38,6 +38,33 @@ LOGGER = logging.getLogger(__name__)
 class ConnectorService(CommonService):
     model = Connector
 
+    @classmethod
+    def cancel_tasks(cls, connector_id):
+        e, conn = cls.get_by_id(connector_id)
+        if not e:
+            return
+
+        logging.info(
+            "[Connector] stop connector=%s(%s)",
+            conn.name,
+            connector_id,
+        )
+        for c2k in Connector2KbService.query(connector_id=connector_id):
+            SyncLogsService.filter_update(
+                [
+                    SyncLogs.connector_id == connector_id,
+                    SyncLogs.kb_id == c2k.kb_id,
+                    SyncLogs.status.in_([TaskStatus.SCHEDULE, TaskStatus.RUNNING]),
+                ],
+                {"status": TaskStatus.CANCEL},
+            )
+        ConnectorService.update_by_id(connector_id, {"status": TaskStatus.CANCEL})
+        logging.info(
+            "[Connector] connector=%s status updated to %s",
+            connector_id,
+            TaskStatus.CANCEL,
+        )
+
     @classmethod
     @DB.connection_context()
     def accessible(cls, connector_id: str, user_id: str) -> bool:
@@ -64,25 +91,39 @@ class ConnectorService(CommonService):
         return has_access
 
     @classmethod
-    def resume(cls, connector_id, status):
+    def schedule_tasks(cls, connector_id):
+        e, conn = cls.get_by_id(connector_id)
+        if not e:
+            return
+
+        logging.info("[Connector] schedule connector=%s(%s)", conn.name, connector_id)
+        prune_enabled = bool((conn.config or {}).get("sync_deleted_files"))
         for c2k in Connector2KbService.query(connector_id=connector_id):
-            task = SyncLogsService.get_latest_task(connector_id, c2k.kb_id)
-            if not task:
-                if status == TaskStatus.SCHEDULE:
-                    SyncLogsService.schedule(connector_id, c2k.kb_id)
-                    ConnectorService.update_by_id(connector_id, {"status": status})
-                    return
+            sync_task = SyncLogsService.get_latest_task(
+                connector_id,
+                c2k.kb_id,
+                ConnectorTaskType.SYNC,
+            )
+            poll_range_start = None
+            total_docs_indexed = 0
+            if sync_task and sync_task.status == TaskStatus.DONE:
+                poll_range_start = sync_task.poll_range_end
+                total_docs_indexed = sync_task.total_docs_indexed
 
-            if task.status == TaskStatus.DONE:
-                if status == TaskStatus.SCHEDULE:
-                    SyncLogsService.schedule(connector_id, c2k.kb_id, task.poll_range_end, total_docs_indexed=task.total_docs_indexed)
-                    ConnectorService.update_by_id(connector_id, {"status": status})
-                    return
+            SyncLogsService.schedule(
+                connector_id,
+                c2k.kb_id,
+                poll_range_start,
+                total_docs_indexed=total_docs_indexed,
+                task_type=ConnectorTaskType.SYNC,
+            )
 
-            task = task.to_dict()
-            task["status"] = status
-            SyncLogsService.update_by_id(task["id"], task)
-        ConnectorService.update_by_id(connector_id, {"status": status})
+            if prune_enabled:
+                SyncLogsService.schedule(
+                    connector_id,
+                    c2k.kb_id,
+                    task_type=ConnectorTaskType.PRUNE,
+                )
 
     @classmethod
     def list(cls, tenant_id):
@@ -105,7 +146,9 @@ class ConnectorService(CommonService):
         SyncLogsService.filter_delete([SyncLogs.connector_id==connector_id, SyncLogs.kb_id==kb_id])
         docs = DocumentService.query(source_type=f"{conn.source}/{conn.id}", kb_id=kb_id)
         err = FileService.delete_docs([d.id for d in docs], tenant_id)
-        SyncLogsService.schedule(connector_id, kb_id, reindex=True)
+        SyncLogsService.schedule(connector_id, kb_id, reindex=True, task_type=ConnectorTaskType.SYNC)
+        if (conn.config or {}).get("sync_deleted_files"):
+            SyncLogsService.schedule(connector_id, kb_id, task_type=ConnectorTaskType.PRUNE)
         return err
 
     @classmethod
@@ -170,30 +213,25 @@ class ConnectorService(CommonService):
 class SyncLogsService(CommonService):
     model = SyncLogs
 
+    
     @classmethod
     def list_sync_tasks(cls, connector_id=None, page_number=None, items_per_page=15) -> Tuple[List[dict], int]:
         fields = [
             cls.model.id,
             cls.model.connector_id,
+            cls.model.task_type,
             cls.model.kb_id,
             cls.model.update_date,
-            cls.model.poll_range_start,
-            cls.model.poll_range_end,
             cls.model.new_docs_indexed,
             cls.model.total_docs_indexed,
+            cls.model.docs_removed_from_index,
             cls.model.error_msg,
-            cls.model.full_exception_trace,
             cls.model.error_count,
-            Connector.name,
-            Connector.source,
-            Connector.tenant_id,
-            Connector.timeout_secs,
+            cls.model.time_started.alias("time_started"),
+            Connector.refresh_freq.alias("refresh_freq"),
+            Connector.prune_freq.alias("prune_freq"),
             Knowledgebase.name.alias("kb_name"),
-            Knowledgebase.avatar.alias("kb_avatar"),
-            Connector2Kb.auto_parse,
-            cls.model.from_beginning.alias("reindex"),
             cls.model.status,
-            cls.model.update_time
         ]
         if not connector_id:
             fields.append(Connector.config)
@@ -225,6 +263,80 @@ class SyncLogsService(CommonService):
 
         return list(query.dicts()), total
 
+    @classmethod
+    def list_due_sync_tasks(cls) -> List[dict]:
+        return cls._list_due_tasks_for_freq(
+            ConnectorTaskType.SYNC,
+            "refresh_freq",
+        )
+
+    @classmethod
+    def list_due_prune_tasks(cls) -> List[dict]:
+        tasks = cls._list_due_tasks_for_freq(
+            ConnectorTaskType.PRUNE,
+            "prune_freq",
+        )
+        return [
+            task for task in tasks
+            # Prune is opt-in at the connector config level; keep the scheduler
+            # blind to prune_freq until the flag is enabled.
+            if bool((task.get("config") or {}).get("sync_deleted_files"))
+            and int(task.get("prune_freq") or 0) > 0
+        ]
+
+    @classmethod
+    def _list_due_tasks_for_freq(cls, task_type: str, freq_field: str) -> List[dict]:
+        fields = [
+            cls.model.id,
+            cls.model.connector_id,
+            cls.model.task_type,
+            cls.model.kb_id,
+            cls.model.update_date,
+            cls.model.poll_range_start,
+            cls.model.poll_range_end,
+            cls.model.new_docs_indexed,
+            cls.model.total_docs_indexed,
+            cls.model.error_msg,
+            cls.model.full_exception_trace,
+            cls.model.error_count,
+            Connector.name,
+            Connector.source,
+            Connector.tenant_id,
+            Connector.timeout_secs,
+            Connector.config,
+            Connector.refresh_freq,
+            Connector.prune_freq,
+            Knowledgebase.name.alias("kb_name"),
+            Knowledgebase.avatar.alias("kb_avatar"),
+            Connector2Kb.auto_parse,
+            cls.model.from_beginning.alias("reindex"),
+            cls.model.status,
+            cls.model.update_time,
+        ]
+
+        query = cls.model.select(*fields)\
+            .join(Connector, on=(cls.model.connector_id==Connector.id))\
+            .join(Connector2Kb, on=(cls.model.kb_id==Connector2Kb.kb_id))\
+            .join(Knowledgebase, on=(cls.model.kb_id==Knowledgebase.id))
+
+        query = query.where(
+            Connector.input_type == InputType.POLL,
+            Connector.status == TaskStatus.SCHEDULE,
+            cls.model.status == TaskStatus.SCHEDULE,
+            cls.model.task_type == task_type,
+        )
+
+        database_type = os.getenv("DB_TYPE", "mysql")
+        if "postgres" in database_type.lower():
+            expr = SQL(
+                f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})"
+            )
+        else:
+            expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")
+        query = query.where(cls.model.update_date < expr)
+
+        return list(query.distinct().order_by(cls.model.update_time.desc()).dicts())
+
     @classmethod
     def start(cls, id, connector_id):
         cls.update_by_id(id, {"status": TaskStatus.RUNNING, "time_started": datetime.now().strftime('%Y-%m-%d %H:%M:%S') })
@@ -236,7 +348,15 @@ class SyncLogsService(CommonService):
         ConnectorService.update_by_id(connector_id, {"status": TaskStatus.DONE})
 
     @classmethod
-    def schedule(cls, connector_id, kb_id, poll_range_start=None, reindex=False, total_docs_indexed=0):
+    def schedule(
+        cls,
+        connector_id,
+        kb_id,
+        poll_range_start=None,
+        reindex=False,
+        total_docs_indexed=0,
+        task_type=ConnectorTaskType.SYNC,
+    ):
         try:
             if cls.model.select().where(cls.model.kb_id == kb_id, cls.model.connector_id == connector_id).count() > 100:
                 rm_ids = [m.id for m in cls.model.select(cls.model.id).where(cls.model.kb_id == kb_id, cls.model.connector_id == connector_id).order_by(cls.model.update_time.asc()).limit(70)]
@@ -246,21 +366,33 @@ class SyncLogsService(CommonService):
             logging.exception(e)
 
         try:
-            e = cls.query(kb_id=kb_id, connector_id=connector_id, status=TaskStatus.SCHEDULE)
+            e = cls.query(
+                kb_id=kb_id,
+                connector_id=connector_id,
+                status=TaskStatus.SCHEDULE,
+                task_type=task_type,
+            )
             if e:
-                logging.warning(f"{kb_id}--{connector_id} has already had a scheduling sync task which is abnormal.")
+                logging.warning(
+                    "%s--%s already has a scheduled %s task.",
+                    kb_id,
+                    connector_id,
+                    task_type,
+                )
                 return None
             reindex = "1" if reindex else "0"
             ConnectorService.update_by_id(connector_id, {"status": TaskStatus.SCHEDULE})
             return cls.save(**{
                 "id": get_uuid(),
                 "kb_id": kb_id, "status": TaskStatus.SCHEDULE, "connector_id": connector_id,
+                "task_type": task_type,
                 "poll_range_start": poll_range_start, "from_beginning": reindex,
-                "total_docs_indexed": total_docs_indexed
+                "total_docs_indexed": total_docs_indexed,
+                "time_started": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
             })
         except Exception as e:
             logging.exception(e)
-            task = cls.get_latest_task(connector_id, kb_id)
+            task = cls.get_latest_task(connector_id, kb_id, task_type)
             if task:
                 cls.model.update(status=TaskStatus.SCHEDULE,
                                  poll_range_start=poll_range_start,
@@ -337,11 +469,14 @@ class SyncLogsService(CommonService):
         return errs, doc_ids
 
     @classmethod
-    def get_latest_task(cls, connector_id, kb_id):
-        return cls.model.select().where(
+    def get_latest_task(cls, connector_id, kb_id, task_type=None):
+        query = cls.model.select().where(
             cls.model.connector_id==connector_id,
             cls.model.kb_id == kb_id
-        ).order_by(cls.model.update_time.desc()).first()
+        )
+        if task_type is not None:
+            query = query.where(cls.model.task_type == task_type)
+        return query.order_by(cls.model.update_time.desc()).first()
 
 
 class Connector2KbService(CommonService):
@@ -364,7 +499,10 @@ class Connector2KbService(CommonService):
                 "kb_id": kb_id,
                 "auto_parse": conn.get("auto_parse", "1")
             })
-            SyncLogsService.schedule(conn_id, kb_id, reindex=True)
+            SyncLogsService.schedule(conn_id, kb_id, reindex=True, task_type=ConnectorTaskType.SYNC)
+            e, full_conn = ConnectorService.get_by_id(conn_id)
+            if e and (full_conn.config or {}).get("sync_deleted_files"):
+                SyncLogsService.schedule(conn_id, kb_id, task_type=ConnectorTaskType.PRUNE)
 
         errs = []
         for conn_id in old_conn_ids: