Feat: enable sync deleted files for more connectors (#14353)

### What problem does this PR solve?

Feat: enable sync delted files for connectors

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

common/data_source/box_connector.py

@@ -1,7 +1,7 @@
 """Box connector"""
 import logging
 from datetime import datetime, timezone
-from typing import Any
+from typing import Any, Generator
 
 from box_sdk_gen import BoxClient
 from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE
@@ -10,21 +10,21 @@ from common.data_source.exceptions import (
     ConnectorValidationError,
 )
 from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch
-from common.data_source.models import Document, GenerateDocumentsOutput
+from common.data_source.models import Document, GenerateDocumentsOutput, GenerateSlimDocumentOutput, SlimDocument
 from common.data_source.utils import get_file_ext
 
+
 class BoxConnector(LoadConnector, PollConnector):
     def __init__(self, folder_id: str, batch_size: int = INDEX_BATCH_SIZE, use_marker: bool = True) -> None:
         self.batch_size = batch_size
         self.folder_id = "0" if not folder_id else folder_id
         self.use_marker = use_marker
-        
+        self.box_client: BoxClient | None = None
 
     def load_credentials(self, auth: Any):
         self.box_client = BoxClient(auth=auth)
         return None
 
-
     def validate_connector_settings(self):
         if self.box_client is None:
             raise ConnectorMissingCredentialError("Box")
@@ -35,79 +35,41 @@ class BoxConnector(LoadConnector, PollConnector):
             logging.exception("[Box]: Failed to validate Box credentials")
             raise ConnectorValidationError(f"Unexpected error during Box settings validation: {e}")
 
-
-    def _yield_files_recursive(
-            self,
-            folder_id: str,
-            start: SecondsSinceUnixEpoch | None,
-            end: SecondsSinceUnixEpoch | None,
-            relative_folder_path: str = "",
-        ) -> GenerateDocumentsOutput:
-
+    def _iter_files_recursive(
+        self,
+        folder_id: str,
+        relative_folder_path: str = "",
+    ) -> Generator[tuple[Any, str], None, None]:
         if self.box_client is None:
             raise ConnectorMissingCredentialError("Box")
 
         result = self.box_client.folders.get_folder_items(
             folder_id=folder_id,
             limit=self.batch_size,
-            usemarker=self.use_marker
+            usemarker=self.use_marker,
         )
 
         while True:
-            batch: list[Document] = []
             for entry in result.entries:
-                if entry.type == 'file' :
-                    file = self.box_client.files.get_file_by_id(
-                        entry.id
-                    )
-                    modified_time: SecondsSinceUnixEpoch | None = None
-                    raw_time = (
-                        getattr(file, "created_at", None)
-                        or getattr(file, "content_created_at", None)
-                    )
-
-                    if raw_time:
-                        modified_time = self._box_datetime_to_epoch_seconds(raw_time)
-                        if start is not None and modified_time <= start:
-                            continue
-                        if end is not None and modified_time > end:
-                            continue
-
-                    content_bytes = self.box_client.downloads.download_file(file.id)
+                if entry.type == "file":
+                    file = self.box_client.files.get_file_by_id(entry.id)
                     semantic_identifier = (
                         f"{relative_folder_path} / {file.name}"
                         if relative_folder_path
                         else file.name
                     )
-
-                    batch.append(
-                        Document(
-                            id=f"box:{file.id}",
-                            blob=content_bytes.read(),
-                            source=DocumentSource.BOX,
-                            semantic_identifier=semantic_identifier,
-                            extension=get_file_ext(file.name),
-                            doc_updated_at=modified_time,
-                            size_bytes=file.size,
-                            metadata=file.metadata
-                        )
-                    )
-                elif entry.type == 'folder':
+                    yield file, semantic_identifier
+                elif entry.type == "folder":
                     child_relative_path = (
                         f"{relative_folder_path} / {entry.name}"
                         if relative_folder_path
                         else entry.name
                     )
-                    yield from self._yield_files_recursive(
+                    yield from self._iter_files_recursive(
                         folder_id=entry.id,
-                        start=start,
-                        end=end,
-                        relative_folder_path=child_relative_path
+                        relative_folder_path=child_relative_path,
                     )
 
-            if batch:
-                yield batch
-
             if not result.next_marker:
                 break
 
@@ -115,9 +77,56 @@ class BoxConnector(LoadConnector, PollConnector):
                 folder_id=folder_id,
                 limit=self.batch_size,
                 marker=result.next_marker,
-                usemarker=True
+                usemarker=True,
             )
 
+    def _yield_files_recursive(
+        self,
+        folder_id: str,
+        start: SecondsSinceUnixEpoch | None,
+        end: SecondsSinceUnixEpoch | None,
+        relative_folder_path: str = "",
+    ) -> GenerateDocumentsOutput:
+        if self.box_client is None:
+            raise ConnectorMissingCredentialError("Box")
+
+        batch: list[Document] = []
+        for file, semantic_identifier in self._iter_files_recursive(
+            folder_id=folder_id,
+            relative_folder_path=relative_folder_path,
+        ):
+            modified_time: SecondsSinceUnixEpoch | None = None
+            raw_time = (
+                getattr(file, "created_at", None)
+                or getattr(file, "content_created_at", None)
+            )
+
+            if raw_time:
+                modified_time = self._box_datetime_to_epoch_seconds(raw_time)
+                if start is not None and modified_time <= start:
+                    continue
+                if end is not None and modified_time > end:
+                    continue
+
+            content_bytes = self.box_client.downloads.download_file(file.id)
+            batch.append(
+                Document(
+                    id=f"box:{file.id}",
+                    blob=content_bytes.read(),
+                    source=DocumentSource.BOX,
+                    semantic_identifier=semantic_identifier,
+                    extension=get_file_ext(file.name),
+                    doc_updated_at=modified_time,
+                    size_bytes=file.size,
+                    metadata=file.metadata,
+                )
+            )
+            if len(batch) >= self.batch_size:
+                yield batch
+                batch = []
+
+        if batch:
+            yield batch
 
     def _box_datetime_to_epoch_seconds(self, dt: datetime) -> SecondsSinceUnixEpoch:
         """Convert a Box SDK datetime to Unix epoch seconds (UTC).
@@ -133,6 +142,21 @@ class BoxConnector(LoadConnector, PollConnector):
 
         return SecondsSinceUnixEpoch(int(dt.timestamp()))
 
+    def retrieve_all_slim_docs_perm_sync(
+        self,
+        callback: Any = None,
+    ) -> GenerateSlimDocumentOutput:
+        del callback
+
+        batch: list[SlimDocument] = []
+        for file, _semantic_identifier in self._iter_files_recursive(folder_id=self.folder_id):
+            batch.append(SlimDocument(id=f"box:{file.id}"))
+            if len(batch) >= self.batch_size:
+                yield batch
+                batch = []
+
+        if batch:
+            yield batch
 
     def poll_source(self, start, end):
         return self._yield_files_recursive(folder_id=self.folder_id, start=start, end=end)