Feat: enable sync deleted files for more connectors (#14353)

### What problem does this PR solve?

Feat: enable sync delted files for connectors

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

common/data_source/bitbucket/connector.py

@@ -269,17 +269,11 @@ class BitbucketConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> Iterator[list[SlimDocument]]:
         """Return only document IDs for all existing pull requests."""
         batch: list[SlimDocument] = []
-        params = self._build_params(
-            fields=SLIM_PR_LIST_RESPONSE_FIELDS,
-            start=start,
-            end=end,
-        )
+        params = self._build_params(fields=SLIM_PR_LIST_RESPONSE_FIELDS)
         with self._client() as client:
             for slug in self._iter_target_repositories(client):
                 for pr in self._iter_pull_requests_for_repo(
@@ -385,4 +379,4 @@ if __name__ == "__main__":
             except StopIteration as e:
                 bitbucket_checkpoint = e.value  
                 break
-        
+        

common/data_source/blob_connector.py

@@ -10,7 +10,6 @@ from common.data_source.utils import (
     download_object,
     extract_size_bytes,
     get_file_ext,
-    is_accepted_file_ext,
 )
 from common.data_source.config import BlobType, DocumentSource, BLOB_STORAGE_SIZE_THRESHOLD, INDEX_BATCH_SIZE
 from common.data_source.exceptions import (
@@ -19,8 +18,14 @@ from common.data_source.exceptions import (
     CredentialExpiredError,
     InsufficientPermissionsError
 )
-from common.data_source.interfaces import LoadConnector, OnyxExtensionType, PollConnector
-from common.data_source.models import Document, SecondsSinceUnixEpoch, GenerateDocumentsOutput
+from common.data_source.interfaces import LoadConnector, PollConnector
+from common.data_source.models import (
+    Document,
+    SecondsSinceUnixEpoch,
+    GenerateDocumentsOutput,
+    GenerateSlimDocumentOutput,
+    SlimDocument,
+)
 
 
 class BlobStorageConnector(LoadConnector, PollConnector):
@@ -123,37 +128,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
         end: datetime,
     ) -> GenerateDocumentsOutput:
         """Generate bucket objects"""
-        if self.s3_client is None:
-            raise ConnectorMissingCredentialError("Blob storage")
-
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=self.prefix)
-
-        # Collect all objects first to count filename occurrences
-        all_objects = []
-        extension_type = OnyxExtensionType.Plain | OnyxExtensionType.Document
-        if bool(self._allow_images):
-            extension_type |= OnyxExtensionType.Multimedia
-        for page in pages:
-            if "Contents" not in page:
-                continue
-            for obj in page["Contents"]:
-                key = obj["Key"]
-                if key.endswith("/"):
-                    continue
-                last_modified = obj["LastModified"].replace(tzinfo=timezone.utc)
-                if not (start < last_modified <= end):
-                    continue
-                file_name = os.path.basename(key)
-                if not is_accepted_file_ext(get_file_ext(file_name), extension_type):
-                    continue
-                all_objects.append(obj)
-        
-        # Count filename occurrences to determine which need full paths
-        filename_counts: dict[str, int] = {}
-        for obj in all_objects:
-            file_name = os.path.basename(obj["Key"])
-            filename_counts[file_name] = filename_counts.get(file_name, 0) + 1
+        all_objects, filename_counts = self._collect_blob_objects(start, end)
 
         batch: list[Document] = []
         for obj in all_objects:
@@ -171,20 +146,15 @@ class BlobStorageConnector(LoadConnector, PollConnector):
                     f"{file_name} exceeds size threshold of {self.size_threshold}. Skipping."
                 )
                 continue
-            
+
             try:
-                blob = download_object(self.s3_client, self.bucket_name, key, self.size_threshold)
+                blob = download_object(
+                    self.s3_client, self.bucket_name, key, self.size_threshold
+                )
                 if blob is None:
                     continue
 
-                # Use full path only if filename appears multiple times
-                if filename_counts.get(file_name, 0) > 1:
-                    relative_path = key
-                    if self.prefix and key.startswith(self.prefix):
-                        relative_path = key[len(self.prefix):]
-                    semantic_id = relative_path.replace('/', ' / ') if relative_path else file_name
-                else:
-                    semantic_id = file_name
+                semantic_id = self._get_semantic_id(key, file_name, filename_counts)
 
                 batch.append(
                     Document(
@@ -194,7 +164,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
                         semantic_identifier=semantic_id,
                         extension=get_file_ext(file_name),
                         doc_updated_at=last_modified,
-                        size_bytes=size_bytes if size_bytes else 0
+                        size_bytes=size_bytes if size_bytes else 0,
                     )
                 )
                 if len(batch) == self.batch_size:
@@ -203,7 +173,76 @@ class BlobStorageConnector(LoadConnector, PollConnector):
 
             except Exception:
                 logging.exception(f"Error decoding object {key}")
-        
+
+        if batch:
+            yield batch
+
+    def _collect_blob_objects(
+        self,
+        start: datetime,
+        end: datetime,
+    ) -> tuple[list[dict[str, Any]], dict[str, int]]:
+        """Collect object metadata for files in the requested window."""
+        if self.s3_client is None:
+            raise ConnectorMissingCredentialError("Blob storage")
+
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=self.prefix)
+
+        # Collect all objects first to count filename occurrences
+        all_objects: list[dict[str, Any]] = []
+        for page in pages:
+            if "Contents" not in page:
+                continue
+            for obj in page["Contents"]:
+                if obj["Key"].endswith("/"):
+                    continue
+                last_modified = obj["LastModified"].replace(tzinfo=timezone.utc)
+                if start < last_modified <= end:
+                    all_objects.append(obj)
+
+        filename_counts: dict[str, int] = {}
+        for obj in all_objects:
+            file_name = os.path.basename(obj["Key"])
+            filename_counts[file_name] = filename_counts.get(file_name, 0) + 1
+
+        return all_objects, filename_counts
+
+    def _get_semantic_id(
+        self,
+        key: str,
+        file_name: str,
+        filename_counts: dict[str, int],
+    ) -> str:
+        """Use full relative path only when filenames collide."""
+        if filename_counts.get(file_name, 0) > 1:
+            relative_path = key
+            if self.prefix and key.startswith(self.prefix):
+                relative_path = key[len(self.prefix):]
+            return relative_path.replace("/", " / ") if relative_path else file_name
+        return file_name
+
+    def retrieve_all_slim_docs_perm_sync(
+        self,
+        callback: Any = None,
+    ) -> GenerateSlimDocumentOutput:
+        """Return a full current snapshot of blob object IDs without downloading content."""
+        del callback
+
+        all_objects, _ = self._collect_blob_objects(
+            start=datetime(1970, 1, 1, tzinfo=timezone.utc),
+            end=datetime.now(timezone.utc),
+        )
+
+        batch: list[SlimDocument] = []
+        for obj in all_objects:
+            batch.append(
+                SlimDocument(id=f"{self.bucket_type}:{self.bucket_name}:{obj['Key']}")
+            )
+            if len(batch) == self.batch_size:
+                yield batch
+                batch = []
+
         if batch:
             yield batch
 

common/data_source/box_connector.py

@@ -1,7 +1,7 @@
 """Box connector"""
 import logging
 from datetime import datetime, timezone
-from typing import Any
+from typing import Any, Generator
 
 from box_sdk_gen import BoxClient
 from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE
@@ -10,21 +10,21 @@ from common.data_source.exceptions import (
     ConnectorValidationError,
 )
 from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch
-from common.data_source.models import Document, GenerateDocumentsOutput
+from common.data_source.models import Document, GenerateDocumentsOutput, GenerateSlimDocumentOutput, SlimDocument
 from common.data_source.utils import get_file_ext
 
+
 class BoxConnector(LoadConnector, PollConnector):
     def __init__(self, folder_id: str, batch_size: int = INDEX_BATCH_SIZE, use_marker: bool = True) -> None:
         self.batch_size = batch_size
         self.folder_id = "0" if not folder_id else folder_id
         self.use_marker = use_marker
-        
+        self.box_client: BoxClient | None = None
 
     def load_credentials(self, auth: Any):
         self.box_client = BoxClient(auth=auth)
         return None
 
-
     def validate_connector_settings(self):
         if self.box_client is None:
             raise ConnectorMissingCredentialError("Box")
@@ -35,79 +35,41 @@ class BoxConnector(LoadConnector, PollConnector):
             logging.exception("[Box]: Failed to validate Box credentials")
             raise ConnectorValidationError(f"Unexpected error during Box settings validation: {e}")
 
-
-    def _yield_files_recursive(
-            self,
-            folder_id: str,
-            start: SecondsSinceUnixEpoch | None,
-            end: SecondsSinceUnixEpoch | None,
-            relative_folder_path: str = "",
-        ) -> GenerateDocumentsOutput:
-
+    def _iter_files_recursive(
+        self,
+        folder_id: str,
+        relative_folder_path: str = "",
+    ) -> Generator[tuple[Any, str], None, None]:
         if self.box_client is None:
             raise ConnectorMissingCredentialError("Box")
 
         result = self.box_client.folders.get_folder_items(
             folder_id=folder_id,
             limit=self.batch_size,
-            usemarker=self.use_marker
+            usemarker=self.use_marker,
         )
 
         while True:
-            batch: list[Document] = []
             for entry in result.entries:
-                if entry.type == 'file' :
-                    file = self.box_client.files.get_file_by_id(
-                        entry.id
-                    )
-                    modified_time: SecondsSinceUnixEpoch | None = None
-                    raw_time = (
-                        getattr(file, "created_at", None)
-                        or getattr(file, "content_created_at", None)
-                    )
-
-                    if raw_time:
-                        modified_time = self._box_datetime_to_epoch_seconds(raw_time)
-                        if start is not None and modified_time <= start:
-                            continue
-                        if end is not None and modified_time > end:
-                            continue
-
-                    content_bytes = self.box_client.downloads.download_file(file.id)
+                if entry.type == "file":
+                    file = self.box_client.files.get_file_by_id(entry.id)
                     semantic_identifier = (
                         f"{relative_folder_path} / {file.name}"
                         if relative_folder_path
                         else file.name
                     )
-
-                    batch.append(
-                        Document(
-                            id=f"box:{file.id}",
-                            blob=content_bytes.read(),
-                            source=DocumentSource.BOX,
-                            semantic_identifier=semantic_identifier,
-                            extension=get_file_ext(file.name),
-                            doc_updated_at=modified_time,
-                            size_bytes=file.size,
-                            metadata=file.metadata
-                        )
-                    )
-                elif entry.type == 'folder':
+                    yield file, semantic_identifier
+                elif entry.type == "folder":
                     child_relative_path = (
                         f"{relative_folder_path} / {entry.name}"
                         if relative_folder_path
                         else entry.name
                     )
-                    yield from self._yield_files_recursive(
+                    yield from self._iter_files_recursive(
                         folder_id=entry.id,
-                        start=start,
-                        end=end,
-                        relative_folder_path=child_relative_path
+                        relative_folder_path=child_relative_path,
                     )
 
-            if batch:
-                yield batch
-
             if not result.next_marker:
                 break
 
@@ -115,9 +77,56 @@ class BoxConnector(LoadConnector, PollConnector):
                 folder_id=folder_id,
                 limit=self.batch_size,
                 marker=result.next_marker,
-                usemarker=True
+                usemarker=True,
             )
 
+    def _yield_files_recursive(
+        self,
+        folder_id: str,
+        start: SecondsSinceUnixEpoch | None,
+        end: SecondsSinceUnixEpoch | None,
+        relative_folder_path: str = "",
+    ) -> GenerateDocumentsOutput:
+        if self.box_client is None:
+            raise ConnectorMissingCredentialError("Box")
+
+        batch: list[Document] = []
+        for file, semantic_identifier in self._iter_files_recursive(
+            folder_id=folder_id,
+            relative_folder_path=relative_folder_path,
+        ):
+            modified_time: SecondsSinceUnixEpoch | None = None
+            raw_time = (
+                getattr(file, "created_at", None)
+                or getattr(file, "content_created_at", None)
+            )
+
+            if raw_time:
+                modified_time = self._box_datetime_to_epoch_seconds(raw_time)
+                if start is not None and modified_time <= start:
+                    continue
+                if end is not None and modified_time > end:
+                    continue
+
+            content_bytes = self.box_client.downloads.download_file(file.id)
+            batch.append(
+                Document(
+                    id=f"box:{file.id}",
+                    blob=content_bytes.read(),
+                    source=DocumentSource.BOX,
+                    semantic_identifier=semantic_identifier,
+                    extension=get_file_ext(file.name),
+                    doc_updated_at=modified_time,
+                    size_bytes=file.size,
+                    metadata=file.metadata,
+                )
+            )
+            if len(batch) >= self.batch_size:
+                yield batch
+                batch = []
+
+        if batch:
+            yield batch
 
     def _box_datetime_to_epoch_seconds(self, dt: datetime) -> SecondsSinceUnixEpoch:
         """Convert a Box SDK datetime to Unix epoch seconds (UTC).
@@ -133,6 +142,21 @@ class BoxConnector(LoadConnector, PollConnector):
 
         return SecondsSinceUnixEpoch(int(dt.timestamp()))
 
+    def retrieve_all_slim_docs_perm_sync(
+        self,
+        callback: Any = None,
+    ) -> GenerateSlimDocumentOutput:
+        del callback
+
+        batch: list[SlimDocument] = []
+        for file, _semantic_identifier in self._iter_files_recursive(folder_id=self.folder_id):
+            batch.append(SlimDocument(id=f"box:{file.id}"))
+            if len(batch) >= self.batch_size:
+                yield batch
+                batch = []
+
+        if batch:
+            yield batch
 
     def poll_source(self, start, end):
         return self._yield_files_recursive(folder_id=self.folder_id, start=start, end=end)

common/data_source/confluence_connector.py

@@ -1904,8 +1904,6 @@ class ConfluenceConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         """
@@ -1913,16 +1911,12 @@ class ConfluenceConnector(
         Does not fetch actual text. Used primarily for incremental permission sync.
         """
         return self._retrieve_all_slim_docs(
-            start=start,
-            end=end,
             callback=callback,
             include_permissions=True,
         )
 
     def _retrieve_all_slim_docs(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
         include_permissions: bool = True,
     ) -> GenerateSlimDocumentOutput:

common/data_source/github/connector.py

@@ -964,11 +964,9 @@ class GithubConnector(CheckpointedConnectorWithPermSyncGH[GithubConnectorCheckpo
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: Any = None,
     ) -> GenerateSlimDocumentOutput:
-        yield from self.retrieve_slim_document(start=start, end=end, callback=callback)
+        yield from self.retrieve_slim_document(callback=callback)
 
     def build_dummy_checkpoint(self) -> GithubConnectorCheckpoint:
         return GithubConnectorCheckpoint(

common/data_source/gmail_connector.py

@@ -270,12 +270,10 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback=None,
     ) -> GenerateSlimDocumentOutput:
         """Retrieve slim documents for permission synchronization."""
-        query = build_time_range_query(start, end)
+        query = build_time_range_query()
         doc_batch = []
 
         for user_email in self._get_all_user_emails():
@@ -343,4 +341,4 @@ if __name__ == "__main__":
                 print(f)
                 print("\n\n")
     except Exception as e:
-        logging.exception(f"Error loading credentials: {e}")
+        logging.exception(f"Error loading credentials: {e}")

common/data_source/google_drive/connector.py

@@ -1087,8 +1087,6 @@ class GoogleDriveConnector(SlimConnectorWithPermSync, CheckpointedConnectorWithP
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         try:
@@ -1096,8 +1094,6 @@ class GoogleDriveConnector(SlimConnectorWithPermSync, CheckpointedConnectorWithP
             while checkpoint.completion_stage != DriveRetrievalStage.DONE:
                 yield from self._extract_slim_docs_from_google_drive(
                     checkpoint=checkpoint,
-                    start=start,
-                    end=end,
                 )
             self.logger.info("Drive perm sync: Slim doc retrieval complete")
 

common/data_source/interfaces.py

@@ -60,8 +60,6 @@ class SlimConnectorWithPermSync(ABC):
     @abstractmethod
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: Any = None,
     ) -> Generator[list[SlimDocument], None, None]:
         """Retrieve all simplified documents (with permission sync)"""

common/data_source/jira/connector.py

@@ -149,7 +149,10 @@ class JiraConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSync
             else:
                 logger.warning("[Jira] Scoped token requested but Jira base URL does not appear to be an Atlassian Cloud domain; scoped token ignored.")
 
-        user_email = credentials.get("jira_user_email") or credentials.get("username")
+        user_email = (
+            credentials.get("jira_user_email")
+            or credentials.get("jira_username")
+        )
         api_token = credentials.get("jira_api_token") or credentials.get("token") or credentials.get("api_token")
         password = credentials.get("jira_password") or credentials.get("password")
         rest_api_version = credentials.get("rest_api_version")
@@ -377,16 +380,14 @@ class JiraConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSync
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
-        callback: Any = None,  # noqa: ARG002 - maintained for interface compatibility
+        callback: Any = None,  # noqa: ARG002 - callback interface hook
     ) -> Generator[list[SlimDocument], None, None]:
         """Return lightweight references to Jira issues (used for permission syncing)."""
         if not self.jira_client:
             raise ConnectorMissingCredentialError("Jira")
 
-        start_ts = start if start is not None else 0
-        end_ts = end if end is not None else datetime.now(timezone.utc).timestamp()
+        start_ts = 0
+        end_ts = datetime.now(timezone.utc).timestamp()
         jql = self._build_jql(start_ts, end_ts)
 
         checkpoint = self.build_dummy_checkpoint()
@@ -962,7 +963,16 @@ def main(config: dict[str, Any] | None = None) -> None:
 
     if not base_url:
         raise RuntimeError("Jira base URL must be provided via config or CLI arguments.")
-    if not (credentials.get("jira_api_token") or (credentials.get("jira_user_email") and credentials.get("jira_password"))):
+    if not (
+        credentials.get("jira_api_token")
+        or (
+            (
+                credentials.get("jira_user_email")
+                or credentials.get("jira_username")
+            )
+            and credentials.get("jira_password")
+        )
+    ):
         raise RuntimeError("Provide either an API token or both email/password for Jira authentication.")
 
     connector_options = {

common/data_source/notion_connector.py

@@ -28,9 +28,11 @@ from common.data_source.interfaces import (
 from common.data_source.models import (
     Document,
     GenerateDocumentsOutput,
+    GenerateSlimDocumentOutput,
     NotionBlock,
     NotionPage,
     NotionSearchResponse,
+    SlimDocument,
     TextSection,
 )
 from common.data_source.utils import (
@@ -433,6 +435,45 @@ class NotionConnector(LoadConnector, PollConnector):
 
         return result_blocks, child_pages, attachments
 
+    def _read_slim_blocks(self, base_block_id: str) -> tuple[list[str], list[str]]:
+        child_pages: list[str] = []
+        attachment_ids: list[str] = []
+        cursor = None
+
+        while True:
+            data = self._fetch_child_blocks(base_block_id, cursor)
+
+            if data is None:
+                return child_pages, attachment_ids
+
+            for result in data["results"]:
+                result_block_id = result["id"]
+                result_type = result["type"]
+
+                if result_type in {"file", "image", "pdf", "video", "audio"}:
+                    attachment_ids.append(result_block_id)
+
+                if result["has_children"]:
+                    if result_type == "child_page":
+                        child_pages.append(result_block_id)
+                    else:
+                        nested_child_pages, nested_attachment_ids = self._read_slim_blocks(
+                            result_block_id
+                        )
+                        child_pages.extend(nested_child_pages)
+                        attachment_ids.extend(nested_attachment_ids)
+
+                if result_type == "child_database" and self.recursive_index_enabled:
+                    _, inner_child_pages = self._read_pages_from_database(result_block_id)
+                    child_pages.extend(inner_child_pages)
+
+            if data["next_cursor"] is None:
+                break
+
+            cursor = data["next_cursor"]
+
+        return child_pages, attachment_ids
+
     def _read_page_title(self, page: NotionPage) -> Optional[str]:
         """Extracts the title from a Notion page."""
         if hasattr(page, "database_name") and page.database_name:
@@ -552,6 +593,79 @@ class NotionConnector(LoadConnector, PollConnector):
         pages = [self._fetch_page(page_id=self.root_page_id)]
         yield from batch_generator(self._read_pages(pages, start, end), self.batch_size)
 
+    def _read_pages_for_slim_docs(
+        self,
+        pages: list[NotionPage],
+        slim_indexed_pages: set[str],
+    ) -> Generator[SlimDocument, None, None]:
+        all_child_page_ids: list[str] = []
+
+        for page in pages:
+            if isinstance(page, dict):
+                page = NotionPage(**page)
+            if page.id in slim_indexed_pages:
+                continue
+
+            child_page_ids, attachment_ids = self._read_slim_blocks(page.id)
+            all_child_page_ids.extend(child_page_ids)
+            slim_indexed_pages.add(page.id)
+
+            yield SlimDocument(id=page.id)
+            for attachment_id in attachment_ids:
+                yield SlimDocument(id=attachment_id)
+
+        if self.recursive_index_enabled and all_child_page_ids:
+            for child_page_batch_ids in batch_generator(all_child_page_ids, INDEX_BATCH_SIZE):
+                child_page_batch = [
+                    self._fetch_page(page_id)
+                    for page_id in child_page_batch_ids
+                    if page_id not in slim_indexed_pages
+                ]
+                yield from self._read_pages_for_slim_docs(
+                    child_page_batch,
+                    slim_indexed_pages,
+                )
+
+    def retrieve_all_slim_docs_perm_sync(
+        self,
+        callback: Any = None,
+    ) -> GenerateSlimDocumentOutput:
+        slim_indexed_pages: set[str] = set()
+
+        if self.recursive_index_enabled and self.root_page_id:
+            root_pages = [self._fetch_page(page_id=self.root_page_id)]
+            yield from batch_generator(
+                self._read_pages_for_slim_docs(root_pages, slim_indexed_pages),
+                self.batch_size,
+            )
+            return
+
+        query_dict = {
+            "filter": {"property": "object", "value": "page"},
+            "page_size": 100,
+        }
+
+        slim_batch: list[SlimDocument] = []
+        while True:
+            db_res = self._search_notion(query_dict)
+            pages = [NotionPage(**page) for page in db_res.results]
+
+            for doc in self._read_pages_for_slim_docs(pages, slim_indexed_pages):
+                slim_batch.append(doc)
+                if len(slim_batch) >= self.batch_size:
+                    yield slim_batch
+                    slim_batch = []
+                    if callback:
+                        callback.progress("notion_slim_document", 1)
+
+            if db_res.has_more:
+                query_dict["start_cursor"] = db_res.next_cursor
+            else:
+                break
+
+        if slim_batch:
+            yield slim_batch
+
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         """Applies integration token to headers."""
         self.headers["Authorization"] = f"Bearer {credentials['notion_integration_token']}"
@@ -653,4 +767,4 @@ if __name__ == "__main__":
     document_batches = connector.load_from_state()
     for doc_batch in document_batches:
         for doc in doc_batch:
-            print(doc)
+            print(doc)

common/data_source/sharepoint_connector.py

@@ -112,10 +112,8 @@ class SharePointConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPe
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: Any = None,
     ) -> Any:
         """Retrieve all simplified documents with permission sync"""
         # Simplified implementation
-        return []
+        return []

common/data_source/slack_connector.py

@@ -528,8 +528,6 @@ class SlackConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: Any = None,
     ) -> GenerateSlimDocumentOutput:
         if self.client is None:
@@ -662,4 +660,4 @@ if __name__ == "__main__":
         connector.validate_connector_settings()
         print("Slack connector settings validated successfully")
     except Exception as e:
-        print(f"Validation failed: {e}")
+        print(f"Validation failed: {e}")

common/data_source/teams_connector.py

@@ -106,10 +106,8 @@ class TeamsConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSyn
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: Any = None,
     ) -> Any:
         """Retrieve all simplified documents with permission sync"""
         # Simplified implementation
-        return []
+        return []

common/data_source/zendesk_connector.py

@@ -553,15 +553,11 @@ class ZendeskConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         slim_doc_batch: list[SlimDocument] = []
         if self.content_type == "articles":
-            articles = _get_articles(
-                self.client, start_time=int(start) if start else None
-            )
+            articles = _get_articles(self.client)
             for article in articles:
                 slim_doc_batch.append(
                     SlimDocument(
@@ -572,9 +568,7 @@ class ZendeskConnector(
                     yield slim_doc_batch
                     slim_doc_batch = []
         elif self.content_type == "tickets":
-            tickets = _get_tickets(
-                self.client, start_time=int(start) if start else None
-            )
+            tickets = _get_tickets(self.client)
             for ticket in tickets:
                 slim_doc_batch.append(
                     SlimDocument(
@@ -664,4 +658,4 @@ if __name__ == "__main__":
 
         checkpoint = next_checkpoint
         if any_doc:
-            break
+            break