feat: implement SharePoint data source connector (#15190)

### What problem does this PR solve?

Closes #15189.

RAGFlow shipped a SharePoint connector stub
(`common/data_source/sharepoint_connector.py`) whose document-loading
methods all returned `[]`, `SharePoint._generate()` was a `pass`, and
SharePoint was commented out of the data-source settings UI. As a result
there was no way to index files stored in SharePoint document libraries.

This PR implements the connector end to end on top of Microsoft Graph
(Office365-REST-Python-Client).

**Backend**

- `common/data_source/sharepoint_connector.py`
- `load_credentials()` now builds the Graph client using an MSAL
client-credentials **token callback** — the form `GraphClient` actually
expects. (The previous stub passed a raw access-token string to
`GraphClient(...)`, which is not how that client is driven.) Token
acquisition is lazy, so credential loading does no network call.
- `validate_connector_settings()` resolves the configured site via
Graph.
- `load_from_checkpoint()` is now a generator that enumerates every
document library under the site, walks folders depth-first, downloads
each file, and yields blob-based `Document` objects (`extension` /
`blob` / `size_bytes` / `doc_updated_at`). Incremental syncs are bounded
by file `lastModifiedDateTime`. Per-file errors are surfaced as
`ConnectorFailure` rather than aborting the run.
- `retrieve_all_slim_docs_perm_sync()` yields id-only `SlimDocument`
batches (no downloads) and the checkpoint helpers return proper
checkpoints.
- ACL → `ExternalAccess` mapping is intentionally left best-effort
(`load_from_checkpoint_with_perm_sync` delegates to the standard load)
because the sync pipeline does not currently persist `ExternalAccess`;
this can be extended once that plumbing exists.
- `rag/svr/sync_data_source.py`
- Implemented `SharePoint._generate()` using the existing
`CheckpointOutputWrapper` pattern (same shape as Confluence/Jira/Google
Drive), supporting full reindex and incremental polling from
`poll_range_start`.
- `SharePointConnector` is already exported from
`common/data_source/__init__.py`.

**Frontend (`web/`)**

- Enabled the `SHAREPOINT` data-source enum and added its form fields
`site_url`, `tenant_id`, `client_id`, `client_secret`), default values,
display metadata, and a SharePoint icon.
- Added `sharepointDescription` / `sharepointSiteUrlTip` to `en.ts` and
`zh.ts`.

**Tests**

- `test/unit_test/data_source/test_sharepoint_connector_unit.py`:
mock-based unit tests covering credential loading (incomplete creds
raise, happy path sets the Graph client, fetch-without-creds raises),
drive traversal + file download, incremental `lastModifiedDateTime`
filtering, and slim-doc listing. All 6 pass; `ruff check` is clean.

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
web-dev0521
2026-05-27 23:26:08 -06:00
committed by GitHub
parent 0aff6a3f32
commit c4c4e228e3
7 changed files with 627 additions and 71 deletions

View File

@@ -0,0 +1,284 @@
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import importlib.util
import sys
from datetime import datetime, timezone
from pathlib import Path
from types import ModuleType
import pytest
def _load_sharepoint_connector_module():
"""Load sharepoint_connector.py in isolation (avoid the package __init__)."""
repo_root = Path(__file__).resolve().parents[3]
package_name = "common.data_source"
saved_modules = {
name: module
for name, module in sys.modules.items()
if name == package_name or name.startswith(f"{package_name}.")
}
package_stub = ModuleType(package_name)
package_stub.__path__ = [str(repo_root / "common" / "data_source")]
sys.modules[package_name] = package_stub
try:
spec = importlib.util.spec_from_file_location(
"_sharepoint_connector_under_test",
repo_root / "common" / "data_source" / "sharepoint_connector.py",
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
finally:
for name in list(sys.modules):
if name == package_name or name.startswith(f"{package_name}."):
if name in saved_modules:
sys.modules[name] = saved_modules[name]
else:
sys.modules.pop(name, None)
sharepoint_connector = _load_sharepoint_connector_module()
SharePointConnector = sharepoint_connector.SharePointConnector
# --- fakes for the office365 fluent API ------------------------------------
class _Query:
"""Mimics the `.get()` / `.get_by_url()` -> `.execute_query()` chain."""
def __init__(self, value):
self._value = value
def execute_query(self):
return self._value
class _Content:
def __init__(self, value: bytes):
self.value = value
def execute_query(self):
return self
class _FakeDriveItem:
def __init__(self, item_id, name=None, content=None, modified=None, children=None, size=None):
self.id = item_id
self.name = name
self.web_url = f"https://contoso.sharepoint.com/{item_id}"
self.last_modified_datetime = modified
self._content = content
self._children = children or []
self.properties = {}
if children is not None:
self.properties["folder"] = {"childCount": len(children)}
else:
self.properties["file"] = {"mimeType": "text/plain"}
if size is not None:
self.properties["size"] = size
@property
def children(self):
return _FakeDrivesAccessor(self._children)
def get_content(self):
return _Content(self._content)
class _FakeDrive:
def __init__(self, name, root, drive_id=None):
self.name = name
self.root = root
self.id = drive_id or f"drive-{name}"
self.properties = {"name": name, "id": self.id}
class _FakeDrivesAccessor:
def __init__(self, drives):
self._drives = drives
def get(self):
return _Query(self._drives)
class _FakeSite:
def __init__(self, drives):
self.drives = _FakeDrivesAccessor(drives)
def __bool__(self):
return True
class _FakeSitesAccessor:
def __init__(self, site):
self._site = site
def get_by_url(self, url):
return _Query(self._site)
class _FakeGraphClient:
def __init__(self, site):
self.sites = _FakeSitesAccessor(site)
def _build_connector_with_tree():
jan = datetime(2026, 1, 1, 12, tzinfo=timezone.utc)
feb = datetime(2026, 2, 1, 12, tzinfo=timezone.utc)
readme = _FakeDriveItem("f1", "readme.txt", b"hello sharepoint", jan, size=16)
nested = _FakeDriveItem("f2", "report.md", b"# Report", feb, size=8)
subfolder = _FakeDriveItem("d2", "sub", children=[nested])
root = _FakeDriveItem("d1", "root", children=[readme, subfolder])
drive = _FakeDrive("Documents", root, drive_id="drv-A")
site = _FakeSite([drive])
connector = SharePointConnector(batch_size=10)
connector.graph_client = _FakeGraphClient(site)
connector._site_url = "https://contoso.sharepoint.com/sites/MySite"
return connector, jan, feb
# --- credential loading -----------------------------------------------------
def test_load_credentials_incomplete_raises():
connector = SharePointConnector()
with pytest.raises(sharepoint_connector.ConnectorMissingCredentialError):
connector.load_credentials({"tenant_id": "t", "client_id": "c"})
def test_load_credentials_sets_graph_client(monkeypatch):
captured = {}
class _FakeApp:
def __init__(self, **kwargs):
captured.update(kwargs)
def acquire_token_for_client(self, scopes):
return {"access_token": "tok"}
monkeypatch.setattr(sharepoint_connector.msal, "ConfidentialClientApplication", _FakeApp)
monkeypatch.setattr(sharepoint_connector, "GraphClient", lambda token_callback: ("client", token_callback))
connector = SharePointConnector()
result = connector.load_credentials(
{
"tenant_id": "tenant",
"client_id": "client",
"client_secret": "secret",
"site_url": "https://contoso.sharepoint.com/sites/MySite",
}
)
assert result is None
assert connector._site_url == "https://contoso.sharepoint.com/sites/MySite"
assert connector.graph_client is not None
def test_fetch_without_credentials_raises():
connector = SharePointConnector()
with pytest.raises(sharepoint_connector.ConnectorMissingCredentialError):
list(connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint()))
# --- document generation ----------------------------------------------------
def _collect(generator):
"""Drain a checkpoint generator, returning (documents, final_checkpoint)."""
docs = []
try:
while True:
docs.append(next(generator))
except StopIteration as stop:
return docs, stop.value
def test_load_from_checkpoint_walks_libraries_and_downloads():
connector, _jan, _feb = _build_connector_with_tree()
docs, checkpoint = _collect(
connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint())
)
assert checkpoint.has_more is False
assert {doc.id for doc in docs} == {"drv-A:f1", "drv-A:f2"}
by_id = {doc.id: doc for doc in docs}
assert by_id["drv-A:f1"].blob == b"hello sharepoint"
assert by_id["drv-A:f1"].extension == ".txt"
assert by_id["drv-A:f1"].size_bytes == 16
assert by_id["drv-A:f1"].source == "sharepoint"
assert by_id["drv-A:f1"].metadata["drive"] == "Documents"
assert by_id["drv-A:f1"].metadata["drive_id"] == "drv-A"
assert by_id["drv-A:f1"].metadata["drive_item_id"] == "f1"
assert by_id["drv-A:f2"].semantic_identifier == "report.md"
assert by_id["drv-A:f2"].extension == ".md"
def test_load_from_checkpoint_filters_by_modified_window():
connector, _jan, feb = _build_connector_with_tree()
# Only include files modified strictly after mid-January -> just report.md (Feb).
start = datetime(2026, 1, 15, tzinfo=timezone.utc).timestamp()
end = datetime(2026, 3, 1, tzinfo=timezone.utc).timestamp()
docs, _ = _collect(
connector.load_from_checkpoint(start, end, connector.build_dummy_checkpoint())
)
assert [doc.id for doc in docs] == ["drv-A:f2"]
def test_retrieve_all_slim_docs_lists_ids_without_download():
connector, _jan, _feb = _build_connector_with_tree()
batches = list(connector.retrieve_all_slim_docs_perm_sync())
ids = [doc.id for batch in batches for doc in batch]
assert sorted(ids) == ["drv-A:f1", "drv-A:f2"]
def test_document_ids_are_unique_across_drives_with_colliding_item_ids():
# Graph driveItem IDs are unique only within a single drive; two libraries
# under the same site can legitimately yield items with identical IDs.
jan = datetime(2026, 1, 1, 12, tzinfo=timezone.utc)
file_a = _FakeDriveItem("same-id", "a.txt", b"A", jan, size=1)
root_a = _FakeDriveItem("rootA", "root", children=[file_a])
drive_a = _FakeDrive("LibraryA", root_a, drive_id="drv-A")
file_b = _FakeDriveItem("same-id", "b.txt", b"B", jan, size=1)
root_b = _FakeDriveItem("rootB", "root", children=[file_b])
drive_b = _FakeDrive("LibraryB", root_b, drive_id="drv-B")
site = _FakeSite([drive_a, drive_b])
connector = SharePointConnector(batch_size=10)
connector.graph_client = _FakeGraphClient(site)
connector._site_url = "https://contoso.sharepoint.com/sites/MySite"
docs, _ = _collect(
connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint())
)
ids = {doc.id for doc in docs}
assert ids == {"drv-A:same-id", "drv-B:same-id"}
slim_ids = [doc.id for batch in connector.retrieve_all_slim_docs_perm_sync() for doc in batch]
assert sorted(slim_ids) == ["drv-A:same-id", "drv-B:same-id"]