feat: implement SharePoint data source connector (#15190)

### What problem does this PR solve? Closes #15189. RAGFlow shipped a SharePoint connector stub (`common/data_source/sharepoint_connector.py`) whose document-loading methods all returned `[]`, `SharePoint._generate()` was a `pass`, and SharePoint was commented out of the data-source settings UI. As a result there was no way to index files stored in SharePoint document libraries. This PR implements the connector end to end on top of Microsoft Graph (Office365-REST-Python-Client). **Backend** - `common/data_source/sharepoint_connector.py` - `load_credentials()` now builds the Graph client using an MSAL client-credentials **token callback** — the form `GraphClient` actually expects. (The previous stub passed a raw access-token string to `GraphClient(...)`, which is not how that client is driven.) Token acquisition is lazy, so credential loading does no network call. - `validate_connector_settings()` resolves the configured site via Graph. - `load_from_checkpoint()` is now a generator that enumerates every document library under the site, walks folders depth-first, downloads each file, and yields blob-based `Document` objects (`extension` / `blob` / `size_bytes` / `doc_updated_at`). Incremental syncs are bounded by file `lastModifiedDateTime`. Per-file errors are surfaced as `ConnectorFailure` rather than aborting the run. - `retrieve_all_slim_docs_perm_sync()` yields id-only `SlimDocument` batches (no downloads) and the checkpoint helpers return proper checkpoints. - ACL → `ExternalAccess` mapping is intentionally left best-effort (`load_from_checkpoint_with_perm_sync` delegates to the standard load) because the sync pipeline does not currently persist `ExternalAccess`; this can be extended once that plumbing exists. - `rag/svr/sync_data_source.py` - Implemented `SharePoint._generate()` using the existing `CheckpointOutputWrapper` pattern (same shape as Confluence/Jira/Google Drive), supporting full reindex and incremental polling from `poll_range_start`. - `SharePointConnector` is already exported from `common/data_source/__init__.py`. **Frontend (`web/`)** - Enabled the `SHAREPOINT` data-source enum and added its form fields `site_url`, `tenant_id`, `client_id`, `client_secret`), default values, display metadata, and a SharePoint icon. - Added `sharepointDescription` / `sharepointSiteUrlTip` to `en.ts` and `zh.ts`. **Tests** - `test/unit_test/data_source/test_sharepoint_connector_unit.py`: mock-based unit tests covering credential loading (incomplete creds raise, happy path sets the Graph client, fetch-without-creds raises), drive traversal + file download, incremental `lastModifiedDateTime` filtering, and slim-doc listing. All 6 pass; `ruff check` is clean. ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2026-07-20 22:51:06 +08:00 · 2026-05-27 23:26:08 -06:00
parent 0aff6a3f32
commit c4c4e228e3
7 changed files with 627 additions and 71 deletions
--- a/web/src/assets/svg/data-source/sharepoint.svg
+++ b/web/src/assets/svg/data-source/sharepoint.svg
@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" width="48" height="48">
+  <circle cx="20" cy="14" r="11" fill="#036C70"/>
+  <circle cx="31" cy="25" r="10" fill="#1A9BA1"/>
+  <circle cx="22.5" cy="36.5" r="8.5" fill="#37C6D0"/>
+  <path d="M22 11H10.5A1.5 1.5 0 0 0 9 12.5v18A1.5 1.5 0 0 0 10.5 32H22a1.5 1.5 0 0 0 1.5-1.5v-18A1.5 1.5 0 0 0 22 11z" opacity=".1"/>
+  <path d="M21 12H10.5A1.5 1.5 0 0 0 9 13.5v18A1.5 1.5 0 0 0 10.5 33H21a1.5 1.5 0 0 0 1.5-1.5v-18A1.5 1.5 0 0 0 21 12z" opacity=".2"/>
+  <rect x="2" y="14" width="22" height="22" rx="1.5" fill="#03787C"/>
+  <path d="M10.2 24.6c-.5-.3-.9-.7-1.2-1.1a2.6 2.6 0 0 1-.4-1.5c0-.8.3-1.5.9-2 .6-.5 1.4-.8 2.5-.8 1 0 1.8.1 2.5.4v1.9a4 4 0 0 0-2.3-.6c-.4 0-.8.1-1 .3-.3.1-.4.4-.4.7 0 .2.1.4.3.6.2.2.6.4 1.2.6.9.3 1.6.7 2 1.1.4.4.6 1 .6 1.6 0 .9-.3 1.5-.9 2-.6.5-1.5.7-2.6.7-.5 0-1-.1-1.5-.2a4 4 0 0 1-1.1-.4v-2c.4.3.8.5 1.3.7.5.2.9.2 1.3.2.5 0 .8-.1 1-.3.2-.1.3-.4.3-.7 0-.3-.1-.5-.4-.7-.2-.2-.7-.5-1.4-.8z" fill="#fff"/>
+</svg>
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1238,6 +1238,10 @@ Example: Virtual Hosted Style`,
        'Upload the OAuth JSON generated from Google Console. If it only contains client credentials, run the browser-based verification once to mint long-lived refresh tokens.',
      dropboxDescription:
        'Connect your Dropbox to sync files and folders from a chosen account.',
+      sharepointDescription:
+        'Connect a SharePoint site via Microsoft Graph to sync its document libraries.',
+      sharepointSiteUrlTip:
+        'Full URL of the SharePoint site to index, e.g. https://contoso.sharepoint.com/sites/MySite. Requires an Azure AD app with Sites.Read.All and Files.Read.All application permissions (admin consent).',
      bitbucketDescription: 'Connect Bitbucket to sync PR content.',
      bitbucketTopWorkspaceTip:
        'The Bitbucket workspace to index (e.g., "atlassian" from https://bitbucket.org/atlassian/workspace ).',
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1099,6 +1099,9 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
      gmailTokenTip:
        '请上传由 Google Console 生成的 OAuth JSON。如果仅包含 client credentials，请通过浏览器授权一次以获取长期有效的刷新 Token。',
      dropboxDescription: '连接 Dropbox，同步指定账号下的文件与文件夹。',
+      sharepointDescription: '通过 Microsoft Graph 连接 SharePoint 站点，同步其文档库。',
+      sharepointSiteUrlTip:
+        '要索引的 SharePoint 站点完整 URL，例如 https://contoso.sharepoint.com/sites/MySite。需要具备 Sites.Read.All 与 Files.Read.All 应用权限（管理员同意）的 Azure AD 应用。',
      boxDescription: '连接你的 Box 云盘以同步文件和文件夹。',
      bitbucketDescription: '连接 Bitbucket，同步 PR 内容。',
      bitbucketTopWorkspaceTip:
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -43,8 +43,8 @@ export enum DataSourceKey {
  POSTGRESQL = 'postgresql',
  REST_API = 'rest_api',
  RSS = 'rss',
+  SHAREPOINT = 'sharepoint',

-  //   SHAREPOINT = 'sharepoint',
  //   SLACK = 'slack',
  //   TEAMS = 'teams',
 }
@@ -213,6 +213,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
      description: t(`setting.${DataSourceKey.MOODLE}Description`),
      icon: <SvgIcon name={'data-source/moodle'} width={38} />,
    },
+    [DataSourceKey.SHAREPOINT]: {
+      name: 'SharePoint',
+      description: t(`setting.${DataSourceKey.SHAREPOINT}Description`),
+      icon: <SvgIcon name={'data-source/sharepoint'} width={38} />,
+    },
    [DataSourceKey.JIRA]: {
      name: 'Jira',
      description: t(`setting.${DataSourceKey.JIRA}Description`),
@@ -654,6 +659,34 @@ export const DataSourceFormFields = {
      required: true,
    },
  ],
+  [DataSourceKey.SHAREPOINT]: [
+    {
+      label: 'Site URL',
+      name: 'config.credentials.site_url',
+      type: FormFieldType.Text,
+      required: true,
+      placeholder: 'https://contoso.sharepoint.com/sites/MySite',
+      tooltip: t('setting.sharepointSiteUrlTip'),
+    },
+    {
+      label: 'Tenant ID',
+      name: 'config.credentials.tenant_id',
+      type: FormFieldType.Text,
+      required: true,
+    },
+    {
+      label: 'Client ID',
+      name: 'config.credentials.client_id',
+      type: FormFieldType.Text,
+      required: true,
+    },
+    {
+      label: 'Client Secret',
+      name: 'config.credentials.client_secret',
+      type: FormFieldType.Password,
+      required: true,
+    },
+  ],
  [DataSourceKey.JIRA]: jiraConstant(t),
  [DataSourceKey.WEBDAV]: [
    {
@@ -1509,6 +1542,18 @@ export const DataSourceFormDefaultValues = {
      },
    },
  },
+  [DataSourceKey.SHAREPOINT]: {
+    name: '',
+    source: DataSourceKey.SHAREPOINT,
+    config: {
+      credentials: {
+        site_url: '',
+        tenant_id: '',
+        client_id: '',
+        client_secret: '',
+      },
+    },
+  },
  [DataSourceKey.JIRA]: {
    name: '',
    source: DataSourceKey.JIRA,