mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-31 21:13:49 +08:00
feat: implement SharePoint data source connector (#15190)
### What problem does this PR solve? Closes #15189. RAGFlow shipped a SharePoint connector stub (`common/data_source/sharepoint_connector.py`) whose document-loading methods all returned `[]`, `SharePoint._generate()` was a `pass`, and SharePoint was commented out of the data-source settings UI. As a result there was no way to index files stored in SharePoint document libraries. This PR implements the connector end to end on top of Microsoft Graph (Office365-REST-Python-Client). **Backend** - `common/data_source/sharepoint_connector.py` - `load_credentials()` now builds the Graph client using an MSAL client-credentials **token callback** — the form `GraphClient` actually expects. (The previous stub passed a raw access-token string to `GraphClient(...)`, which is not how that client is driven.) Token acquisition is lazy, so credential loading does no network call. - `validate_connector_settings()` resolves the configured site via Graph. - `load_from_checkpoint()` is now a generator that enumerates every document library under the site, walks folders depth-first, downloads each file, and yields blob-based `Document` objects (`extension` / `blob` / `size_bytes` / `doc_updated_at`). Incremental syncs are bounded by file `lastModifiedDateTime`. Per-file errors are surfaced as `ConnectorFailure` rather than aborting the run. - `retrieve_all_slim_docs_perm_sync()` yields id-only `SlimDocument` batches (no downloads) and the checkpoint helpers return proper checkpoints. - ACL → `ExternalAccess` mapping is intentionally left best-effort (`load_from_checkpoint_with_perm_sync` delegates to the standard load) because the sync pipeline does not currently persist `ExternalAccess`; this can be extended once that plumbing exists. - `rag/svr/sync_data_source.py` - Implemented `SharePoint._generate()` using the existing `CheckpointOutputWrapper` pattern (same shape as Confluence/Jira/Google Drive), supporting full reindex and incremental polling from `poll_range_start`. - `SharePointConnector` is already exported from `common/data_source/__init__.py`. **Frontend (`web/`)** - Enabled the `SHAREPOINT` data-source enum and added its form fields `site_url`, `tenant_id`, `client_id`, `client_secret`), default values, display metadata, and a SharePoint icon. - Added `sharepointDescription` / `sharepointSiteUrlTip` to `en.ts` and `zh.ts`. **Tests** - `test/unit_test/data_source/test_sharepoint_connector_unit.py`: mock-based unit tests covering credential loading (incomplete creds raise, happy path sets the Graph client, fetch-without-creds raises), drive traversal + file download, incremental `lastModifiedDateTime` filtering, and slim-doc listing. All 6 pass; `ruff check` is clean. ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -43,8 +43,8 @@ export enum DataSourceKey {
|
||||
POSTGRESQL = 'postgresql',
|
||||
REST_API = 'rest_api',
|
||||
RSS = 'rss',
|
||||
SHAREPOINT = 'sharepoint',
|
||||
|
||||
// SHAREPOINT = 'sharepoint',
|
||||
// SLACK = 'slack',
|
||||
// TEAMS = 'teams',
|
||||
}
|
||||
@@ -213,6 +213,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
|
||||
description: t(`setting.${DataSourceKey.MOODLE}Description`),
|
||||
icon: <SvgIcon name={'data-source/moodle'} width={38} />,
|
||||
},
|
||||
[DataSourceKey.SHAREPOINT]: {
|
||||
name: 'SharePoint',
|
||||
description: t(`setting.${DataSourceKey.SHAREPOINT}Description`),
|
||||
icon: <SvgIcon name={'data-source/sharepoint'} width={38} />,
|
||||
},
|
||||
[DataSourceKey.JIRA]: {
|
||||
name: 'Jira',
|
||||
description: t(`setting.${DataSourceKey.JIRA}Description`),
|
||||
@@ -654,6 +659,34 @@ export const DataSourceFormFields = {
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
[DataSourceKey.SHAREPOINT]: [
|
||||
{
|
||||
label: 'Site URL',
|
||||
name: 'config.credentials.site_url',
|
||||
type: FormFieldType.Text,
|
||||
required: true,
|
||||
placeholder: 'https://contoso.sharepoint.com/sites/MySite',
|
||||
tooltip: t('setting.sharepointSiteUrlTip'),
|
||||
},
|
||||
{
|
||||
label: 'Tenant ID',
|
||||
name: 'config.credentials.tenant_id',
|
||||
type: FormFieldType.Text,
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
label: 'Client ID',
|
||||
name: 'config.credentials.client_id',
|
||||
type: FormFieldType.Text,
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
label: 'Client Secret',
|
||||
name: 'config.credentials.client_secret',
|
||||
type: FormFieldType.Password,
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
[DataSourceKey.JIRA]: jiraConstant(t),
|
||||
[DataSourceKey.WEBDAV]: [
|
||||
{
|
||||
@@ -1509,6 +1542,18 @@ export const DataSourceFormDefaultValues = {
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.SHAREPOINT]: {
|
||||
name: '',
|
||||
source: DataSourceKey.SHAREPOINT,
|
||||
config: {
|
||||
credentials: {
|
||||
site_url: '',
|
||||
tenant_id: '',
|
||||
client_id: '',
|
||||
client_secret: '',
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.JIRA]: {
|
||||
name: '',
|
||||
source: DataSourceKey.JIRA,
|
||||
|
||||
Reference in New Issue
Block a user