feat(connectors): add Azure Blob Storage data source connector (#15466)

### What problem does this PR solve?

Closes #15465.

RAGFlow supports S3, Google Cloud Storage, R2, and OCI as data sources
but not Azure Blob Storage, leaving Azure users without a way to index
container objects into a knowledge base. This adds a first-class Azure
Blob Storage data-source connector — distinct from RAGFlow's existing
Azure storage *backends* (`rag/utils/azure_sas_conn.py`,
`rag/utils/azure_spn_conn.py`) which store RAGFlow's own files.

**Highlights**
- `common/data_source/azure_blob_connector.py`: new `AzureBlobConnector`
(`CheckpointedConnectorWithPermSync` + `SlimConnectorWithPermSync`).
- Uses the existing `azure-storage-blob` dependency (already in
`pyproject.toml`).
  - Three auth modes, tried in order of precedence:
1. **Account key** — `account_name` + `account_key` + `container_name`.
    2. **Connection string** — `connection_string` + `container_name`.
3. **SAS token** — `container_url` + `sas_token` (same shape as
`RAGFlowAzureSasBlob`).
- ETag fingerprint stored per blob in `AzureBlobCheckpoint.etags` —
unchanged blobs (same ETag as last run) are skipped without a download.
Only new/modified blobs are fetched.
  - Optional `prefix` scopes indexing to a virtual folder.
- `validate_connector_settings()` probes `get_container_properties()`
and maps `AuthenticationFailed / 403 / ContainerNotFound` to typed
connector exceptions.
  - Slim-doc IDs are blob names so prune reconciles correctly.
- `common/constants.py`, `common/data_source/config.py`,
`common/data_source/__init__.py`: register `azure_blob` in `FileSource`
/ `DocumentSource` and export `AzureBlobConnector`.
- `rag/svr/sync_data_source.py`: new `AzureBlob(SyncBase)` class routed
through `load_from_checkpoint` (ETag fingerprint owns change-detection)
and added to `func_factory`.
- Frontend:
- `web/src/pages/user-setting/data-source/constant/index.tsx`: new
`DataSourceKey.AZURE_BLOB`, auth-mode selector (account key / connection
string / SAS token), all credential fields, prefix + batch-size,
`syncDeletedFiles` capability, default form values, tile entry with
icon.
- `web/src/locales/{en,zh}.ts`: description + per-field tooltips for all
9 new keys.
- `web/src/assets/svg/data-source/azure-blob.svg`: Azure-branded
stacked-cylinders icon.

**Verification**
- `npm run build` (vite + esbuild) passes (37 s).

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
web-dev0521
2026-06-04 07:06:01 -06:00
committed by GitHub
parent a78a3fdd47
commit 98f2a2e60b
9 changed files with 692 additions and 0 deletions

View File

@@ -45,6 +45,7 @@ export enum DataSourceKey {
RSS = 'rss',
ONEDRIVE = 'onedrive',
OUTLOOK = 'outlook',
AZURE_BLOB = 'azure_blob',
TEAMS = 'teams',
SLACK = 'slack',
SHAREPOINT = 'sharepoint',
@@ -137,6 +138,9 @@ export const DataSourceFeatureVisibilityMap: Partial<
[DataSourceKey.OUTLOOK]: {
syncDeletedFiles: true,
},
[DataSourceKey.AZURE_BLOB]: {
syncDeletedFiles: true,
},
[DataSourceKey.TEAMS]: {
syncDeletedFiles: true,
},
@@ -335,6 +339,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
description: t(`setting.${DataSourceKey.OUTLOOK}Description`),
icon: <Mail className="text-text-primary" size={22} />,
},
[DataSourceKey.AZURE_BLOB]: {
name: 'Azure Blob Storage',
description: t(`setting.${DataSourceKey.AZURE_BLOB}Description`),
icon: <SvgIcon name={'data-source/azure-blob'} width={38} />,
},
};
};
@@ -515,6 +524,124 @@ export const DataSourceFormFields = {
},
},
],
[DataSourceKey.AZURE_BLOB]: [
{
label: 'Auth Mode',
name: 'config.auth_mode',
type: FormFieldType.Select,
required: true,
options: [
{ label: 'Account Key', value: 'account_key' },
{ label: 'Connection String', value: 'connection_string' },
{ label: 'SAS Token', value: 'sas_token' },
],
tooltip: t('setting.azureBlobAuthModeTip'),
},
{
label: 'Account Name',
name: 'config.credentials.account_name',
type: FormFieldType.Text,
required: false,
placeholder: 'mystorageaccount',
tooltip: t('setting.azureBlobAccountNameTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'account_key' && !(val ?? '').trim()
? 'Account name is required for account key auth'
: true,
},
{
label: 'Account Key',
name: 'config.credentials.account_key',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobAccountKeyTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'account_key' && !val
? 'Account key is required for account key auth'
: true,
},
{
label: 'Connection String',
name: 'config.credentials.connection_string',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobConnectionStringTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'connection_string',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'connection_string' && !val
? 'Connection string is required for connection string auth'
: true,
},
{
label: 'Container URL',
name: 'config.credentials.container_url',
type: FormFieldType.Text,
required: false,
placeholder: 'https://account.blob.core.windows.net/container',
tooltip: t('setting.azureBlobContainerUrlTip'),
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'sas_token' && !(val ?? '').trim()
? 'Container URL is required for SAS token auth'
: true,
},
{
label: 'SAS Token',
name: 'config.credentials.sas_token',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobSasTokenTip'),
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'sas_token' && !val
? 'SAS token is required for SAS token auth'
: true,
},
{
label: 'Container Name',
name: 'config.credentials.container_name',
type: FormFieldType.Text,
required: false,
placeholder: 'my-container',
tooltip: t('setting.azureBlobContainerNameTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key' ||
values?.config?.auth_mode === 'connection_string',
customValidate: (val: string, values: any) => {
const mode = values?.config?.auth_mode;
if (
(mode === 'account_key' || mode === 'connection_string') &&
!(val ?? '').trim()
) {
return 'Container name is required for this auth mode';
}
return true;
},
},
{
label: 'Prefix (optional)',
name: 'config.prefix',
type: FormFieldType.Text,
required: false,
placeholder: 'documents/reports/',
tooltip: t('setting.azureBlobPrefixTip'),
},
{
label: 'Batch Size',
name: 'config.batch_size',
type: FormFieldType.Number,
required: false,
validation: {
min: 1,
message: 'Batch Size must be at least 1',
},
},
],
[DataSourceKey.RSS]: [
{
label: 'Feed URL',
@@ -1982,6 +2109,23 @@ export const DataSourceFormDefaultValues = {
},
},
},
[DataSourceKey.AZURE_BLOB]: {
name: '',
source: DataSourceKey.AZURE_BLOB,
config: {
auth_mode: 'account_key',
prefix: '',
batch_size: 2,
credentials: {
account_name: '',
account_key: '',
connection_string: '',
container_url: '',
sas_token: '',
container_name: '',
},
},
},
[DataSourceKey.REST_API]: {
name: '',
source: DataSourceKey.REST_API,