feat(connectors): add Azure Blob Storage data source connector (#15466)

### What problem does this PR solve?

Closes #15465.

RAGFlow supports S3, Google Cloud Storage, R2, and OCI as data sources
but not Azure Blob Storage, leaving Azure users without a way to index
container objects into a knowledge base. This adds a first-class Azure
Blob Storage data-source connector — distinct from RAGFlow's existing
Azure storage *backends* (`rag/utils/azure_sas_conn.py`,
`rag/utils/azure_spn_conn.py`) which store RAGFlow's own files.

**Highlights**
- `common/data_source/azure_blob_connector.py`: new `AzureBlobConnector`
(`CheckpointedConnectorWithPermSync` + `SlimConnectorWithPermSync`).
- Uses the existing `azure-storage-blob` dependency (already in
`pyproject.toml`).
  - Three auth modes, tried in order of precedence:
1. **Account key** — `account_name` + `account_key` + `container_name`.
    2. **Connection string** — `connection_string` + `container_name`.
3. **SAS token** — `container_url` + `sas_token` (same shape as
`RAGFlowAzureSasBlob`).
- ETag fingerprint stored per blob in `AzureBlobCheckpoint.etags` —
unchanged blobs (same ETag as last run) are skipped without a download.
Only new/modified blobs are fetched.
  - Optional `prefix` scopes indexing to a virtual folder.
- `validate_connector_settings()` probes `get_container_properties()`
and maps `AuthenticationFailed / 403 / ContainerNotFound` to typed
connector exceptions.
  - Slim-doc IDs are blob names so prune reconciles correctly.
- `common/constants.py`, `common/data_source/config.py`,
`common/data_source/__init__.py`: register `azure_blob` in `FileSource`
/ `DocumentSource` and export `AzureBlobConnector`.
- `rag/svr/sync_data_source.py`: new `AzureBlob(SyncBase)` class routed
through `load_from_checkpoint` (ETag fingerprint owns change-detection)
and added to `func_factory`.
- Frontend:
- `web/src/pages/user-setting/data-source/constant/index.tsx`: new
`DataSourceKey.AZURE_BLOB`, auth-mode selector (account key / connection
string / SAS token), all credential fields, prefix + batch-size,
`syncDeletedFiles` capability, default form values, tile entry with
icon.
- `web/src/locales/{en,zh}.ts`: description + per-field tooltips for all
9 new keys.
- `web/src/assets/svg/data-source/azure-blob.svg`: Azure-branded
stacked-cylinders icon.

**Verification**
- `npm run build` (vite + esbuild) passes (37 s).

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
web-dev0521
2026-06-04 07:06:01 -06:00
committed by GitHub
parent a78a3fdd47
commit 98f2a2e60b
9 changed files with 692 additions and 0 deletions

View File

@@ -0,0 +1,17 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" width="48" height="48">
<!-- Azure blue gradient background circle -->
<defs>
<linearGradient id="azureGrad" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" style="stop-color:#0078D4"/>
<stop offset="100%" style="stop-color:#0050A0"/>
</linearGradient>
</defs>
<!-- Storage blob icon: stacked cylinders representing blob tiers -->
<ellipse cx="24" cy="14" rx="14" ry="5" fill="url(#azureGrad)"/>
<rect x="10" y="14" width="28" height="8" fill="#0078D4"/>
<ellipse cx="24" cy="22" rx="14" ry="5" fill="#50A0E0"/>
<rect x="10" y="22" width="28" height="8" fill="#4090D0"/>
<ellipse cx="24" cy="30" rx="14" ry="5" fill="#6AB0E8"/>
<rect x="10" y="30" width="28" height="5" fill="#5090C8"/>
<ellipse cx="24" cy="35" rx="14" ry="5" fill="#80C0F0"/>
</svg>

After

Width:  |  Height:  |  Size: 860 B

View File

@@ -1407,6 +1407,24 @@ Example: Virtual Hosted Style`,
'Mail folder to sync (e.g. inbox, sentitems, archive). Defaults to inbox.',
outlookUserIdsTip:
'Comma-separated UPNs or object IDs of mailboxes to sync. Leave blank to sync every mailbox in the tenant (requires User.Read.All).',
azure_blobDescription:
'Index blobs from an Azure Blob Storage container into a knowledge base. Supports account-key, connection-string, and SAS-token auth. Unchanged blobs are skipped via ETag fingerprinting.',
azureBlobAuthModeTip:
'Choose the authentication method. Account Key and Connection String require container_name; SAS Token requires container_url + sas_token.',
azureBlobAccountNameTip:
'Azure storage account name (e.g. mystorageaccount). Required for account-key auth.',
azureBlobAccountKeyTip:
'Storage account access key (Base64-encoded). Required for account-key auth.',
azureBlobConnectionStringTip:
'Full Azure Storage connection string (DefaultEndpointsProtocol=https;AccountName=...;...). Required for connection-string auth.',
azureBlobContainerUrlTip:
'Full HTTPS URL of the container (e.g. https://account.blob.core.windows.net/container). Required for SAS-token auth.',
azureBlobSasTokenTip:
'SAS query string (without the leading "?"). Required for SAS-token auth.',
azureBlobContainerNameTip:
'Name of the container to index. Required for account-key and connection-string auth.',
azureBlobPrefixTip:
'Optional blob name prefix to limit indexing to a virtual folder (e.g. documents/reports/). Leave blank to index the entire container.',
restApiQueryParamsTip:
'Key=value pairs (one per line) sent as URL query parameters. Use this instead of embedding params in the URL.',
restApiHeadersTip:

View File

@@ -1119,6 +1119,24 @@ NER使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
'要同步的邮件文件夹(例如 inbox、sentitems、archive默认为 inbox。',
outlookUserIdsTip:
'要同步的邮箱 UPN 或对象 ID 列表(逗号分隔)。留空则同步租户内的所有邮箱(需要 User.Read.All 权限)。',
azure_blobDescription:
'将 Azure Blob 存储容器中的文件索引到知识库。支持账户密钥、连接字符串和 SAS 令牌三种认证方式,通过 ETag 指纹跳过未变更的文件。',
azureBlobAuthModeTip:
'选择认证方式。账户密钥和连接字符串需填写容器名称SAS 令牌方式需填写容器 URL 和 SAS 令牌。',
azureBlobAccountNameTip:
'Azure 存储账户名称(例如 mystorageaccount账户密钥认证时必填。',
azureBlobAccountKeyTip:
'存储账户访问密钥Base64 编码),账户密钥认证时必填。',
azureBlobConnectionStringTip:
'完整的 Azure 存储连接字符串DefaultEndpointsProtocol=https;AccountName=...;...),连接字符串认证时必填。',
azureBlobContainerUrlTip:
'容器的完整 HTTPS 地址(例如 https://account.blob.core.windows.net/containerSAS 令牌认证时必填。',
azureBlobSasTokenTip:
'SAS 查询字符串(不含开头的"?"SAS 令牌认证时必填。',
azureBlobContainerNameTip:
'要索引的容器名称,账户密钥和连接字符串认证时必填。',
azureBlobPrefixTip:
'可选的 Blob 名称前缀,用于限定索引范围(例如 documents/reports/)。留空则索引整个容器。',
teamsDescription:
'通过 Microsoft Graph 连接 Microsoft Teams同步频道帖子与回复。',
teamsTenantIdTip:

View File

@@ -45,6 +45,7 @@ export enum DataSourceKey {
RSS = 'rss',
ONEDRIVE = 'onedrive',
OUTLOOK = 'outlook',
AZURE_BLOB = 'azure_blob',
TEAMS = 'teams',
SLACK = 'slack',
SHAREPOINT = 'sharepoint',
@@ -137,6 +138,9 @@ export const DataSourceFeatureVisibilityMap: Partial<
[DataSourceKey.OUTLOOK]: {
syncDeletedFiles: true,
},
[DataSourceKey.AZURE_BLOB]: {
syncDeletedFiles: true,
},
[DataSourceKey.TEAMS]: {
syncDeletedFiles: true,
},
@@ -335,6 +339,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
description: t(`setting.${DataSourceKey.OUTLOOK}Description`),
icon: <Mail className="text-text-primary" size={22} />,
},
[DataSourceKey.AZURE_BLOB]: {
name: 'Azure Blob Storage',
description: t(`setting.${DataSourceKey.AZURE_BLOB}Description`),
icon: <SvgIcon name={'data-source/azure-blob'} width={38} />,
},
};
};
@@ -515,6 +524,124 @@ export const DataSourceFormFields = {
},
},
],
[DataSourceKey.AZURE_BLOB]: [
{
label: 'Auth Mode',
name: 'config.auth_mode',
type: FormFieldType.Select,
required: true,
options: [
{ label: 'Account Key', value: 'account_key' },
{ label: 'Connection String', value: 'connection_string' },
{ label: 'SAS Token', value: 'sas_token' },
],
tooltip: t('setting.azureBlobAuthModeTip'),
},
{
label: 'Account Name',
name: 'config.credentials.account_name',
type: FormFieldType.Text,
required: false,
placeholder: 'mystorageaccount',
tooltip: t('setting.azureBlobAccountNameTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'account_key' && !(val ?? '').trim()
? 'Account name is required for account key auth'
: true,
},
{
label: 'Account Key',
name: 'config.credentials.account_key',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobAccountKeyTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'account_key' && !val
? 'Account key is required for account key auth'
: true,
},
{
label: 'Connection String',
name: 'config.credentials.connection_string',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobConnectionStringTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'connection_string',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'connection_string' && !val
? 'Connection string is required for connection string auth'
: true,
},
{
label: 'Container URL',
name: 'config.credentials.container_url',
type: FormFieldType.Text,
required: false,
placeholder: 'https://account.blob.core.windows.net/container',
tooltip: t('setting.azureBlobContainerUrlTip'),
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'sas_token' && !(val ?? '').trim()
? 'Container URL is required for SAS token auth'
: true,
},
{
label: 'SAS Token',
name: 'config.credentials.sas_token',
type: FormFieldType.Password,
required: false,
tooltip: t('setting.azureBlobSasTokenTip'),
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
customValidate: (val: string, values: any) =>
values?.config?.auth_mode === 'sas_token' && !val
? 'SAS token is required for SAS token auth'
: true,
},
{
label: 'Container Name',
name: 'config.credentials.container_name',
type: FormFieldType.Text,
required: false,
placeholder: 'my-container',
tooltip: t('setting.azureBlobContainerNameTip'),
shouldRender: (values: any) =>
values?.config?.auth_mode === 'account_key' ||
values?.config?.auth_mode === 'connection_string',
customValidate: (val: string, values: any) => {
const mode = values?.config?.auth_mode;
if (
(mode === 'account_key' || mode === 'connection_string') &&
!(val ?? '').trim()
) {
return 'Container name is required for this auth mode';
}
return true;
},
},
{
label: 'Prefix (optional)',
name: 'config.prefix',
type: FormFieldType.Text,
required: false,
placeholder: 'documents/reports/',
tooltip: t('setting.azureBlobPrefixTip'),
},
{
label: 'Batch Size',
name: 'config.batch_size',
type: FormFieldType.Number,
required: false,
validation: {
min: 1,
message: 'Batch Size must be at least 1',
},
},
],
[DataSourceKey.RSS]: [
{
label: 'Feed URL',
@@ -1982,6 +2109,23 @@ export const DataSourceFormDefaultValues = {
},
},
},
[DataSourceKey.AZURE_BLOB]: {
name: '',
source: DataSourceKey.AZURE_BLOB,
config: {
auth_mode: 'account_key',
prefix: '',
batch_size: 2,
credentials: {
account_name: '',
account_key: '',
connection_string: '',
container_url: '',
sas_token: '',
container_name: '',
},
},
},
[DataSourceKey.REST_API]: {
name: '',
source: DataSourceKey.REST_API,