mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-04 23:00:30 +08:00
feat(connectors): add Azure Blob Storage data source connector (#15466)
### What problem does this PR solve? Closes #15465. RAGFlow supports S3, Google Cloud Storage, R2, and OCI as data sources but not Azure Blob Storage, leaving Azure users without a way to index container objects into a knowledge base. This adds a first-class Azure Blob Storage data-source connector — distinct from RAGFlow's existing Azure storage *backends* (`rag/utils/azure_sas_conn.py`, `rag/utils/azure_spn_conn.py`) which store RAGFlow's own files. **Highlights** - `common/data_source/azure_blob_connector.py`: new `AzureBlobConnector` (`CheckpointedConnectorWithPermSync` + `SlimConnectorWithPermSync`). - Uses the existing `azure-storage-blob` dependency (already in `pyproject.toml`). - Three auth modes, tried in order of precedence: 1. **Account key** — `account_name` + `account_key` + `container_name`. 2. **Connection string** — `connection_string` + `container_name`. 3. **SAS token** — `container_url` + `sas_token` (same shape as `RAGFlowAzureSasBlob`). - ETag fingerprint stored per blob in `AzureBlobCheckpoint.etags` — unchanged blobs (same ETag as last run) are skipped without a download. Only new/modified blobs are fetched. - Optional `prefix` scopes indexing to a virtual folder. - `validate_connector_settings()` probes `get_container_properties()` and maps `AuthenticationFailed / 403 / ContainerNotFound` to typed connector exceptions. - Slim-doc IDs are blob names so prune reconciles correctly. - `common/constants.py`, `common/data_source/config.py`, `common/data_source/__init__.py`: register `azure_blob` in `FileSource` / `DocumentSource` and export `AzureBlobConnector`. - `rag/svr/sync_data_source.py`: new `AzureBlob(SyncBase)` class routed through `load_from_checkpoint` (ETag fingerprint owns change-detection) and added to `func_factory`. - Frontend: - `web/src/pages/user-setting/data-source/constant/index.tsx`: new `DataSourceKey.AZURE_BLOB`, auth-mode selector (account key / connection string / SAS token), all credential fields, prefix + batch-size, `syncDeletedFiles` capability, default form values, tile entry with icon. - `web/src/locales/{en,zh}.ts`: description + per-field tooltips for all 9 new keys. - `web/src/assets/svg/data-source/azure-blob.svg`: Azure-branded stacked-cylinders icon. **Verification** - `npm run build` (vite + esbuild) passes (37 s). ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
17
web/src/assets/svg/data-source/azure-blob.svg
Normal file
17
web/src/assets/svg/data-source/azure-blob.svg
Normal file
@@ -0,0 +1,17 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" width="48" height="48">
|
||||
<!-- Azure blue gradient background circle -->
|
||||
<defs>
|
||||
<linearGradient id="azureGrad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#0078D4"/>
|
||||
<stop offset="100%" style="stop-color:#0050A0"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<!-- Storage blob icon: stacked cylinders representing blob tiers -->
|
||||
<ellipse cx="24" cy="14" rx="14" ry="5" fill="url(#azureGrad)"/>
|
||||
<rect x="10" y="14" width="28" height="8" fill="#0078D4"/>
|
||||
<ellipse cx="24" cy="22" rx="14" ry="5" fill="#50A0E0"/>
|
||||
<rect x="10" y="22" width="28" height="8" fill="#4090D0"/>
|
||||
<ellipse cx="24" cy="30" rx="14" ry="5" fill="#6AB0E8"/>
|
||||
<rect x="10" y="30" width="28" height="5" fill="#5090C8"/>
|
||||
<ellipse cx="24" cy="35" rx="14" ry="5" fill="#80C0F0"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 860 B |
@@ -1407,6 +1407,24 @@ Example: Virtual Hosted Style`,
|
||||
'Mail folder to sync (e.g. inbox, sentitems, archive). Defaults to inbox.',
|
||||
outlookUserIdsTip:
|
||||
'Comma-separated UPNs or object IDs of mailboxes to sync. Leave blank to sync every mailbox in the tenant (requires User.Read.All).',
|
||||
azure_blobDescription:
|
||||
'Index blobs from an Azure Blob Storage container into a knowledge base. Supports account-key, connection-string, and SAS-token auth. Unchanged blobs are skipped via ETag fingerprinting.',
|
||||
azureBlobAuthModeTip:
|
||||
'Choose the authentication method. Account Key and Connection String require container_name; SAS Token requires container_url + sas_token.',
|
||||
azureBlobAccountNameTip:
|
||||
'Azure storage account name (e.g. mystorageaccount). Required for account-key auth.',
|
||||
azureBlobAccountKeyTip:
|
||||
'Storage account access key (Base64-encoded). Required for account-key auth.',
|
||||
azureBlobConnectionStringTip:
|
||||
'Full Azure Storage connection string (DefaultEndpointsProtocol=https;AccountName=...;...). Required for connection-string auth.',
|
||||
azureBlobContainerUrlTip:
|
||||
'Full HTTPS URL of the container (e.g. https://account.blob.core.windows.net/container). Required for SAS-token auth.',
|
||||
azureBlobSasTokenTip:
|
||||
'SAS query string (without the leading "?"). Required for SAS-token auth.',
|
||||
azureBlobContainerNameTip:
|
||||
'Name of the container to index. Required for account-key and connection-string auth.',
|
||||
azureBlobPrefixTip:
|
||||
'Optional blob name prefix to limit indexing to a virtual folder (e.g. documents/reports/). Leave blank to index the entire container.',
|
||||
restApiQueryParamsTip:
|
||||
'Key=value pairs (one per line) sent as URL query parameters. Use this instead of embedding params in the URL.',
|
||||
restApiHeadersTip:
|
||||
|
||||
@@ -1119,6 +1119,24 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
|
||||
'要同步的邮件文件夹(例如 inbox、sentitems、archive),默认为 inbox。',
|
||||
outlookUserIdsTip:
|
||||
'要同步的邮箱 UPN 或对象 ID 列表(逗号分隔)。留空则同步租户内的所有邮箱(需要 User.Read.All 权限)。',
|
||||
azure_blobDescription:
|
||||
'将 Azure Blob 存储容器中的文件索引到知识库。支持账户密钥、连接字符串和 SAS 令牌三种认证方式,通过 ETag 指纹跳过未变更的文件。',
|
||||
azureBlobAuthModeTip:
|
||||
'选择认证方式。账户密钥和连接字符串需填写容器名称;SAS 令牌方式需填写容器 URL 和 SAS 令牌。',
|
||||
azureBlobAccountNameTip:
|
||||
'Azure 存储账户名称(例如 mystorageaccount),账户密钥认证时必填。',
|
||||
azureBlobAccountKeyTip:
|
||||
'存储账户访问密钥(Base64 编码),账户密钥认证时必填。',
|
||||
azureBlobConnectionStringTip:
|
||||
'完整的 Azure 存储连接字符串(DefaultEndpointsProtocol=https;AccountName=...;...),连接字符串认证时必填。',
|
||||
azureBlobContainerUrlTip:
|
||||
'容器的完整 HTTPS 地址(例如 https://account.blob.core.windows.net/container),SAS 令牌认证时必填。',
|
||||
azureBlobSasTokenTip:
|
||||
'SAS 查询字符串(不含开头的"?"),SAS 令牌认证时必填。',
|
||||
azureBlobContainerNameTip:
|
||||
'要索引的容器名称,账户密钥和连接字符串认证时必填。',
|
||||
azureBlobPrefixTip:
|
||||
'可选的 Blob 名称前缀,用于限定索引范围(例如 documents/reports/)。留空则索引整个容器。',
|
||||
teamsDescription:
|
||||
'通过 Microsoft Graph 连接 Microsoft Teams,同步频道帖子与回复。',
|
||||
teamsTenantIdTip:
|
||||
|
||||
@@ -45,6 +45,7 @@ export enum DataSourceKey {
|
||||
RSS = 'rss',
|
||||
ONEDRIVE = 'onedrive',
|
||||
OUTLOOK = 'outlook',
|
||||
AZURE_BLOB = 'azure_blob',
|
||||
TEAMS = 'teams',
|
||||
SLACK = 'slack',
|
||||
SHAREPOINT = 'sharepoint',
|
||||
@@ -137,6 +138,9 @@ export const DataSourceFeatureVisibilityMap: Partial<
|
||||
[DataSourceKey.OUTLOOK]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
[DataSourceKey.TEAMS]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
@@ -335,6 +339,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
|
||||
description: t(`setting.${DataSourceKey.OUTLOOK}Description`),
|
||||
icon: <Mail className="text-text-primary" size={22} />,
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
name: 'Azure Blob Storage',
|
||||
description: t(`setting.${DataSourceKey.AZURE_BLOB}Description`),
|
||||
icon: <SvgIcon name={'data-source/azure-blob'} width={38} />,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -515,6 +524,124 @@ export const DataSourceFormFields = {
|
||||
},
|
||||
},
|
||||
],
|
||||
[DataSourceKey.AZURE_BLOB]: [
|
||||
{
|
||||
label: 'Auth Mode',
|
||||
name: 'config.auth_mode',
|
||||
type: FormFieldType.Select,
|
||||
required: true,
|
||||
options: [
|
||||
{ label: 'Account Key', value: 'account_key' },
|
||||
{ label: 'Connection String', value: 'connection_string' },
|
||||
{ label: 'SAS Token', value: 'sas_token' },
|
||||
],
|
||||
tooltip: t('setting.azureBlobAuthModeTip'),
|
||||
},
|
||||
{
|
||||
label: 'Account Name',
|
||||
name: 'config.credentials.account_name',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'mystorageaccount',
|
||||
tooltip: t('setting.azureBlobAccountNameTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' && !(val ?? '').trim()
|
||||
? 'Account name is required for account key auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Account Key',
|
||||
name: 'config.credentials.account_key',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobAccountKeyTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' && !val
|
||||
? 'Account key is required for account key auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Connection String',
|
||||
name: 'config.credentials.connection_string',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobConnectionStringTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'connection_string',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'connection_string' && !val
|
||||
? 'Connection string is required for connection string auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Container URL',
|
||||
name: 'config.credentials.container_url',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'https://account.blob.core.windows.net/container',
|
||||
tooltip: t('setting.azureBlobContainerUrlTip'),
|
||||
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'sas_token' && !(val ?? '').trim()
|
||||
? 'Container URL is required for SAS token auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'SAS Token',
|
||||
name: 'config.credentials.sas_token',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobSasTokenTip'),
|
||||
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'sas_token' && !val
|
||||
? 'SAS token is required for SAS token auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Container Name',
|
||||
name: 'config.credentials.container_name',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'my-container',
|
||||
tooltip: t('setting.azureBlobContainerNameTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' ||
|
||||
values?.config?.auth_mode === 'connection_string',
|
||||
customValidate: (val: string, values: any) => {
|
||||
const mode = values?.config?.auth_mode;
|
||||
if (
|
||||
(mode === 'account_key' || mode === 'connection_string') &&
|
||||
!(val ?? '').trim()
|
||||
) {
|
||||
return 'Container name is required for this auth mode';
|
||||
}
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{
|
||||
label: 'Prefix (optional)',
|
||||
name: 'config.prefix',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'documents/reports/',
|
||||
tooltip: t('setting.azureBlobPrefixTip'),
|
||||
},
|
||||
{
|
||||
label: 'Batch Size',
|
||||
name: 'config.batch_size',
|
||||
type: FormFieldType.Number,
|
||||
required: false,
|
||||
validation: {
|
||||
min: 1,
|
||||
message: 'Batch Size must be at least 1',
|
||||
},
|
||||
},
|
||||
],
|
||||
[DataSourceKey.RSS]: [
|
||||
{
|
||||
label: 'Feed URL',
|
||||
@@ -1982,6 +2109,23 @@ export const DataSourceFormDefaultValues = {
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
name: '',
|
||||
source: DataSourceKey.AZURE_BLOB,
|
||||
config: {
|
||||
auth_mode: 'account_key',
|
||||
prefix: '',
|
||||
batch_size: 2,
|
||||
credentials: {
|
||||
account_name: '',
|
||||
account_key: '',
|
||||
connection_string: '',
|
||||
container_url: '',
|
||||
sas_token: '',
|
||||
container_name: '',
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.REST_API]: {
|
||||
name: '',
|
||||
source: DataSourceKey.REST_API,
|
||||
|
||||
Reference in New Issue
Block a user