mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 07:10:29 +08:00
feat(connectors): add Azure Blob Storage data source connector (#15466)
### What problem does this PR solve? Closes #15465. RAGFlow supports S3, Google Cloud Storage, R2, and OCI as data sources but not Azure Blob Storage, leaving Azure users without a way to index container objects into a knowledge base. This adds a first-class Azure Blob Storage data-source connector — distinct from RAGFlow's existing Azure storage *backends* (`rag/utils/azure_sas_conn.py`, `rag/utils/azure_spn_conn.py`) which store RAGFlow's own files. **Highlights** - `common/data_source/azure_blob_connector.py`: new `AzureBlobConnector` (`CheckpointedConnectorWithPermSync` + `SlimConnectorWithPermSync`). - Uses the existing `azure-storage-blob` dependency (already in `pyproject.toml`). - Three auth modes, tried in order of precedence: 1. **Account key** — `account_name` + `account_key` + `container_name`. 2. **Connection string** — `connection_string` + `container_name`. 3. **SAS token** — `container_url` + `sas_token` (same shape as `RAGFlowAzureSasBlob`). - ETag fingerprint stored per blob in `AzureBlobCheckpoint.etags` — unchanged blobs (same ETag as last run) are skipped without a download. Only new/modified blobs are fetched. - Optional `prefix` scopes indexing to a virtual folder. - `validate_connector_settings()` probes `get_container_properties()` and maps `AuthenticationFailed / 403 / ContainerNotFound` to typed connector exceptions. - Slim-doc IDs are blob names so prune reconciles correctly. - `common/constants.py`, `common/data_source/config.py`, `common/data_source/__init__.py`: register `azure_blob` in `FileSource` / `DocumentSource` and export `AzureBlobConnector`. - `rag/svr/sync_data_source.py`: new `AzureBlob(SyncBase)` class routed through `load_from_checkpoint` (ETag fingerprint owns change-detection) and added to `func_factory`. - Frontend: - `web/src/pages/user-setting/data-source/constant/index.tsx`: new `DataSourceKey.AZURE_BLOB`, auth-mode selector (account key / connection string / SAS token), all credential fields, prefix + batch-size, `syncDeletedFiles` capability, default form values, tile entry with icon. - `web/src/locales/{en,zh}.ts`: description + per-field tooltips for all 9 new keys. - `web/src/assets/svg/data-source/azure-blob.svg`: Azure-branded stacked-cylinders icon. **Verification** - `npm run build` (vite + esbuild) passes (37 s). ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -45,6 +45,7 @@ export enum DataSourceKey {
|
||||
RSS = 'rss',
|
||||
ONEDRIVE = 'onedrive',
|
||||
OUTLOOK = 'outlook',
|
||||
AZURE_BLOB = 'azure_blob',
|
||||
TEAMS = 'teams',
|
||||
SLACK = 'slack',
|
||||
SHAREPOINT = 'sharepoint',
|
||||
@@ -137,6 +138,9 @@ export const DataSourceFeatureVisibilityMap: Partial<
|
||||
[DataSourceKey.OUTLOOK]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
[DataSourceKey.TEAMS]: {
|
||||
syncDeletedFiles: true,
|
||||
},
|
||||
@@ -335,6 +339,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
|
||||
description: t(`setting.${DataSourceKey.OUTLOOK}Description`),
|
||||
icon: <Mail className="text-text-primary" size={22} />,
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
name: 'Azure Blob Storage',
|
||||
description: t(`setting.${DataSourceKey.AZURE_BLOB}Description`),
|
||||
icon: <SvgIcon name={'data-source/azure-blob'} width={38} />,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -515,6 +524,124 @@ export const DataSourceFormFields = {
|
||||
},
|
||||
},
|
||||
],
|
||||
[DataSourceKey.AZURE_BLOB]: [
|
||||
{
|
||||
label: 'Auth Mode',
|
||||
name: 'config.auth_mode',
|
||||
type: FormFieldType.Select,
|
||||
required: true,
|
||||
options: [
|
||||
{ label: 'Account Key', value: 'account_key' },
|
||||
{ label: 'Connection String', value: 'connection_string' },
|
||||
{ label: 'SAS Token', value: 'sas_token' },
|
||||
],
|
||||
tooltip: t('setting.azureBlobAuthModeTip'),
|
||||
},
|
||||
{
|
||||
label: 'Account Name',
|
||||
name: 'config.credentials.account_name',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'mystorageaccount',
|
||||
tooltip: t('setting.azureBlobAccountNameTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' && !(val ?? '').trim()
|
||||
? 'Account name is required for account key auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Account Key',
|
||||
name: 'config.credentials.account_key',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobAccountKeyTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' && !val
|
||||
? 'Account key is required for account key auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Connection String',
|
||||
name: 'config.credentials.connection_string',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobConnectionStringTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'connection_string',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'connection_string' && !val
|
||||
? 'Connection string is required for connection string auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Container URL',
|
||||
name: 'config.credentials.container_url',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'https://account.blob.core.windows.net/container',
|
||||
tooltip: t('setting.azureBlobContainerUrlTip'),
|
||||
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'sas_token' && !(val ?? '').trim()
|
||||
? 'Container URL is required for SAS token auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'SAS Token',
|
||||
name: 'config.credentials.sas_token',
|
||||
type: FormFieldType.Password,
|
||||
required: false,
|
||||
tooltip: t('setting.azureBlobSasTokenTip'),
|
||||
shouldRender: (values: any) => values?.config?.auth_mode === 'sas_token',
|
||||
customValidate: (val: string, values: any) =>
|
||||
values?.config?.auth_mode === 'sas_token' && !val
|
||||
? 'SAS token is required for SAS token auth'
|
||||
: true,
|
||||
},
|
||||
{
|
||||
label: 'Container Name',
|
||||
name: 'config.credentials.container_name',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'my-container',
|
||||
tooltip: t('setting.azureBlobContainerNameTip'),
|
||||
shouldRender: (values: any) =>
|
||||
values?.config?.auth_mode === 'account_key' ||
|
||||
values?.config?.auth_mode === 'connection_string',
|
||||
customValidate: (val: string, values: any) => {
|
||||
const mode = values?.config?.auth_mode;
|
||||
if (
|
||||
(mode === 'account_key' || mode === 'connection_string') &&
|
||||
!(val ?? '').trim()
|
||||
) {
|
||||
return 'Container name is required for this auth mode';
|
||||
}
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{
|
||||
label: 'Prefix (optional)',
|
||||
name: 'config.prefix',
|
||||
type: FormFieldType.Text,
|
||||
required: false,
|
||||
placeholder: 'documents/reports/',
|
||||
tooltip: t('setting.azureBlobPrefixTip'),
|
||||
},
|
||||
{
|
||||
label: 'Batch Size',
|
||||
name: 'config.batch_size',
|
||||
type: FormFieldType.Number,
|
||||
required: false,
|
||||
validation: {
|
||||
min: 1,
|
||||
message: 'Batch Size must be at least 1',
|
||||
},
|
||||
},
|
||||
],
|
||||
[DataSourceKey.RSS]: [
|
||||
{
|
||||
label: 'Feed URL',
|
||||
@@ -1982,6 +2109,23 @@ export const DataSourceFormDefaultValues = {
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.AZURE_BLOB]: {
|
||||
name: '',
|
||||
source: DataSourceKey.AZURE_BLOB,
|
||||
config: {
|
||||
auth_mode: 'account_key',
|
||||
prefix: '',
|
||||
batch_size: 2,
|
||||
credentials: {
|
||||
account_name: '',
|
||||
account_key: '',
|
||||
connection_string: '',
|
||||
container_url: '',
|
||||
sas_token: '',
|
||||
container_name: '',
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.REST_API]: {
|
||||
name: '',
|
||||
source: DataSourceKey.REST_API,
|
||||
|
||||
Reference in New Issue
Block a user