mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-23 08:56:42 +08:00
# feat: Add Generic REST API Connector
## What problem does this PR solve?
RAGFlow supports many specific data source connectors (MySQL, Slack,
Google Drive, etc.), but there was no way to connect an arbitrary REST
API as a data source. Users with custom or third-party APIs had to write
a new connector class for each one.
This PR adds a **generic, configuration-driven REST API connector** that
lets users connect any REST API as a data source entirely through the UI
— no code changes needed per API.
---
## Features
### Core Connector (`common/data_source/rest_api_connector.py`)
- Implements `LoadConnector` and `PollConnector` interfaces for full and
incremental sync
- **Configurable authentication:** None, API Key (custom header), Bearer
Token, Basic Auth
- **Pluggable pagination:** Page-based, Offset-based, Cursor-based, or
None
- Smart page-size inference from user's query parameters to avoid
duplicate/conflicting params
- Configurable request delay between pages to prevent API rate limiting
- Auto-detection of the items array in JSON responses (`items`,
`results`, `data`, `records`, or first list found)
- **Advanced field mapping** with dot-notation (`country.name`), array
wildcards (`newsType[*].name`), type hints, and default values
- Optional content template rendering (`"Title: {title}\nBody: {body}"`)
- HTML stripping for content fields
- Stable document IDs via `hash128` from a configurable ID field or
auto-generated from item content
- Pydantic configuration schema with automatic coercion of UI string
inputs to dicts/lists
### Backend Registration (`rag/svr/sync_data_source.py`,
`common/constants.py`, `common/data_source/config.py`)
- `REST_API` sync class wired into RAGFlow's `func_factory`
- Full sync (`load_from_state`) and incremental polling (`poll_source`)
support
- Credentials and config passed from task to connector following
existing patterns (MySQL, SeaFile, etc.)
### Test Connection Endpoint (`api/apps/connector_app.py`)
- `POST /v1/connector/<id>/test` validates config schema,
authentication, and API connectivity without triggering a sync
- Clear error messages for auth failures vs. config issues
### Frontend UI (`web/src/pages/user-setting/data-source/constant/`)
- **Postman-style configuration:** Base URL, Query Parameters (key=value
per line), Auth, Content Fields, Metadata Fields, Pagination Type
- Auth-type-aware form: fields for API key header/value, Bearer token,
or Basic username/password appear only when relevant
- **Advanced Settings** toggle for: Custom Headers, Max Pages, Request
Delay, Poll Timestamp Field, Request Body (POST)
- Connector icon (SVG) and i18n strings (English)
- **"Test Connection"** button to validate before syncing
---
## Controls & Safety
- Configurable max pages safety cap (default: 1000, adjustable in UI)
- Configurable request delay between pages (default: 0.5s, adjustable in
UI)
- Auth errors (401/403) fail immediately without retries; transient
errors retry with exponential backoff
- Diagnostic logging: auth setup confirmation, request details on
failure, content field extraction status
---
## Type of change
- [x] New Feature (non-breaking change which adds functionality)
##Visual Screenshots of Features
<img width="482" height="510" alt="Screenshot 2026-03-11 at 5 19 52 PM"
src="https://github.com/user-attachments/assets/dcb7ab4a-1622-44f3-bb02-d6f0527314c4"
/>
(Connector can be configured within the external data sources tab)
Configuration Parameters:
<img width="661" height="682" alt="Screenshot 2026-03-11 at 5 20 46 PM"
src="https://github.com/user-attachments/assets/5e154e71-4ab5-4872-bfb2-04f02b73c18a"
/>
<img width="661" height="682" alt="Screenshot 2026-03-11 at 5 20 54 PM"
src="https://github.com/user-attachments/assets/00cb14b7-0bcf-4b94-9d71-34e93369ecb2"
/>
Connection can be tested before attaching to dataset:
<img width="981" height="681" alt="Screenshot 2026-03-11 at 5 21 40 PM"
src="https://github.com/user-attachments/assets/aaa6eeeb-89a7-4349-bc34-2423bf8be9ee"
/>
Ingestion tested with API connector (works perfectly fine):
<img width="1062" height="705" alt="Screenshot 2026-03-11 at 5 22 30 PM"
src="https://github.com/user-attachments/assets/afcd0d58-cadd-4152-badc-d2f14d96fbec"
/>
Search & Retrieval works as well with metadata flow:
<img width="1062" height="705" alt="Screenshot 2026-03-11 at 5 23 05 PM"
src="https://github.com/user-attachments/assets/d41ee935-dcf7-4456-b317-22a76ca032c0"
/>
---------
Co-authored-by: Ahmad Intisar <ahmadintisar@Ahmads-MacBook-M4-Pro.local>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
232 lines
5.2 KiB
TypeScript
232 lines
5.2 KiB
TypeScript
import api from '@/utils/api';
|
|
import request from '@/utils/request';
|
|
|
|
export interface SkillSpace {
|
|
id: string;
|
|
tenant_id: string;
|
|
name: string;
|
|
folder_id: string;
|
|
description?: string;
|
|
embd_id?: string;
|
|
rerank_id?: string;
|
|
top_k?: number;
|
|
status?: string;
|
|
create_time?: number;
|
|
update_time?: string;
|
|
}
|
|
|
|
export interface CreateSpaceRequest {
|
|
name: string;
|
|
description?: string;
|
|
embd_id?: string;
|
|
rerank_id?: string;
|
|
}
|
|
|
|
export interface UpdateSpaceRequest {
|
|
name?: string;
|
|
description?: string;
|
|
embd_id?: string;
|
|
rerank_id?: string;
|
|
top_k?: number;
|
|
}
|
|
|
|
export interface SkillSearchConfig {
|
|
id: string;
|
|
tenant_id: string;
|
|
space_id: string;
|
|
embd_id: string;
|
|
vector_similarity_weight: number;
|
|
similarity_threshold: number;
|
|
field_config: Record<string, any>;
|
|
rerank_id?: string;
|
|
tenant_rerank_id?: number;
|
|
top_k: number;
|
|
index_version: string;
|
|
status: string;
|
|
create_time?: number;
|
|
update_time?: string;
|
|
}
|
|
|
|
export interface UpdateConfigRequest {
|
|
tenant_id?: string;
|
|
space_id?: string;
|
|
embd_id: string;
|
|
vector_similarity_weight: number;
|
|
similarity_threshold: number;
|
|
field_config: Record<string, any>;
|
|
rerank_id?: string;
|
|
top_k: number;
|
|
}
|
|
|
|
export interface SearchRequest {
|
|
tenant_id?: string;
|
|
space_id?: string;
|
|
query: string;
|
|
page?: number;
|
|
page_size?: number;
|
|
}
|
|
|
|
export interface SearchResult {
|
|
skills: Array<{
|
|
skill_id: string;
|
|
folder_id: string;
|
|
name: string;
|
|
description: string;
|
|
tags: string[];
|
|
score: number;
|
|
bm25_score?: number;
|
|
vector_score?: number;
|
|
index_version?: string;
|
|
}>;
|
|
total: number;
|
|
query: string;
|
|
search_type: string;
|
|
}
|
|
|
|
export interface SkillInfo {
|
|
id: string;
|
|
folder_id: string;
|
|
name: string;
|
|
description: string;
|
|
tags: string[];
|
|
content: string;
|
|
}
|
|
|
|
export interface IndexSkillsRequest {
|
|
tenant_id?: string;
|
|
space_id?: string;
|
|
skills: SkillInfo[];
|
|
embd_id?: string;
|
|
}
|
|
|
|
class SkillSpaceService {
|
|
private async request<T>(
|
|
method: string,
|
|
url: string,
|
|
data?: any,
|
|
params?: any,
|
|
): Promise<T> {
|
|
const response: any = await request(url, {
|
|
method: method as any,
|
|
data,
|
|
params,
|
|
});
|
|
|
|
const jsonData = response?.data ?? response;
|
|
|
|
if (jsonData?.code !== 0) {
|
|
throw new Error(jsonData?.message || 'Request failed');
|
|
}
|
|
|
|
return jsonData.data;
|
|
}
|
|
|
|
// ==================== Skill Space Management ====================
|
|
|
|
// List all skill spaces
|
|
async listSpaces(): Promise<{ spaces: SkillSpace[]; total: number }> {
|
|
return await this.request<{ spaces: SkillSpace[]; total: number }>(
|
|
'GET',
|
|
api.skillSpaces,
|
|
);
|
|
}
|
|
|
|
// Create a new skill space
|
|
async createSpace(request: CreateSpaceRequest): Promise<SkillSpace> {
|
|
return await this.request<SkillSpace>('POST', api.skillSpaces, request);
|
|
}
|
|
|
|
// Get a skill space by ID
|
|
async getSpace(spaceId: string): Promise<SkillSpace> {
|
|
return await this.request<SkillSpace>('GET', api.skillSpace(spaceId));
|
|
}
|
|
|
|
// Update a skill space
|
|
async updateSpace(
|
|
spaceId: string,
|
|
request: UpdateSpaceRequest,
|
|
): Promise<SkillSpace> {
|
|
return await this.request<SkillSpace>(
|
|
'PUT',
|
|
api.skillSpace(spaceId),
|
|
request,
|
|
);
|
|
}
|
|
|
|
// Delete a skill space
|
|
async deleteSpace(spaceId: string): Promise<void> {
|
|
await this.request<void>('DELETE', api.skillSpace(spaceId));
|
|
}
|
|
|
|
// Get space by folder ID
|
|
async getSpaceByFolder(folderId: string): Promise<SkillSpace> {
|
|
return await this.request<SkillSpace>('GET', api.skillSpaceByFolder, null, {
|
|
folder_id: folderId,
|
|
});
|
|
}
|
|
|
|
// ==================== Skill Search Config ====================
|
|
|
|
// Get skill search config
|
|
async getConfig(
|
|
spaceId?: string,
|
|
embdId?: string,
|
|
): Promise<SkillSearchConfig> {
|
|
const params: Record<string, string> = {};
|
|
if (spaceId) params.space_id = spaceId;
|
|
if (embdId) params.embd_id = embdId;
|
|
|
|
return await this.request<SkillSearchConfig>(
|
|
'GET',
|
|
api.skillConfig,
|
|
null,
|
|
params,
|
|
);
|
|
}
|
|
|
|
// Update skill search config
|
|
async updateConfig(request: UpdateConfigRequest): Promise<SkillSearchConfig> {
|
|
return await this.request<SkillSearchConfig>(
|
|
'POST',
|
|
api.skillConfig,
|
|
request,
|
|
);
|
|
}
|
|
|
|
// ==================== Skill Search ====================
|
|
|
|
// Search skills
|
|
async search(request: SearchRequest): Promise<SearchResult> {
|
|
return await this.request<SearchResult>('POST', api.skillSearch, request);
|
|
}
|
|
|
|
// ==================== Skill Indexing ====================
|
|
|
|
// Index skills
|
|
async indexSkills(
|
|
request: IndexSkillsRequest,
|
|
): Promise<{ indexed_count: number }> {
|
|
return await this.request<{ indexed_count: number }>(
|
|
'POST',
|
|
api.skillIndex,
|
|
request,
|
|
);
|
|
}
|
|
|
|
// Delete skill index
|
|
async deleteSkillIndex(skillId: string, spaceId?: string): Promise<void> {
|
|
const params: Record<string, string> = { skill_id: skillId };
|
|
if (spaceId) params.space_id = spaceId;
|
|
|
|
await this.request<void>('DELETE', api.skillIndex, null, params);
|
|
}
|
|
|
|
// Reindex all skills
|
|
async reindex(request: IndexSkillsRequest): Promise<any> {
|
|
return await this.request<any>('POST', api.skillReindex, request);
|
|
}
|
|
}
|
|
|
|
export const skillSpaceService = new SkillSpaceService();
|
|
export default skillSpaceService;
|