zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-11 06:05:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: add ingestion server (#15094)

Browse Source 
### What problem does this PR solve?

1. Go ingestion server will connected with admin server with gRPC stream
2. Go ingestion server will be responsible for ingestion tasks
```

RAGFlow(admin)> list ingestors;
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+
| address         | cpu_usage | id                               | last_heartbeat            | name     | process_id | rss_usage    | status | task_count | vms_usage     |
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+
| 127.0.0.1:58564 | 0         | bdd1870eea2646e0aacb8a2cd3307aa2 | 2026-05-24T18:16:17+08:00 | ingestor | 680152     | 212.72265625 | active | 0          | 2589.12109375 |
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+

RAGFlow(admin)> start ingestion 'abc';
+----------------------------------+
| task_id                          |
+----------------------------------+
| e714777639ca4760ab427b5f211e81ad |
+----------------------------------+

RAGFlow(admin)> stop ingestion 'f7bd39d0a724457eb5fdce6d81699776';
+----------------------------------+
| task_id                          |
+----------------------------------+
| f7bd39d0a724457eb5fdce6d81699776 |
+----------------------------------+

RAGFlow(admin)> list tasks;
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+
| ETA | assign_to                        | error | from | id                               | last_update               | start_time | status     |
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | eae6431da72a40e796cff3a03008091b | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 6cccdd174bd049ecb05a774bbb47593f | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | ef360d777e57485799adb96b30f2b4b8 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | bcc5c5448cb64de48b6b6171c36fb790 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | bfc25384c43a443294fe2da979a38ac2 | 2026-05-24T19:46:03+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 84960537b85d413b8990a9efd5952d67 | 2026-05-24T19:46:04+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 3d223c1b51e24b36861a3bfb2f1d58d4 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | e433b0e356b846c89c301621a3c54494 | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 7c93a3880f074ebd8eca14e6b51bb7ef | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | df2e4ef51aaf4390bff9a23f2692486e | 2026-05-24T19:46:04+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 7377c53010194ef7a83aa206698d66ff | 2026-05-24T19:46:05+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | df64d1a1f9d348e3a2f174c4d7d69e73 | 2026-05-24T19:46:05+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | b59834512e2847e1bdf13ace04b8a456 | 2026-05-24T19:46:06+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 0064bb0ab69344028d1ecfda053826f4 | 2026-05-24T19:46:03+08:00 |            | QUEUED     |
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+


```


### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-05-25 14:00:08 +08:00
committed by GitHub
parent 5d022d83e8
commit f8c626bbc8
36 changed files with 3447 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
internal/cli/admin_command.go
View File

@@ -1281,3 +1281,162 @@ func (c *RAGFlowClient) ListAdminTasks(cmd *Command) (ResponseIf, error) {
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) ListAdminIngestors(cmd *Command) (ResponseIf, error) {
if c.ServerType != "admin" {
return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
}
resp, err := c.HTTPClient.Request("GET", "/admin/ingestors", "admin", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to list ingestors: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to list ingestors: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("list ingestors failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) ListAdminIngestionTasks(cmd *Command) (ResponseIf, error) {
if c.ServerType != "admin" {
return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
}
resp, err := c.HTTPClient.Request("GET", "/admin/ingestion/tasks", "admin", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to list admin tasks: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to list admin tasks: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("list admin tasks failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) AdminStartIngestionCommand(cmd *Command) (ResponseIf, error) {
if c.ServerType != "admin" {
return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
}
fileURI, ok := cmd.Params["uri"].(string)
if !ok {
return nil, fmt.Errorf("uri not provided")
}
payload := map[string]interface{}{
"uri": fileURI,
"from": "CLI",
}
resp, err := c.HTTPClient.Request("POST", "/admin/ingestion", "admin", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to ingest file: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to ingest file: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonDataResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("ingest file failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) AdminStopIngestionCommand(cmd *Command) (ResponseIf, error) {
if c.ServerType != "admin" {
return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
}
taskID, ok := cmd.Params["task_id"].(string)
if !ok {
return nil, fmt.Errorf("uri not provided")
}
payload := map[string]interface{}{
"task_id": taskID,
"from": "CLI",
}
resp, err := c.HTTPClient.Request("DELETE", "/admin/ingestion", "admin", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to ingest file: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to ingest file: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonDataResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("ingest file failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) AdminShutdownIngestor(cmd *Command) (ResponseIf, error) {
if c.ServerType != "admin" {
return nil, fmt.Errorf("this command is only allowed in ADMIN mode")
}
ingestorName, ok := cmd.Params["ingestor_name"].(string)
if !ok {
return nil, fmt.Errorf("ingestor_name not provided")
}
payload := map[string]interface{}{
"ingestor_name": ingestorName,
}
resp, err := c.HTTPClient.Request("DELETE", "/admin/ingestors", "admin", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to shutdown ingestor: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to shutdown ingestor: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonDataResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("shutdown ingestor failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}