zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-29 04:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: add file parse command (#14892)

Browse Source 
### What problem does this PR solve?

```
RAGFlow(user)> ocr with 'hunyuanocr@test@gitee' file './picture.png'
+----------------------------------------------------------+
| text                                                     |
+----------------------------------------------------------+
| 生活不是等待风暴过去，而是学会在雨中翩翩起舞。
——佚名                                                       |
+----------------------------------------------------------+

RAGFlow(user)> list 'test@gitee' tasks;
+---------+----------------------------------+
| status  | task_id                          |
+---------+----------------------------------+
| success | C3FX4MQNKY5MGC6ZFMIXIAMJKHCEBQB5 |
+---------+----------------------------------+
RAGFlow(user)> show 'test@gitee' task 'C3FX4MQNKY5MGC6ZFMIXIAMJKHCEBQB5';
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
| content                                                                                                                                                                                                                                                          | index |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+
| # PDF 1: Purpose of RAGFlow  

RAGFlow is an open source Retrieval-Augmented Generation (RAG) engine designed to turn raw documents into reliable context for large language models.Its purpose is to make it practical to build an Al assistant that can ans... | 1     |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+

```

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-05-15 12:29:52 +08:00
committed by GitHub
parent 547b8cf9d8
commit 3a5df08c76
46 changed files with 1533 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/cli/client.go
View File

@@ -273,6 +273,8 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ASRUserCommand(cmd)
case "ocr_user_command":
return c.OCRUserCommand(cmd)
case "parse_file_user_command":
return c.ParseFileUserCommand(cmd)
case "check_provider_connection":
return c.CheckProviderConnection(cmd)
case "use_model":
@@ -285,6 +287,10 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ResetDefaultModel(cmd)
case "list_user_default_models":
return c.ListDefaultModels(cmd)
case "list_tasks_user_command":
return c.ListTasksUserCommand(cmd)
case "show_task_user_command":
return c.ShowTaskUserCommand(cmd)
// Dataset, metadata commands
case "create_dataset_table":
return c.CreateDatasetInDocEngine(cmd)

2
internal/cli/lexer.go
View File

@@ -437,6 +437,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenRegion, Value: ident}
case "URL":
return Token{Type: TokenURL, Value: ident}
case "TASK":
return Token{Type: TokenTask, Value: ident}
case "TASKS":
return Token{Type: TokenTasks, Value: ident}
case "LOG":

38
internal/cli/response.go
View File

@@ -319,6 +319,44 @@ func (r *EmbeddingsResponse) PrintOut() {
}
}
type SegmentResponse struct {
Segments []map[string]interface{} `json:"segments"`
}
type TaskResponse struct {
Code int `json:"code"`
Data map[string]interface{} `json:"data"`
Message string `json:"message"`
Duration float64
OutputFormat OutputFormat
}
func (r *TaskResponse) Type() string {
return "task"
}
func (r *TaskResponse) TimeCost() float64 {
return r.Duration
}
func (r *TaskResponse) SetOutputFormat(format OutputFormat) {
r.OutputFormat = format
}
func (r *TaskResponse) PrintOut() {
if r.Code == 0 {
segmentsRaw := r.Data["segments"].([]interface{})
segments := make([]map[string]interface{}, len(segmentsRaw))
for i, v := range segmentsRaw {
segments[i] = v.(map[string]interface{})
}
PrintTableSimpleByFormat(segments, r.OutputFormat)
} else {
fmt.Println("ERROR")
fmt.Printf("%d, %s\n", r.Code, r.Message)
}
}
// ==================== ContextEngine Commands ====================
// ContextListResponse represents the response for ls command

1
internal/cli/types.go
View File

@@ -152,6 +152,7 @@ const (
TokenTag
TokenRegion
TokenURL
TokenTask
TokenTasks
TokenLog
TokenLevel

173
internal/cli/user_command.go
View File

@@ -2284,6 +2284,179 @@ func (c *RAGFlowClient) OCRUserCommand(cmd *Command) (ResponseIf, error) {
return &result, nil
}
func (c *RAGFlowClient) ParseFileUserCommand(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
var providerName, instanceName, modelName string
// Check if composite_model_name is provided in command
if compositeModelName, ok := cmd.Params["composite_model_name"].(string); ok && compositeModelName != "" {
names := strings.Split(compositeModelName, "@")
if len(names) != 3 {
return nil, fmt.Errorf("model name must be in format 'model@instance@provider'")
}
providerName = names[2]
instanceName = names[1]
modelName = names[0]
} else if c.CurrentModel != nil {
// Use current model if set
providerName = c.CurrentModel.Provider
instanceName = c.CurrentModel.Instance
modelName = c.CurrentModel.Model
} else {
return nil, fmt.Errorf("model name not provided and no current model set. Use 'use model' command first")
}
var filename string
var fileURL string
var ok bool
var fileContent []byte
filename, ok = cmd.Params["file"].(string)
if ok {
// read file and convert to base64
var err error
fileContent, err = os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
} else {
fileURL, ok = cmd.Params["url"].(string)
if !ok {
return nil, fmt.Errorf("file or url not provided")
}
}
payload := map[string]interface{}{
"provider_name": providerName,
"instance_name": instanceName,
"model_name": modelName,
}
if fileContent != nil {
payload["content"] = fileContent
} else {
payload["url"] = fileURL
}
url := "/file/parse"
resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to PARSE document: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to PARSE document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonDataResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("PARSE document failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) ListTasksUserCommand(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
var providerName, instanceName string
// Check if composite_instance_name is provided in command
if compositeModelName, ok := cmd.Params["composite_instance_name"].(string); ok && compositeModelName != "" {
names := strings.Split(compositeModelName, "@")
if len(names) != 2 {
return nil, fmt.Errorf("model name must be in format 'instance@provider'")
}
providerName = names[1]
instanceName = names[0]
} else {
return nil, fmt.Errorf("no provider name or instance name")
}
url := fmt.Sprintf("/providers/%s/instances/%s/tasks", providerName, instanceName)
resp, err := c.HTTPClient.Request("GET", url, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to list tasks: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to list tasks: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result CommonResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("list tasks failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) ShowTaskUserCommand(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
var providerName, instanceName string
// Check if composite_instance_name is provided in command
if compositeModelName, ok := cmd.Params["composite_instance_name"].(string); ok && compositeModelName != "" {
names := strings.Split(compositeModelName, "@")
if len(names) != 2 {
return nil, fmt.Errorf("model name must be in format 'instance@provider'")
}
providerName = names[1]
instanceName = names[0]
} else {
return nil, fmt.Errorf("no provider name or instance name")
}
taskID, ok := cmd.Params["task_id"].(string)
if !ok {
return nil, fmt.Errorf("task id not provided")
}
url := fmt.Sprintf("/providers/%s/instances/%s/tasks/%s", providerName, instanceName, taskID)
resp, err := c.HTTPClient.Request("GET", url, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to get task: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to get task: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
var result TaskResponse
if err = json.Unmarshal(resp.Body, &result); err != nil {
return nil, fmt.Errorf("get task failed: invalid JSON (%w)", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("%s", result.Message)
}
result.Duration = resp.Duration
return &result, nil
}
func (c *RAGFlowClient) CheckProviderConnection(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")

125
internal/cli/user_parser.go
View File

@@ -163,6 +163,8 @@ func (p *Parser) parseListCommand() (*Command, error) {
return NewCommand("list_user_chats"), nil
case TokenFiles:
return p.parseListFiles()
case TokenQuotedString:
return p.parseListQuotedStringCommand()
default:
return nil, fmt.Errorf("unknown LIST target: %s", p.curToken.Value)
}
@@ -280,9 +282,57 @@ func (p *Parser) parseListFiles() (*Command, error) {
return cmd, nil
}
func (p *Parser) parseListQuotedStringCommand() (*Command, error) {
str, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken() // consume str
switch p.curToken.Type {
case TokenTasks:
p.nextToken() // consume TASKS
cmd := NewCommand("list_tasks_user_command")
cmd.Params["composite_instance_name"] = str
return cmd, nil
default:
return nil, fmt.Errorf("unknown command: %s", str)
}
}
func (p *Parser) parseShowQuotedStringCommand() (*Command, error) {
str, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken() // consume str
switch p.curToken.Type {
case TokenTask:
p.nextToken() // consume TASK
var taskID string
taskID, err = p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected string: %w", err)
}
p.nextToken()
cmd := NewCommand("show_task_user_command")
cmd.Params["task_id"] = taskID
cmd.Params["composite_instance_name"] = str
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
default:
return nil, fmt.Errorf("unknown command: %s", str)
}
}
func (p *Parser) parseShowCommand() (*Command, error) {
p.nextToken() // consume SHOW
switch p.curToken.Type {
case TokenVersion:
p.nextToken()
@@ -333,6 +383,10 @@ func (p *Parser) parseShowCommand() (*Command, error) {
return p.parseShowInstance()
case TokenBalance:
return p.parseShowBalance()
case TokenTask:
return p.parseShowTask()
case TokenQuotedString:
return p.parseShowQuotedStringCommand()
default:
return nil, fmt.Errorf("unknown SHOW target: %s", p.curToken.Value)
}
@@ -1454,6 +1508,27 @@ func (p *Parser) parseShowBalance() (*Command, error) {
return cmd, nil
}
// parseShowTask parses SHOW TASK <task>
func (p *Parser) parseShowTask() (*Command, error) {
p.nextToken() // consume TASK
taskID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected string: %w", err)
}
p.nextToken()
cmd := NewCommand("show_task_user_command")
cmd.Params["task_id"] = taskID
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseAlterInstance parses ALTER INSTANCE <name> NAME <new_name> FROM PROVIDER <name> command
func (p *Parser) parseAlterInstance() (*Command, error) {
p.nextToken() // consume INSTANCE
@@ -2900,6 +2975,45 @@ func (p *Parser) parseOCRCommand() (*Command, error) {
return cmd, nil
}
func (p *Parser) parseModelParseCommand() (*Command, error) {
p.nextToken() // consume WITH
compositeModelName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
cmd := NewCommand("parse_file_user_command")
switch p.curToken.Type {
case TokenFile:
p.nextToken()
var file string
file, err = p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["file"] = file
p.nextToken()
case TokenURL:
p.nextToken()
var url string
url, err = p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["url"] = url
p.nextToken()
default:
return nil, fmt.Errorf("expected FILE or URL")
}
cmd.Params["composite_model_name"] = compositeModelName
return cmd, nil
}
func (p *Parser) parseCheckCommand() (*Command, error) {
p.nextToken() // consume CHECK
@@ -2964,11 +3078,14 @@ func (p *Parser) parseUseCommand() (*Command, error) {
func (p *Parser) parseParseCommand() (*Command, error) {
p.nextToken() // consume PARSE
if p.curToken.Type == TokenDataset {
switch p.curToken.Type {
case TokenDataset:
return p.parseParseDataset()
case TokenWith:
return p.parseModelParseCommand()
default:
return p.parseParseDocs()
}
return p.parseParseDocs()
}
func (p *Parser) parseParseDataset() (*Command, error) {