From ca00d23aac069e942bf964e2329c91cb492a7f39 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Thu, 11 Jun 2026 13:33:26 +0800 Subject: [PATCH] Go: add parse and chunk command (#15936) ### What problem does this PR solve? Two commands are used for ingestion file testing ``` RAGFlow(api/default)> chunk 'file' with 'dsl'; Chunk file: file, DSL: dsl SUCCESS RAGFlow(api/default)> parse file 'filename' chat 'xxx'; Success to parse local file "filename", vision: , chat: xxx, asr: , ocr: , embedding: , doc_parse: SUCCESS ``` ### Type of change - [x] New Feature (non-breaking change which adds functionality) Signed-off-by: Jin Hai --- internal/cli/cli_http.go | 4 ++ internal/cli/lexer.go | 2 + internal/cli/parser.go | 4 ++ internal/cli/types.go | 1 + internal/cli/user_command.go | 73 +++++++++++++++++++- internal/cli/user_parser.go | 128 +++++++++++++++++++++++++++++++++++ 6 files changed, 211 insertions(+), 1 deletion(-) diff --git a/internal/cli/cli_http.go b/internal/cli/cli_http.go index 33435a2024..54f489e873 100644 --- a/internal/cli/cli_http.go +++ b/internal/cli/cli_http.go @@ -285,6 +285,8 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.GetMetadata(cmd) case "parse_documents_user_command": return c.ParseDocumentsUserCommand(cmd) + case "user_parse_local_file_command": + return c.UserParseLocalFile(cmd) case "show_admin_server": return c.ShowAdminServer(cmd) case "show_api_server": @@ -299,6 +301,8 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.AddAdminServer(cmd) case "delete_admin_server": return c.DeleteAdminServer(cmd) + case "user_chunk_command": + return c.ChunkCommand(cmd) case "save_config_command": return c.SaveServerConfig(cmd) case "file_system_command": diff --git a/internal/cli/lexer.go b/internal/cli/lexer.go index 71f14c98e7..ae1159b461 100644 --- a/internal/cli/lexer.go +++ b/internal/cli/lexer.go @@ -435,6 +435,8 @@ func (l *Lexer) lookupIdent(ident string) Token { return Token{Type: TokenUpdate, Value: ident} case "REMOVE": return Token{Type: TokenRemove, Value: ident} + case "EXPLAIN": + return Token{Type: TokenExplain, Value: ident} case "CHUNK": // Check if followed by STORE for compound token if strings.ToUpper(l.peekToken()) == "STORE" { diff --git a/internal/cli/parser.go b/internal/cli/parser.go index 67af77b9e5..3ff50d1ee8 100644 --- a/internal/cli/parser.go +++ b/internal/cli/parser.go @@ -226,6 +226,10 @@ func (p *Parser) parseUserCommand() (*Command, error) { return p.parseRemoveCommand() case TokenGet: return p.parseGetCommand() + case TokenExplain: + return p.parseExplainCommand() + case TokenChunk: + return p.parseChunkCommand(false) case TokenLS, TokenCat, TokenSearch: // For context engine diff --git a/internal/cli/types.go b/internal/cli/types.go index 1f144bf012..329ff70d71 100644 --- a/internal/cli/types.go +++ b/internal/cli/types.go @@ -155,6 +155,7 @@ const ( TokenGet TokenUpdate TokenRemove + TokenExplain TokenChunk TokenChunks TokenDocument diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go index b783ebc6d5..1d1c85b06c 100644 --- a/internal/cli/user_command.go +++ b/internal/cli/user_command.go @@ -2734,7 +2734,6 @@ func (c *CLI) AddCustomModel(cmd *Command) (ResponseIf, error) { } - // InsertChunksFromFile inserts chunks from a JSON file func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { @@ -3215,6 +3214,47 @@ func (c *CLI) ParseDocumentsUserCommand(cmd *Command) (ResponseIf, error) { return &result, nil } +func (c *CLI) UserParseLocalFile(cmd *Command) (ResponseIf, error) { + if c.Config.CLIMode != APIMode { + return nil, fmt.Errorf("this command is only allowed in USER mode") + } + + filename, ok := cmd.Params["filename"].(string) + if !ok { + return nil, fmt.Errorf("filename not provided") + } + visionModel, ok := cmd.Params["vision_model"].(string) + if !ok { + visionModel = "" + } + chatModel, ok := cmd.Params["chat_model"].(string) + if !ok { + chatModel = "" + } + asrModel, ok := cmd.Params["asr_model"].(string) + if !ok { + asrModel = "" + } + ocrModel, ok := cmd.Params["ocr_model"].(string) + if !ok { + ocrModel = "" + } + embeddingModel, ok := cmd.Params["embedding_model"].(string) + if !ok { + embeddingModel = "" + } + docParseModel, ok := cmd.Params["doc_parse_model"].(string) + if !ok { + docParseModel = "" + } + + var result SimpleResponse + result.Code = 0 + result.Message = fmt.Sprintf("Success to parse local file %q, vision: %v, chat: %v, asr: %v, ocr: %v, embedding: %v, doc_parse: %v", filename, visionModel, chatModel, asrModel, ocrModel, embeddingModel, docParseModel) + fmt.Println(result.Message) + return &result, nil +} + // formatRequestError Uniformly handle and format network errors in HTTP requests func formatRequestError(action string, err error) error { if err == nil { @@ -3232,3 +3272,34 @@ func formatRequestError(action string, err error) error { return fmt.Errorf("%s failed: %w", action, err) } } + +func (c *CLI) ChunkCommand(cmd *Command) (ResponseIf, error) { + if c.Config.CLIMode != APIMode { + return nil, fmt.Errorf("this command is only allowed in USER mode") + } + + filename, ok := cmd.Params["filename"].(string) + if !ok { + return nil, fmt.Errorf("filename not provided") + } + dsl, ok := cmd.Params["dsl"].(string) + if !ok { + return nil, fmt.Errorf("dsl not provided") + } + explain, ok := cmd.Params["explain"].(bool) + if !ok { + explain = false + } + + if explain { + fmt.Printf("Explain chunk file: %s, DSL: %s\n", filename, dsl) + } else { + fmt.Printf("Chunk file: %s, DSL: %s\n", filename, dsl) + } + + var result SimpleResponse + result.Code = 0 + result.Message = fmt.Sprintf("Success to chunk %s", filename) + + return &result, nil +} diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go index 6f5640a7dc..693dce6b47 100644 --- a/internal/cli/user_parser.go +++ b/internal/cli/user_parser.go @@ -3763,6 +3763,8 @@ func (p *Parser) parseParseCommand() (*Command, error) { return p.parseModelParseCommand() case TokenDocument: return p.parseParseDocs() + case TokenFile: + return p.parseParseLocalFileCommand() default: return nil, fmt.Errorf("expected DATASET, WITH, or DOCUMENT") } @@ -3831,6 +3833,88 @@ func (p *Parser) parseParseDocs() (*Command, error) { return cmd, nil } +func (p *Parser) parseParseLocalFileCommand() (*Command, error) { + p.nextToken() // consume FILE + + filename, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("user_parse_local_file_command") + cmd.Params["filename"] = filename + +optionsLoop: + for { + switch p.curToken.Type { + case TokenVision: + p.nextToken() + var visionModel string + visionModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["vision_model"] = visionModel + p.nextToken() + case TokenASR: + p.nextToken() + var asrModel string + asrModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["asr_model"] = asrModel + p.nextToken() + + case TokenOCR: + p.nextToken() + var ocrModel string + ocrModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["ocr_model"] = ocrModel + p.nextToken() + case TokenChat: + p.nextToken() + var chatModel string + chatModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["chat_model"] = chatModel + p.nextToken() + case TokenEmbed: + p.nextToken() + var embedModel string + embedModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["embed_model"] = embedModel + p.nextToken() + case TokenDocParse: + p.nextToken() + var docParseModel string + docParseModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["doc_parse_model"] = docParseModel + p.nextToken() + case TokenSemicolon: + p.nextToken() + break optionsLoop // done + default: + // No more options to process + break optionsLoop + } + } + + return cmd, nil +} + func (p *Parser) parseBenchmarkCommand() (*Command, error) { cmd := NewCommand("benchmark") @@ -4461,3 +4545,47 @@ func (p *Parser) parseListApiCommand() (*Command, error) { return cmd, nil } + +func (p *Parser) parseExplainCommand() (*Command, error) { + p.nextToken() // consume EXPLAIN + + switch p.curToken.Type { + case TokenChunk: + return p.parseChunkCommand(true) + default: + return nil, fmt.Errorf("expected CHUNK after EXPLAIN") + } +} + +func (p *Parser) parseChunkCommand(explain bool) (*Command, error) { + p.nextToken() // consume CHUNK + + filename, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected filename: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenWith { + return nil, fmt.Errorf("expected WITH after filename") + } + p.nextToken() + + dsl, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected DSL: %w", err) + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + p.nextToken() + + cmd := NewCommand("user_chunk_command") + cmd.Params["dsl"] = dsl + cmd.Params["filename"] = filename + cmd.Params["explain"] = explain + + return cmd, nil +}