Go: add parse and chunk command (#15936)

### What problem does this PR solve?

Two commands are used for ingestion file testing
```
RAGFlow(api/default)> chunk 'file' with 'dsl';
Chunk file: file, DSL: dsl
SUCCESS
RAGFlow(api/default)> parse file 'filename' chat 'xxx';
Success to parse local file "filename", vision: , chat: xxx, asr: , ocr: , embedding: , doc_parse: 
SUCCESS
```

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

internal/cli/cli_http.go

@@ -285,6 +285,8 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.GetMetadata(cmd)
 	case "parse_documents_user_command":
 		return c.ParseDocumentsUserCommand(cmd)
+	case "user_parse_local_file_command":
+		return c.UserParseLocalFile(cmd)
 	case "show_admin_server":
 		return c.ShowAdminServer(cmd)
 	case "show_api_server":
@@ -299,6 +301,8 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.AddAdminServer(cmd)
 	case "delete_admin_server":
 		return c.DeleteAdminServer(cmd)
+	case "user_chunk_command":
+		return c.ChunkCommand(cmd)
 	case "save_config_command":
 		return c.SaveServerConfig(cmd)
 	case "file_system_command":

internal/cli/lexer.go

@@ -435,6 +435,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenUpdate, Value: ident}
 	case "REMOVE":
 		return Token{Type: TokenRemove, Value: ident}
+	case "EXPLAIN":
+		return Token{Type: TokenExplain, Value: ident}
 	case "CHUNK":
 		// Check if followed by STORE for compound token
 		if strings.ToUpper(l.peekToken()) == "STORE" {

internal/cli/parser.go

@@ -226,6 +226,10 @@ func (p *Parser) parseUserCommand() (*Command, error) {
 		return p.parseRemoveCommand()
 	case TokenGet:
 		return p.parseGetCommand()
+	case TokenExplain:
+		return p.parseExplainCommand()
+	case TokenChunk:
+		return p.parseChunkCommand(false)
 
 	case TokenLS, TokenCat, TokenSearch:
 		// For context engine

internal/cli/types.go

@@ -155,6 +155,7 @@ const (
 	TokenGet
 	TokenUpdate
 	TokenRemove
+	TokenExplain
 	TokenChunk
 	TokenChunks
 	TokenDocument

internal/cli/user_command.go

@@ -2734,7 +2734,6 @@ func (c *CLI) AddCustomModel(cmd *Command) (ResponseIf, error) {
 
 }
 
-
 // InsertChunksFromFile inserts chunks from a JSON file
 func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) {
 	if c.Config.CLIMode != APIMode {
@@ -3215,6 +3214,47 @@ func (c *CLI) ParseDocumentsUserCommand(cmd *Command) (ResponseIf, error) {
 	return &result, nil
 }
 
+func (c *CLI) UserParseLocalFile(cmd *Command) (ResponseIf, error) {
+	if c.Config.CLIMode != APIMode {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	filename, ok := cmd.Params["filename"].(string)
+	if !ok {
+		return nil, fmt.Errorf("filename not provided")
+	}
+	visionModel, ok := cmd.Params["vision_model"].(string)
+	if !ok {
+		visionModel = ""
+	}
+	chatModel, ok := cmd.Params["chat_model"].(string)
+	if !ok {
+		chatModel = ""
+	}
+	asrModel, ok := cmd.Params["asr_model"].(string)
+	if !ok {
+		asrModel = ""
+	}
+	ocrModel, ok := cmd.Params["ocr_model"].(string)
+	if !ok {
+		ocrModel = ""
+	}
+	embeddingModel, ok := cmd.Params["embedding_model"].(string)
+	if !ok {
+		embeddingModel = ""
+	}
+	docParseModel, ok := cmd.Params["doc_parse_model"].(string)
+	if !ok {
+		docParseModel = ""
+	}
+
+	var result SimpleResponse
+	result.Code = 0
+	result.Message = fmt.Sprintf("Success to parse local file %q, vision: %v, chat: %v, asr: %v, ocr: %v, embedding: %v, doc_parse: %v", filename, visionModel, chatModel, asrModel, ocrModel, embeddingModel, docParseModel)
+	fmt.Println(result.Message)
+	return &result, nil
+}
+
 // formatRequestError Uniformly handle and format network errors in HTTP requests
 func formatRequestError(action string, err error) error {
 	if err == nil {
@@ -3232,3 +3272,34 @@ func formatRequestError(action string, err error) error {
 		return fmt.Errorf("%s failed: %w", action, err)
 	}
 }
+
+func (c *CLI) ChunkCommand(cmd *Command) (ResponseIf, error) {
+	if c.Config.CLIMode != APIMode {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	filename, ok := cmd.Params["filename"].(string)
+	if !ok {
+		return nil, fmt.Errorf("filename not provided")
+	}
+	dsl, ok := cmd.Params["dsl"].(string)
+	if !ok {
+		return nil, fmt.Errorf("dsl not provided")
+	}
+	explain, ok := cmd.Params["explain"].(bool)
+	if !ok {
+		explain = false
+	}
+
+	if explain {
+		fmt.Printf("Explain chunk file: %s, DSL: %s\n", filename, dsl)
+	} else {
+		fmt.Printf("Chunk file: %s, DSL: %s\n", filename, dsl)
+	}
+
+	var result SimpleResponse
+	result.Code = 0
+	result.Message = fmt.Sprintf("Success to chunk %s", filename)
+
+	return &result, nil
+}

internal/cli/user_parser.go

@@ -3763,6 +3763,8 @@ func (p *Parser) parseParseCommand() (*Command, error) {
 		return p.parseModelParseCommand()
 	case TokenDocument:
 		return p.parseParseDocs()
+	case TokenFile:
+		return p.parseParseLocalFileCommand()
 	default:
 		return nil, fmt.Errorf("expected DATASET, WITH, or DOCUMENT")
 	}
@@ -3831,6 +3833,88 @@ func (p *Parser) parseParseDocs() (*Command, error) {
 	return cmd, nil
 }
 
+func (p *Parser) parseParseLocalFileCommand() (*Command, error) {
+	p.nextToken() // consume FILE
+
+	filename, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	cmd := NewCommand("user_parse_local_file_command")
+	cmd.Params["filename"] = filename
+
+optionsLoop:
+	for {
+		switch p.curToken.Type {
+		case TokenVision:
+			p.nextToken()
+			var visionModel string
+			visionModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["vision_model"] = visionModel
+			p.nextToken()
+		case TokenASR:
+			p.nextToken()
+			var asrModel string
+			asrModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["asr_model"] = asrModel
+			p.nextToken()
+
+		case TokenOCR:
+			p.nextToken()
+			var ocrModel string
+			ocrModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["ocr_model"] = ocrModel
+			p.nextToken()
+		case TokenChat:
+			p.nextToken()
+			var chatModel string
+			chatModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["chat_model"] = chatModel
+			p.nextToken()
+		case TokenEmbed:
+			p.nextToken()
+			var embedModel string
+			embedModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["embed_model"] = embedModel
+			p.nextToken()
+		case TokenDocParse:
+			p.nextToken()
+			var docParseModel string
+			docParseModel, err = p.parseQuotedString()
+			if err != nil {
+				return nil, err
+			}
+			cmd.Params["doc_parse_model"] = docParseModel
+			p.nextToken()
+		case TokenSemicolon:
+			p.nextToken()
+			break optionsLoop // done
+		default:
+			// No more options to process
+			break optionsLoop
+		}
+	}
+
+	return cmd, nil
+}
+
 func (p *Parser) parseBenchmarkCommand() (*Command, error) {
 	cmd := NewCommand("benchmark")
 
@@ -4461,3 +4545,47 @@ func (p *Parser) parseListApiCommand() (*Command, error) {
 
 	return cmd, nil
 }
+
+func (p *Parser) parseExplainCommand() (*Command, error) {
+	p.nextToken() // consume EXPLAIN
+
+	switch p.curToken.Type {
+	case TokenChunk:
+		return p.parseChunkCommand(true)
+	default:
+		return nil, fmt.Errorf("expected CHUNK after EXPLAIN")
+	}
+}
+
+func (p *Parser) parseChunkCommand(explain bool) (*Command, error) {
+	p.nextToken() // consume CHUNK
+
+	filename, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected filename: %w", err)
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenWith {
+		return nil, fmt.Errorf("expected WITH after filename")
+	}
+	p.nextToken()
+
+	dsl, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected DSL: %w", err)
+	}
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+	p.nextToken()
+
+	cmd := NewCommand("user_chunk_command")
+	cmd.Params["dsl"] = dsl
+	cmd.Params["filename"] = filename
+	cmd.Params["explain"] = explain
+
+	return cmd, nil
+}