Go: add ASR, TTS, OCR command (#14836)

### What problem does this PR solve? ``` RAGFlow(user)> asr with 'glm-asr-2512@test@zhipu-ai' audio './speech.wav'; CLI error: zhipu, no such method RAGFlow(user)> stream asr with 'glm-asr-2512@test@zhipu-ai' audio './speech.wav'; CLI error: zhipu, no such method RAGFlow(user)> tts with 'glm-tts@test@zhipu-ai' text 'how are you'; CLI error: zhipu, no such method RAGFlow(user)> stream tts with 'glm-tts@test@zhipu-ai' text 'how are you'; CLI error: zhipu, no such method RAGFlow(user)> ocr with 'glm-ocr@test@zhipu-ai' file './test.log'; CLI error: zhipu, no such method ``` ### Type of change - [x] New Feature (non-breaking change which adds functionality) Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2026-08-03 14:27:32 +08:00 · 2026-05-12 17:17:44 +08:00
parent 9ee481807f
commit d08bf02d9b
32 changed files with 1869 additions and 71 deletions
--- a/internal/cli/client.go
+++ b/internal/cli/client.go
@@ -267,6 +267,12 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.EmbedUserText(cmd)
 	case "rarank_user_document":
 		return c.RerankUserDocument(cmd)
+	case "tts_user_command":
+		return c.TTSUserCommand(cmd)
+	case "asr_user_command":
+		return c.ASRUserCommand(cmd)
+	case "ocr_user_command":
+		return c.OCRUserCommand(cmd)
 	case "check_provider_connection":
 		return c.CheckProviderConnection(cmd)
 	case "use_model":
--- a/internal/cli/parser.go
+++ b/internal/cli/parser.go
@@ -201,6 +201,12 @@ func (p *Parser) parseUserCommand() (*Command, error) {
 		return p.parseEmbedCommand()
 	case TokenRerank:
 		return p.parseRerankCommand()
+	case TokenASR:
+		return p.parseASRCommand()
+	case TokenTTS:
+		return p.parseTTSCommand()
+	case TokenOCR:
+		return p.parseOCRCommand()
 	case TokenCheck:
 		return p.parseCheckCommand()
 	case TokenLS:
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -27,6 +27,7 @@ import (
 	"net"
 	netUrl "net/url"
 	"os"
+	"path/filepath"
 	ce "ragflow/internal/cli/filesystem"
 	"strings"
 	"time"
@@ -1622,16 +1623,35 @@ func (c *RAGFlowClient) ChatToModel(cmd *Command) (ResponseIf, error) {
 		}
 	}

-	//audios, ok := cmd.Params["audios"].([]string)
-	//if !ok {
-	//	return nil, fmt.Errorf("images not provided")
-	//}
+	audios, ok := cmd.Params["audios"].([]string)
+	if !ok {
+		return nil, fmt.Errorf("images not provided")
+	}
+	if len(audios) > 0 {
+		if len(audios) != 1 {
+			return nil, fmt.Errorf("only one audio file is supported")
+		}
+		audioFile := audios[0]
+		audioContent, err := os.ReadFile(audioFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read audio: %w", err)
+		}
+		// file type: wav or mp3
+		format := filepath.Ext(audioFile) // file type: wav or mp3
+		format = strings.TrimPrefix(format, ".")
+		contents = append(contents, map[string]interface{}{
+			"type": "input_audio",
+			"input_audio": map[string]interface{}{
+				"data":   base64.StdEncoding.EncodeToString(audioContent),
+				"format": format,
+			},
+		})
+	}

 	files, ok := cmd.Params["files"].([]string)
 	if !ok {
 		return nil, fmt.Errorf("images not provided")
 	}
-
 	if len(files) > 0 {
 		for _, file := range files {
 			if isValidURL(file) {
@@ -1660,21 +1680,6 @@ func (c *RAGFlowClient) ChatToModel(cmd *Command) (ResponseIf, error) {

 	url := "/chat/completions"

-	//message = strings.TrimSpace(message)
-	//var content interface{} = message
-	//if strings.HasPrefix(message, "[") && strings.HasSuffix(message, "]") {
-	//	var parts []map[string]interface{}
-	//	if err := json.Unmarshal([]byte(message), &parts); err == nil {
-	//		content = parts
-	//	}
-	//}
-	//formattedMessage := []map[string]interface{}{
-	//	{
-	//		"role":    "user",
-	//		"content": content,
-	//	},
-	//}
-
 	payload := map[string]interface{}{
 		"provider_name": providerName,
 		"instance_name": instanceName,
@@ -1922,6 +1927,210 @@ func (c *RAGFlowClient) RerankUserDocument(cmd *Command) (ResponseIf, error) {
 	return &result, nil
 }

+func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) {
+	if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+		return nil, fmt.Errorf("API token not set. Please login first")
+	}
+
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	var providerName, instanceName, modelName string
+
+	// Check if composite_model_name is provided in command
+	if compositeModelName, ok := cmd.Params["composite_model_name"].(string); ok && compositeModelName != "" {
+		names := strings.Split(compositeModelName, "@")
+		if len(names) != 3 {
+			return nil, fmt.Errorf("model name must be in format 'model@instance@provider'")
+		}
+		providerName = names[2]
+		instanceName = names[1]
+		modelName = names[0]
+	} else if c.CurrentModel != nil {
+		// Use current model if set
+		providerName = c.CurrentModel.Provider
+		instanceName = c.CurrentModel.Instance
+		modelName = c.CurrentModel.Model
+	} else {
+		return nil, fmt.Errorf("model name not provided and no current model set. Use 'use model' command first")
+	}
+
+	text, ok := cmd.Params["text"].(string)
+	if !ok {
+		return nil, fmt.Errorf("text not provided")
+	}
+
+	//fileToSave, ok := cmd.Params["file"].(string)
+	//if !ok {
+	//	return nil, fmt.Errorf("file not provided")
+	//}
+
+	payload := map[string]interface{}{
+		"provider_name": providerName,
+		"instance_name": instanceName,
+		"model_name":    modelName,
+		"text":          text,
+	}
+
+	url := "/audio/speech"
+
+	resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to TTS document: %w", err)
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to TTS document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+	var result CommonResponse
+	if err = json.Unmarshal(resp.Body, &result); err != nil {
+		return nil, fmt.Errorf("TTS document failed: invalid JSON (%w)", err)
+	}
+	if result.Code != 0 {
+		return nil, fmt.Errorf("%s", result.Message)
+	}
+	result.Duration = resp.Duration
+
+	// save file
+	//err = os.WriteFile(fileToSave, resp.Body, 0644)
+	//if err != nil {
+	//	result.Message += fmt.Sprintf("failed to save file: %s", err.Error())
+	//	result.Code = 1
+	//}
+
+	return &result, nil
+}
+
+func (c *RAGFlowClient) ASRUserCommand(cmd *Command) (ResponseIf, error) {
+	if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+		return nil, fmt.Errorf("API token not set. Please login first")
+	}
+
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	var providerName, instanceName, modelName string
+
+	// Check if composite_model_name is provided in command
+	if compositeModelName, ok := cmd.Params["composite_model_name"].(string); ok && compositeModelName != "" {
+		names := strings.Split(compositeModelName, "@")
+		if len(names) != 3 {
+			return nil, fmt.Errorf("model name must be in format 'model@instance@provider'")
+		}
+		providerName = names[2]
+		instanceName = names[1]
+		modelName = names[0]
+	} else if c.CurrentModel != nil {
+		// Use current model if set
+		providerName = c.CurrentModel.Provider
+		instanceName = c.CurrentModel.Instance
+		modelName = c.CurrentModel.Model
+	} else {
+		return nil, fmt.Errorf("model name not provided and no current model set. Use 'use model' command first")
+	}
+
+	audioFile, ok := cmd.Params["audio_file"].(string)
+	if !ok {
+		return nil, fmt.Errorf("text not provided")
+	}
+
+	payload := map[string]interface{}{
+		"provider_name": providerName,
+		"instance_name": instanceName,
+		"model_name":    modelName,
+		"audio_file":    audioFile,
+	}
+
+	url := "/audio/transcriptions"
+
+	resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to ASR document: %w", err)
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to ASR document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+	var result CommonResponse
+	if err = json.Unmarshal(resp.Body, &result); err != nil {
+		return nil, fmt.Errorf("ASR document failed: invalid JSON (%w)", err)
+	}
+	if result.Code != 0 {
+		return nil, fmt.Errorf("%s", result.Message)
+	}
+	result.Duration = resp.Duration
+
+	return &result, nil
+}
+
+func (c *RAGFlowClient) OCRUserCommand(cmd *Command) (ResponseIf, error) {
+	if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
+		return nil, fmt.Errorf("API token not set. Please login first")
+	}
+
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	var providerName, instanceName, modelName string
+
+	// Check if composite_model_name is provided in command
+	if compositeModelName, ok := cmd.Params["composite_model_name"].(string); ok && compositeModelName != "" {
+		names := strings.Split(compositeModelName, "@")
+		if len(names) != 3 {
+			return nil, fmt.Errorf("model name must be in format 'model@instance@provider'")
+		}
+		providerName = names[2]
+		instanceName = names[1]
+		modelName = names[0]
+	} else if c.CurrentModel != nil {
+		// Use current model if set
+		providerName = c.CurrentModel.Provider
+		instanceName = c.CurrentModel.Instance
+		modelName = c.CurrentModel.Model
+	} else {
+		return nil, fmt.Errorf("model name not provided and no current model set. Use 'use model' command first")
+	}
+
+	filename, ok := cmd.Params["file"].(string)
+	if !ok {
+		return nil, fmt.Errorf("text not provided")
+	}
+
+	// read file and convert to base64
+	text, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file: %w", err)
+	}
+	base64Text := base64.StdEncoding.EncodeToString(text)
+	payload := map[string]interface{}{
+		"provider_name": providerName,
+		"instance_name": instanceName,
+		"model_name":    modelName,
+		"content":       base64Text,
+	}
+
+	url := "/file/ocr"
+
+	resp, err := c.HTTPClient.Request("POST", url, "web", nil, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to OCR document: %w", err)
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to OCR document: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+	var result CommonResponse
+	if err = json.Unmarshal(resp.Body, &result); err != nil {
+		return nil, fmt.Errorf("OCR document failed: invalid JSON (%w)", err)
+	}
+	if result.Code != 0 {
+		return nil, fmt.Errorf("%s", result.Message)
+	}
+	result.Duration = resp.Duration
+
+	return &result, nil
+}
+
 func (c *RAGFlowClient) CheckProviderConnection(cmd *Command) (ResponseIf, error) {
 	if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
 		return nil, fmt.Errorf("API token not set. Please login first")
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -2587,16 +2587,29 @@ func (p *Parser) parseStreamCommand() (*Command, error) {
 	var command *Command
 	var err error

-	if p.curToken.Type == TokenChat {
+	switch p.curToken.Type {
+	case TokenChat:
 		command, err = p.parseChatCommand()
 		if err != nil {
 			return nil, err
 		}
-	} else if p.curToken.Type == TokenThink {
+	case TokenThink:
 		command, err = p.parseThinkCommand()
 		if err != nil {
 			return nil, err
 		}
+	case TokenASR:
+		command, err = p.parseASRCommand()
+		if err != nil {
+			return nil, err
+		}
+	case TokenTTS:
+		command, err = p.parseTTSCommand()
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("expected CHAT, THINK, ASR, or TTS after STREAM")
 	}

 	command.Params["stream"] = true
@@ -2723,6 +2736,109 @@ documentLoop:
 	return cmd, nil
 }

+func (p *Parser) parseASRCommand() (*Command, error) {
+	p.nextToken() // consume ASR
+
+	if p.curToken.Type != TokenWith {
+		return nil, fmt.Errorf("expected WITH after ASR")
+	}
+	p.nextToken() // consume WITH
+
+	compositeModelName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenAudio {
+		return nil, fmt.Errorf("expected AUDIO to ASR")
+	}
+	p.nextToken() // consume FILE
+
+	audioFile, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	// Semicolon is optional for UNSET TOKEN
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("asr_user_command")
+	cmd.Params["composite_model_name"] = compositeModelName
+	cmd.Params["audio_file"] = audioFile
+	return cmd, nil
+}
+
+func (p *Parser) parseTTSCommand() (*Command, error) {
+	p.nextToken() // consume TTS
+
+	if p.curToken.Type != TokenWith {
+		return nil, fmt.Errorf("expected WITH after TTS")
+	}
+	p.nextToken() // consume WITH
+
+	compositeModelName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenText {
+		return nil, fmt.Errorf("expected TEXT to TTS")
+	}
+	p.nextToken() // consume FILE
+
+	text, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	// Semicolon is optional for UNSET TOKEN
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("tts_user_command")
+	cmd.Params["composite_model_name"] = compositeModelName
+	cmd.Params["text"] = text
+	return cmd, nil
+}
+
+func (p *Parser) parseOCRCommand() (*Command, error) {
+	p.nextToken() // consume OCR
+
+	if p.curToken.Type != TokenWith {
+		return nil, fmt.Errorf("expected WITH after OCR")
+	}
+	p.nextToken() // consume WITH
+
+	compositeModelName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenFile {
+		return nil, fmt.Errorf("expected FILE to OCR")
+	}
+	p.nextToken() // consume FILE
+
+	file, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+	p.nextToken()
+
+	cmd := NewCommand("ocr_user_command")
+	cmd.Params["composite_model_name"] = compositeModelName
+	cmd.Params["file"] = file
+	return cmd, nil
+}
+
 func (p *Parser) parseCheckCommand() (*Command, error) {
 	p.nextToken() // consume CHECK