Go: add ASR, TTS, OCR command (#14836)

### What problem does this PR solve? ``` RAGFlow(user)> asr with 'glm-asr-2512@test@zhipu-ai' audio './speech.wav'; CLI error: zhipu, no such method RAGFlow(user)> stream asr with 'glm-asr-2512@test@zhipu-ai' audio './speech.wav'; CLI error: zhipu, no such method RAGFlow(user)> tts with 'glm-tts@test@zhipu-ai' text 'how are you'; CLI error: zhipu, no such method RAGFlow(user)> stream tts with 'glm-tts@test@zhipu-ai' text 'how are you'; CLI error: zhipu, no such method RAGFlow(user)> ocr with 'glm-ocr@test@zhipu-ai' file './test.log'; CLI error: zhipu, no such method ``` ### Type of change - [x] New Feature (non-breaking change which adds functionality) Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2026-08-02 13:57:30 +08:00 · 2026-05-12 17:17:44 +08:00
parent 9ee481807f
commit d08bf02d9b
32 changed files with 1869 additions and 71 deletions
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -1047,3 +1047,311 @@ func (h *ProviderHandler) RerankDocument(c *gin.Context) {
 		"message": "success",
 	})
 }
+
+type TranscribeAudioRequest struct {
+	ProviderName *string  `json:"provider_name"`
+	InstanceName *string  `json:"instance_name"`
+	ModelName    *string  `json:"model_name"`
+	File         *string  `json:"file"`
+	Language     []string `json:"language"`
+	Prompt       int      `json:"prompt"`
+	Stream       bool     `json:"stream"`
+}
+
+func (h *ProviderHandler) TranscribeAudio(c *gin.Context) {
+	var req TranscribeAudioRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		println("JSON bind error: %v (type: %T)", err, err)
+		c.JSON(http.StatusOK, gin.H{
+			"code":    common.CodeBadRequest,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	if req.ProviderName == nil || *req.ProviderName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Provider name is required",
+		})
+		return
+	}
+
+	if req.InstanceName == nil || *req.InstanceName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Instance name is required",
+		})
+		return
+	}
+
+	if req.ModelName == nil || *req.ModelName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Model name is required",
+		})
+		return
+	}
+
+	userID := c.GetString("user_id")
+
+	apiConfig := models.APIConfig{
+		ApiKey: nil,
+		Region: nil,
+	}
+
+	asrConfig := models.ASRConfig{}
+
+	// Check if it's a stream request
+	if req.Stream {
+		// Set SSE headers
+		c.Header("Content-Type", "text/event-stream")
+		c.Header("Cache-Control", "no-cache")
+		c.Header("Connection", "keep-alive")
+		c.Writer.WriteHeader(http.StatusOK)
+		c.Writer.Flush()
+
+		// Create sender function that writes directly to response
+		sender := func(content, reasoningContent *string) error {
+			// Check for [DONE] marker (OpenAI compatible)
+			if content != nil {
+				if *content == "[DONE]" {
+					c.SSEvent("done", "[DONE]")
+					return nil
+				}
+				message := fmt.Sprintf("[MESSAGE]%s", *content)
+				c.SSEvent("message", message)
+				c.Writer.Flush()
+			}
+
+			if reasoningContent != nil {
+				message := fmt.Sprintf("[REASONING]%s", *reasoningContent)
+				c.SSEvent("message", message)
+				c.Writer.Flush()
+			}
+
+			//logger.Info(data)
+			return nil
+		}
+
+		// Stream response using sender function (best performance, no channel)
+		errorCode, err := h.modelProviderService.TranscribeAudioStream(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.File, &apiConfig, &asrConfig, sender)
+
+		if errorCode != common.CodeSuccess {
+			c.SSEvent("error", err.Error())
+		}
+		return
+	}
+
+	// Non-stream response
+	var response *models.ASRResponse
+	var errorCode common.ErrorCode
+	var err error
+
+	response, errorCode, err = h.modelProviderService.TranscribeAudio(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.File, &apiConfig, &asrConfig)
+
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{
+			"code":    errorCode,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"data":    response,
+		"message": "success",
+	})
+}
+
+type AudioSpeechRequest struct {
+	ProviderName *string  `json:"provider_name"`
+	InstanceName *string  `json:"instance_name"`
+	ModelName    *string  `json:"model_name"`
+	Text         *string  `json:"text"`
+	Language     []string `json:"language"`
+	Voice        int      `json:"voice"`
+	Stream       bool     `json:"stream"`
+	Volume       bool     `json:"volume"`
+}
+
+func (h *ProviderHandler) AudioSpeech(c *gin.Context) {
+	var req AudioSpeechRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		println("JSON bind error: %v (type: %T)", err, err)
+		c.JSON(http.StatusOK, gin.H{
+			"code":    common.CodeBadRequest,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	if req.ProviderName == nil || *req.ProviderName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Provider name is required",
+		})
+		return
+	}
+
+	if req.InstanceName == nil || *req.InstanceName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Instance name is required",
+		})
+		return
+	}
+
+	if req.ModelName == nil || *req.ModelName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Model name is required",
+		})
+		return
+	}
+
+	userID := c.GetString("user_id")
+
+	apiConfig := models.APIConfig{
+		ApiKey: nil,
+		Region: nil,
+	}
+
+	ttsConfig := models.TTSConfig{}
+
+	// Check if it's a stream request
+	if req.Stream {
+		// Set SSE headers
+		c.Header("Content-Type", "text/event-stream")
+		c.Header("Cache-Control", "no-cache")
+		c.Header("Connection", "keep-alive")
+		c.Writer.WriteHeader(http.StatusOK)
+		c.Writer.Flush()
+
+		// Create sender function that writes directly to response
+		sender := func(content, reasoningContent *string) error {
+			// Check for [DONE] marker (OpenAI compatible)
+			if content != nil {
+				if *content == "[DONE]" {
+					c.SSEvent("done", "[DONE]")
+					return nil
+				}
+				message := fmt.Sprintf("[MESSAGE]%s", *content)
+				c.SSEvent("message", message)
+				c.Writer.Flush()
+			}
+
+			if reasoningContent != nil {
+				message := fmt.Sprintf("[REASONING]%s", *reasoningContent)
+				c.SSEvent("message", message)
+				c.Writer.Flush()
+			}
+
+			//logger.Info(data)
+			return nil
+		}
+
+		// Stream response using sender function (best performance, no channel)
+		errorCode, err := h.modelProviderService.AudioSpeechStream(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.Text, &apiConfig, &ttsConfig, sender)
+
+		if errorCode != common.CodeSuccess {
+			c.SSEvent("error", err.Error())
+		}
+		return
+	}
+
+	// Non-stream response
+	var response *models.TTSResponse
+	var errorCode common.ErrorCode
+	var err error
+
+	response, errorCode, err = h.modelProviderService.AudioSpeech(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.Text, &apiConfig, &ttsConfig)
+
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{
+			"code":    errorCode,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"data":    response,
+		"message": "success",
+	})
+}
+
+type OCRFileRequest struct {
+	ProviderName *string `json:"provider_name"`
+	InstanceName *string `json:"instance_name"`
+	ModelName    *string `json:"model_name"`
+	File         *string `json:"file"`
+}
+
+func (h *ProviderHandler) OCRFile(c *gin.Context) {
+	var req OCRFileRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		println("JSON bind error: %v (type: %T)", err, err)
+		c.JSON(http.StatusOK, gin.H{
+			"code":    common.CodeBadRequest,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	if req.ProviderName == nil || *req.ProviderName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Provider name is required",
+		})
+		return
+	}
+
+	if req.InstanceName == nil || *req.InstanceName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Instance name is required",
+		})
+		return
+	}
+
+	if req.ModelName == nil || *req.ModelName == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "Model name is required",
+		})
+		return
+	}
+
+	userID := c.GetString("user_id")
+
+	apiConfig := models.APIConfig{
+		ApiKey: nil,
+		Region: nil,
+	}
+
+	OCRConfig := models.OCRConfig{}
+
+	// Non-stream response
+	var response *models.OCRResponse
+	var errorCode common.ErrorCode
+	var err error
+
+	response, errorCode, err = h.modelProviderService.OCRFile(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.File, &apiConfig, &OCRConfig)
+
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{
+			"code":    errorCode,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"data":    response,
+		"message": "success",
+	})
+}