mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-06-29 15:31:05 +08:00
Go: implement TTS and ASR for X.AI (#15247)
### What problem does this PR solve? As title ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [x] Refactoring
This commit is contained in:
@@ -5,39 +5,60 @@
|
||||
},
|
||||
"url_suffix": {
|
||||
"chat": "chat/completions",
|
||||
"models": "models"
|
||||
"models": "models",
|
||||
"tts": "tts",
|
||||
"ast": "stt"
|
||||
},
|
||||
"class": "grok",
|
||||
"models": [
|
||||
{
|
||||
"name": "grok-4",
|
||||
"max_tokens": 256000,
|
||||
"model_types": ["chat"]
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grok-3",
|
||||
"max_tokens": 131072,
|
||||
"model_types": ["chat"]
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grok-3-fast",
|
||||
"max_tokens": 131072,
|
||||
"model_types": ["chat"]
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grok-3-mini",
|
||||
"max_tokens": 131072,
|
||||
"model_types": ["chat"]
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grok-3-mini-mini-fast",
|
||||
"max_tokens": 131072,
|
||||
"model_types": ["chat"]
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grok-2-vision",
|
||||
"max_tokens": 32768,
|
||||
"model_types": ["vision"]
|
||||
"model_types": [
|
||||
"vision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "eve",
|
||||
"model_types": [
|
||||
"tts"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ func (m *MinerUModel) NewInstance(baseURL map[string]string) ModelDriver {
|
||||
}
|
||||
|
||||
func (m *MinerUModel) Name() string {
|
||||
return "mineru"
|
||||
return "mineru.net"
|
||||
}
|
||||
|
||||
func (m *MinerUModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
|
||||
|
||||
@@ -49,7 +49,7 @@ func (m *MinerULocalModel) NewInstance(baseURL map[string]string) ModelDriver {
|
||||
}
|
||||
|
||||
func (m *MinerULocalModel) Name() string {
|
||||
return "mineru_local"
|
||||
return "mineru"
|
||||
}
|
||||
|
||||
func (m *MinerULocalModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
|
||||
|
||||
@@ -51,7 +51,7 @@ func (p PaddleOCRModel) NewInstance(baseURL map[string]string) ModelDriver {
|
||||
}
|
||||
|
||||
func (p *PaddleOCRModel) Name() string {
|
||||
return "paddle_ocr"
|
||||
return "paddle_ocr.net"
|
||||
}
|
||||
|
||||
func (p *PaddleOCRModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
|
||||
|
||||
@@ -50,7 +50,7 @@ func (p *PaddleOCRLocalModel) NewInstance(baseURL map[string]string) ModelDriver
|
||||
}
|
||||
|
||||
func (p *PaddleOCRLocalModel) Name() string {
|
||||
return "paddleocr_local"
|
||||
return "paddleocr"
|
||||
}
|
||||
|
||||
func (p *PaddleOCRLocalModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
|
||||
|
||||
@@ -23,7 +23,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -499,7 +503,127 @@ func (z *XAIModel) Rerank(modelName *string, query string, documents []string, a
|
||||
|
||||
// TranscribeAudio transcribe audio
|
||||
func (o *XAIModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
|
||||
return nil, fmt.Errorf("%s, no such method", o.Name())
|
||||
if file == nil || *file == "" {
|
||||
return nil, fmt.Errorf("file is missing")
|
||||
}
|
||||
|
||||
region := "default"
|
||||
if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" {
|
||||
region = *apiConfig.Region
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.ASR)
|
||||
|
||||
// multipart body
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
|
||||
// 1. model field
|
||||
if modelName != nil && *modelName != "" {
|
||||
if err := writer.WriteField("model", *modelName); err != nil {
|
||||
return nil, fmt.Errorf("failed to write model field: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. extra params
|
||||
if asrConfig != nil && asrConfig.Params != nil {
|
||||
for key, value := range asrConfig.Params {
|
||||
|
||||
switch v := value.(type) {
|
||||
case []interface{}:
|
||||
for _, item := range v {
|
||||
if err := writer.WriteField(key, fmt.Sprintf("%v", item)); err != nil {
|
||||
return nil, fmt.Errorf("failed to write array field %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
for _, item := range v {
|
||||
if err := writer.WriteField(key, item); err != nil {
|
||||
return nil, fmt.Errorf("failed to write array field %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
default:
|
||||
var val string
|
||||
switch v2 := value.(type) {
|
||||
case string:
|
||||
val = v2
|
||||
case bool:
|
||||
val = strconv.FormatBool(v2)
|
||||
case int:
|
||||
val = strconv.Itoa(v2)
|
||||
case int64:
|
||||
val = strconv.FormatInt(v2, 10)
|
||||
case float32:
|
||||
val = strconv.FormatFloat(float64(v2), 'f', -1, 32)
|
||||
case float64:
|
||||
val = strconv.FormatFloat(v2, 'f', -1, 64)
|
||||
default:
|
||||
val = fmt.Sprintf("%v", v2)
|
||||
}
|
||||
|
||||
if err := writer.WriteField(key, val); err != nil {
|
||||
return nil, fmt.Errorf("failed to write field %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// open audio file
|
||||
audioFile, err := os.Open(*file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open audio file: %w", err)
|
||||
}
|
||||
defer audioFile.Close()
|
||||
|
||||
// create multipart file field
|
||||
part, err := writer.CreateFormFile("file", filepath.Base(*file))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create multipart file: %w", err)
|
||||
}
|
||||
|
||||
// copy file content
|
||||
if _, err = io.Copy(part, audioFile); err != nil {
|
||||
return nil, fmt.Errorf("failed to copy audio data: %w", err)
|
||||
}
|
||||
if err = writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
// build request
|
||||
req, err := http.NewRequest("POST", url, &body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
// send request
|
||||
resp, err := o.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("xAI ASR API error: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
// xAI response parsing (assuming standard format matching OpenAI)
|
||||
var result struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(respBody, &result); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody))
|
||||
}
|
||||
|
||||
return &ASRResponse{Text: result.Text}, nil
|
||||
}
|
||||
|
||||
func (z *XAIModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
|
||||
@@ -508,7 +632,66 @@ func (z *XAIModel) TranscribeAudioWithSender(modelName *string, file *string, ap
|
||||
|
||||
// AudioSpeech convert text to audio
|
||||
func (o *XAIModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
|
||||
return nil, fmt.Errorf("%s, no such method", o.Name())
|
||||
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
|
||||
return nil, fmt.Errorf("xai API key is missing")
|
||||
}
|
||||
|
||||
if audioContent == nil || *audioContent == "" {
|
||||
return nil, fmt.Errorf("text content is missing")
|
||||
}
|
||||
|
||||
var region = "default"
|
||||
if apiConfig.Region != nil && *apiConfig.Region != "" {
|
||||
region = *apiConfig.Region
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.TTS)
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"text": *audioContent,
|
||||
"voice_id": modelName,
|
||||
}
|
||||
|
||||
if ttsConfig != nil && ttsConfig.Params != nil {
|
||||
for key, value := range ttsConfig.Params {
|
||||
reqBody[key] = value
|
||||
}
|
||||
}
|
||||
if ttsConfig != nil && ttsConfig.Format != "" {
|
||||
reqBody["output_format"] = map[string]interface{}{
|
||||
"codec": ttsConfig.Format,
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
|
||||
|
||||
resp, err := o.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%s - %s", resp.Status, string(body))
|
||||
}
|
||||
|
||||
return &TTSResponse{Audio: body}, nil
|
||||
}
|
||||
|
||||
func (z *XAIModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
|
||||
|
||||
Reference in New Issue
Block a user