diff --git a/conf/models/siliconflow.json b/conf/models/siliconflow.json index 4da3e0dcab..320d21aba5 100644 --- a/conf/models/siliconflow.json +++ b/conf/models/siliconflow.json @@ -8,7 +8,9 @@ "models": "models", "embedding": "embeddings", "rerank": "rerank", - "balance": "user/info" + "balance": "user/info", + "tts": "audio/speech", + "asr": "audio/transcriptions" }, "models": [ { @@ -45,6 +47,27 @@ "model_types": [ "embedding" ] + }, + { + "name": "fnlp/MOSS-TTSD-v0.5", + "max_tokens": 8192, + "model_types": [ + "tts" + ] + }, + { + "name": "FunAudioLLM/CosyVoice2-0.5B", + "max_tokens": 8192, + "model_types": [ + "tts" + ] + }, + { + "name": "FunAudioLLM/SenseVoiceSmall", + "max_tokens": 8192, + "model_types": [ + "asr" + ] } ] } diff --git a/conf/models/stepfun.json b/conf/models/stepfun.json index f13b227a49..34898c5a1b 100644 --- a/conf/models/stepfun.json +++ b/conf/models/stepfun.json @@ -5,7 +5,8 @@ }, "url_suffix": { "chat": "chat/completions", - "models": "models" + "models": "models", + "tts": "audio/speech" }, "class": "step", "models": [ @@ -88,6 +89,27 @@ "chat", "vision" ] + }, + { + "name": "step-tts-2 ", + "max_tokens": 8192, + "model_types": [ + "tts" + ] + }, + { + "name": "stepaudio-2.5-tts", + "max_tokens": 8192, + "model_types": [ + "tts" + ] + }, + { + "name": "step-tts-mini", + "max_tokens": 8192, + "model_types": [ + "tts" + ] } ] } diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go index 1d4cc970ae..7ead18285c 100644 --- a/internal/cli/user_command.go +++ b/internal/cli/user_command.go @@ -2112,7 +2112,10 @@ func (c *RAGFlowClient) TTSUserCommand(cmd *Command) (ResponseIf, error) { shouldSave, _ := cmd.Params["save"].(bool) saveDir, _ := cmd.Params["save_path"].(string) - fileName := fmt.Sprintf("%s_output.%s", modelName, explicitFormat) + // format file name + safeModelName := strings.ReplaceAll(modelName, "/", "_") + safeModelName = strings.ReplaceAll(safeModelName, ":", "-") + fileName := fmt.Sprintf("%s_output.%s", safeModelName, explicitFormat) cwd, err := os.Getwd() if err != nil { @@ -2252,7 +2255,9 @@ func (c *RAGFlowClient) ASRUserCommand(cmd *Command) (ResponseIf, error) { var result CommonResponse result.Code = rawResult.Code - result.Message = rawResult.Data["text"].(string) // TODO + result.Data = []map[string]interface{}{ + {"text": rawResult.Data["text"].(string)}, + } result.Duration = resp.Duration return &result, nil diff --git a/internal/development.md b/internal/development.md index c477e7a732..7e31a8df3e 100644 --- a/internal/development.md +++ b/internal/development.md @@ -358,9 +358,18 @@ RAGFlow(user)> list datasets; ``` ### 6.23 Text to Speech - ``` RAGFlow(user)> tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}' Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav SUCCESS ``` + +### 6.24 Audio to Speech +``` +RAGFlow(user)> asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './internal/test.wav' param '' ++----------------------------------------------------------------------------------------------------------------------+ +| text | ++----------------------------------------------------------------------------------------------------------------------+ +| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. | ++----------------------------------------------------------------------------------------------------------------------+ +``` \ No newline at end of file diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go index 8a823c2ba2..552ef90361 100644 --- a/internal/entity/models/siliconflow.go +++ b/internal/entity/models/siliconflow.go @@ -22,7 +22,10 @@ import ( "encoding/json" "fmt" "io" + "mime/multipart" "net/http" + "os" + "path/filepath" "ragflow/internal/common" "strconv" "strings" @@ -723,7 +726,119 @@ func (s *SiliconflowModel) Rerank(modelName *string, query string, documents []s // TranscribeAudio transcribe audio func (o *SiliconflowModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) { - return nil, fmt.Errorf("%s, no such method", o.Name()) + if file == nil || *file == "" { + return nil, fmt.Errorf("file is missing") + } + + region := "default" + if apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.ASR) + + // multipart body + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + // open audio file + audioFile, err := os.Open(*file) + if err != nil { + return nil, fmt.Errorf("failed to open audio file: %w", err) + } + defer audioFile.Close() + + // create multipart file field + part, err := writer.CreateFormFile( + "file", + filepath.Base(*file), + ) + if err != nil { + return nil, fmt.Errorf("failed to create multipart file: %w", err) + } + + // copy file content + if _, err = io.Copy(part, audioFile); err != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", err) + } + + // model field + if err := writer.WriteField("model", *modelName); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + // extra params + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + + var val string + + switch v := value.(type) { + case string: + val = v + case bool: + val = strconv.FormatBool(v) + case int: + val = strconv.Itoa(v) + case int64: + val = strconv.FormatInt(v, 10) + case float32: + val = strconv.FormatFloat(float64(v), 'f', -1, 32) + case float64: + val = strconv.FormatFloat(v, 'f', -1, 64) + default: + val = fmt.Sprintf("%v", v) + } + + if err = writer.WriteField(key, val); err != nil { + return nil, fmt.Errorf("failed to write field %s: %w", key, err) + } + } + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + // build request + req, err := http.NewRequest("POST", url, &body) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Accept", "application/json") + + // send request + resp, err := o.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("SiliconFlow ASR error: %s - %s", resp.Status, string(respBody)) + } + + // SiliconFlow response + var result struct { + Text string `json:"text"` + } + + if err = json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody)) + } + + var res ASRResponse + res.Text = result.Text + + return &ASRResponse{Text: result.Text}, nil } func (z *SiliconflowModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error { @@ -732,11 +847,138 @@ func (z *SiliconflowModel) TranscribeAudioWithSender(modelName *string, file *st // AudioSpeech convert audio to text func (o *SiliconflowModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) { - return nil, fmt.Errorf("%s, no such method", o.Name()) + if audioContent == nil || *audioContent == "" { + return nil, fmt.Errorf("audio content is empty") + } + + var region = "default" + if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + url := fmt.Sprintf("%s/%s", o.BaseURL[region], o.URLSuffix.TTS) + + reqBody := map[string]interface{}{ + "model": *modelName, + "input": *audioContent, + "stream": false, + } + + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + reqBody[key] = value + } + } + if asrConfig != nil && asrConfig.Format != "" { + reqBody["response_format"] = asrConfig.Format + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + + resp, err := o.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s - %s", resp.Status, string(body)) + } + + return &TTSResponse{Audio: body}, nil } func (z *SiliconflowModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error { - return fmt.Errorf("%s, no such method", z.Name()) + if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { + return fmt.Errorf("SiliconFlow API key is missing") + } + + if audioContent == nil || *audioContent == "" { + return fmt.Errorf("audio content is empty") + } + + var region = "default" + if apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.TTS) + + reqBody := map[string]interface{}{ + "model": *modelName, + "input": *audioContent, + "stream": true, + } + + if ttsConfig != nil && ttsConfig.Params != nil { + for key, value := range ttsConfig.Params { + reqBody[key] = value + } + } + if ttsConfig != nil && ttsConfig.Format != "" { + reqBody["response_format"] = ttsConfig.Format + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + + resp, err := z.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("SiliconFlow stream TTS API error: %d, body: %s", resp.StatusCode, string(body)) + } + + buf := make([]byte, 32*1024) + + for { + n, err := resp.Body.Read(buf) + if n > 0 { + chunk := string(buf[:n]) + if errSend := sender(&chunk, nil); errSend != nil { + return errSend + } + } + + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("error reading SiliconFlow binary audio stream: %w", err) + } + } + + return nil } // OCRFile OCR file diff --git a/internal/entity/models/stepfun.go b/internal/entity/models/stepfun.go index 30d226e250..941f815fd2 100644 --- a/internal/entity/models/stepfun.go +++ b/internal/entity/models/stepfun.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -68,6 +69,15 @@ func NewStepFunModel(baseURL map[string]string, urlSuffix URLSuffix) *StepFunMod } } +/* + +RAGFlow(user)> tts with 'fnlp/MOSS-TTSD-v0.5@test@siliconflow' text 'He who desires but acts not, breeds pestilence.' play format 'wav' param '{"voice": "fnlp/MOSS-TTSD-v0.5:alex"}' +SUCCESS +RAGFlow(user)> stream tts with 'fnlp/MOSS-TTSD-v0.5@test@siliconflow' text 'He who desires but acts not, breeds pestilence.' play format 'wav' param '{"voice": "fnlp/MOSS-TTSD-v0.5:claire"}' +SUCCESS + +*/ + func (s *StepFunModel) NewInstance(baseURL map[string]string) ModelDriver { return NewStepFunModel(baseURL, s.URLSuffix) } @@ -459,21 +469,177 @@ func (s *StepFunModel) Rerank(modelName *string, query string, documents []strin } // TranscribeAudio transcribe audio -func (z *StepFunModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) { - return nil, fmt.Errorf("%s, no such method", z.Name()) +func (s *StepFunModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) { + return nil, fmt.Errorf("%s, no such method", s.Name()) } -func (z *StepFunModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error { - return fmt.Errorf("%s, no such method", z.Name()) +func (s *StepFunModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error { + return fmt.Errorf("%s, no such method", s.Name()) } // AudioSpeech convert audio to text -func (z *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) { - return nil, fmt.Errorf("%s, no such method", z.Name()) +func (s *StepFunModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) { + // TODO Test it + if audioContent == nil || *audioContent == "" { + return nil, fmt.Errorf("audio content is empty") + } + + var region = "default" + if apiConfig != nil && apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + url := fmt.Sprintf("%s/%s", s.BaseURL[region], s.URLSuffix.TTS) + + reqBody := map[string]interface{}{ + "model": *modelName, + "input": *audioContent, + } + + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + reqBody[key] = value + } + } + if asrConfig != nil && asrConfig.Format != "" { + reqBody["response_format"] = asrConfig.Format + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s - %s", resp.Status, string(body)) + } + + return &TTSResponse{Audio: body}, nil } -func (z *StepFunModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error { - return fmt.Errorf("%s, no such method", z.Name()) +// AudioSpeechWithSender for Streaming TTS +func (s *StepFunModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error { + // TODO Test it + if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { + return fmt.Errorf("StepFun API key is missing") + } + + if audioContent == nil || *audioContent == "" { + return fmt.Errorf("audio content is empty") + } + + var region = "default" + if apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + url := fmt.Sprintf("%s/%s", s.BaseURL[region], s.URLSuffix.TTS) + + reqBody := map[string]interface{}{ + "model": *modelName, + "input": *audioContent, + "stream_format": "sse", + } + + if ttsConfig != nil && ttsConfig.Params != nil { + for key, value := range ttsConfig.Params { + reqBody[key] = value + } + } + if ttsConfig != nil && ttsConfig.Format != "" { + reqBody["response_format"] = ttsConfig.Format + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + + resp, err := s.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("StepFun stream TTS API error: %d - %s", resp.StatusCode, string(body)) + } + + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(make([]byte, 64*1024), 8*1024*1024) + + for scanner.Scan() { + line := scanner.Text() + + if !strings.HasPrefix(line, "data: ") { + continue + } + + dataStr := strings.TrimSpace(line[6:]) + // [DONE] + if dataStr == "" || dataStr == "[DONE]" { + continue + } + + // Parse + var event struct { + Type string `json:"type"` + Audio string `json:"audio"` + } + + if err := json.Unmarshal([]byte(dataStr), &event); err != nil { + continue + } + + if event.Type == "speech.audio.error" { + return fmt.Errorf("StepFun stream encountered an error during generation") + } + + // Extract the Base64 string containing the audio and decode it + if event.Type == "speech.audio.delta" && event.Audio != "" { + audioBytes, err := base64.StdEncoding.DecodeString(event.Audio) + if err == nil && len(audioBytes) > 0 { + chunk := string(audioBytes) + if errSend := sender(&chunk, nil); errSend != nil { + return errSend + } + } + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading StepFun stream: %w", err) + } + + return nil } // OCRFile OCR file