diff --git a/conf/models/longcat.json b/conf/models/longcat.json index b0f41d9ce0..00896e7b85 100644 --- a/conf/models/longcat.json +++ b/conf/models/longcat.json @@ -10,47 +10,7 @@ "class": "longcat", "models": [ { - "name": "LongCat-Flash-Chat", - "max_tokens": 131072, - "model_types": [ - "chat" - ], - "tools": { - "support": true - } - }, - { - "name": "LongCat-Flash-Lite", - "max_tokens": 131072, - "model_types": [ - "chat" - ], - "tools": { - "support": true - } - }, - { - "name": "LongCat-Flash-Thinking-2601", - "max_tokens": 131072, - "model_types": [ - "chat" - ], - "tools": { - "support": true - } - }, - { - "name": "LongCat-Flash-Omni-2603", - "max_tokens": 131072, - "model_types": [ - "chat" - ], - "tools": { - "support": true - } - }, - { - "name": "LongCat-2.0-Preview", + "name": "LongCat-2.0", "max_tokens": 131072, "model_types": [ "chat" diff --git a/conf/models/mistral.json b/conf/models/mistral.json index aac4c14f82..fd016e49ca 100644 --- a/conf/models/mistral.json +++ b/conf/models/mistral.json @@ -7,6 +7,8 @@ "chat": "v1/chat/completions", "models": "v1/models", "embedding": "v1/embeddings", + "asr": "v1/audio/transcriptions", + "tts": "v1/audio/speech", "ocr": "v1/ocr" }, "class": "mistral", @@ -21,6 +23,18 @@ "support": true } }, + { + "name": "voxtral-mini-tts-2603", + "model_types": [ + "tts" + ] + }, + { + "name": "voxtral-mini-transcribe-realtime-2602", + "model_types": [ + "asr" + ] + }, { "name": "mistral-medium-latest", "max_tokens": 128000, diff --git a/internal/entity/models/baidu.go b/internal/entity/models/baidu.go index f40af83afa..8a31d77bf6 100644 --- a/internal/entity/models/baidu.go +++ b/internal/entity/models/baidu.go @@ -50,6 +50,37 @@ func (b *BaiduModel) Name() string { return "BaiduYiyan" } +type BaiduChatResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []map[string]any `json:"tool_calls"` + ReasoningContent *string `json:"reasoning_content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + PromptTokensDetails struct { + SearchTokens int `json:"search_tokens"` + CachedTokens int `json:"cached_tokens"` + } `json:"prompt_tokens_details"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + SearchResult struct { + Index int `json:"index"` + URL string `json:"url"` + Title string `json:"title"` + } `json:"search_result"` +} + func (b *BaiduModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) { if err := b.baseModel.APIConfigCheck(apiConfig); err != nil { return nil, err @@ -142,65 +173,51 @@ func (b *BaiduModel) ChatWithMessages(ctx context.Context, modelName string, mes return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } - // Parse response - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } + return parseChatCompletionResponse(body, chatModelConfig, modelUsage, func(body []byte, chatConfig *ChatConfig) (chatResponseParts, error) { + var result BaiduChatResponse + if err := json.Unmarshal(body, &result); err != nil { + return chatResponseParts{}, fmt.Errorf("failed to parse response: %w", err) + } + if len(result.Choices) == 0 { + return chatResponseParts{}, fmt.Errorf("no choices in response") + } - choices, ok := result["choices"].([]interface{}) - if !ok || len(choices) == 0 { - return nil, fmt.Errorf("no choices in response") - } - - firstChoice, ok := choices[0].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid choice format") - } - - messageMap, ok := firstChoice["message"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid message format") - } - - content, ok := messageMap["content"].(string) - if !ok { - return nil, fmt.Errorf("invalid content format") - } - - var toolCalls []map[string]any - if tcs, ok := messageMap["tool_calls"].([]any); ok { - for _, tc := range tcs { - if toolCall, ok := tc.(map[string]any); ok { - toolCalls = append(toolCalls, toolCall) + choice := &result.Choices[0] + content := "" + if choice.Message.Content != nil { + content = *choice.Message.Content + } + reasonContent := "" + if chatConfig != nil && chatConfig.Thinking != nil && *chatConfig.Thinking { + if choice.Message.ReasoningContent != nil { + reasonContent = *choice.Message.ReasoningContent + } + if reasonContent != "" && reasonContent[0] == '\n' { + reasonContent = reasonContent[1:] } } - } - var reasonContent string - if chatModelConfig != nil && chatModelConfig.Thinking != nil && *chatModelConfig.Thinking { - reasonContent, ok = messageMap["reasoning_content"].(string) - if !ok { - return nil, fmt.Errorf("invalid reasoning content format") + totalTokens := result.Usage.TotalTokens + if totalTokens == 0 { + totalTokens = result.Usage.PromptTokens + result.Usage.CompletionTokens } - // if first char of reasonContent is \n remove the '\n' - if reasonContent != "" && reasonContent[0] == '\n' { - reasonContent = reasonContent[1:] + var usage *TokenUsage + if totalTokens > 0 { + usage = &TokenUsage{ + PromptTokens: result.Usage.PromptTokens, + CompletionTokens: result.Usage.CompletionTokens, + TotalTokens: totalTokens, + } } - } - chatResponse := &ChatResponse{ - Answer: &content, - ReasonContent: &reasonContent, - ToolCalls: toolCalls, - } - if pt, ct, tt := extractUsageFromMap(result); tt > 0 { - chatResponse.Usage = &TokenUsage{ - PromptTokens: pt, CompletionTokens: ct, TotalTokens: tt, - } - } - - return chatResponse, nil + return chatResponseParts{ + RequestID: result.ID, + Content: &content, + ReasonContent: &reasonContent, + ToolCalls: choice.Message.ToolCalls, + Usage: usage, + }, nil + }) } func (b *BaiduModel) ChatStreamlyWithSender(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { @@ -294,6 +311,14 @@ func (b *BaiduModel) ChatStreamlyWithSender(ctx context.Context, modelName strin done, err := ParseSSEStream[map[string]interface{}](resp.Body, func(event map[string]interface{}) error { common.Info(fmt.Sprintf("%v", event)) + tokenUsage, found, usageErr := decodeOpenAICompatibleStreamUsage(event) + if usageErr != nil { + return usageErr + } + if found { + applyStreamUsage(modelConfig, modelUsage, tokenUsage) + } + choices, ok := event["choices"].([]interface{}) if !ok || len(choices) == 0 { return nil diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go index 28c9cd27b5..aa84385a6e 100644 --- a/internal/entity/models/base_model.go +++ b/internal/entity/models/base_model.go @@ -47,7 +47,7 @@ type chatResponseParts struct { RequestID string Content *string ReasonContent *string - ToolCalls []map[string]interface{} + ToolCalls []map[string]any Usage *TokenUsage } @@ -323,10 +323,9 @@ func ReadErrorBody(r io.Reader) string { func buildRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any { reqBody := map[string]any{ - "model": modelName, - "messages": buildChatMessages(messages), - "stream": stream, - "temperature": 1, + "model": modelName, + "messages": buildChatMessages(messages), + "stream": stream, } if cfg != nil { diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go index e947ace845..1f4b2c0ba3 100644 --- a/internal/entity/models/minimax.go +++ b/internal/entity/models/minimax.go @@ -52,6 +52,53 @@ func (m *MinimaxModel) Name() string { return "minimax" } +type MinimaxChatResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []map[string]any `json:"tool_calls"` + ReasoningContent *string `json:"reasoning_content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + PromptTokensDetails struct { + CachedTokens int `json:"cached_tokens"` + } `json:"prompt_tokens_details"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + TotalCharacters int `json:"total_characters"` + } `json:"usage"` + BaseResp struct { + StatusCode int `json:"status_code"` + StatusMsg string `json:"status_msg"` + } `json:"base_resp"` + Error struct { + Message string `json:"message"` + Type string `json:"type"` + } `json:"error"` +} + +func extractMinimaxChatResponseError(result *MinimaxChatResponse) string { + if result == nil { + return "" + } + if result.BaseResp.StatusCode != 0 { + if result.BaseResp.StatusMsg != "" { + return result.BaseResp.StatusMsg + } + return fmt.Sprintf("status_code %d", result.BaseResp.StatusCode) + } + return result.Error.Message +} + func validateMinimaxModelName(modelName string) (string, error) { if strings.TrimSpace(modelName) == "" { return "", fmt.Errorf("model name is required") @@ -173,65 +220,56 @@ func (m *MinimaxModel) ChatWithMessages(ctx context.Context, modelName string, m return nil, fmt.Errorf("minimax API error: status %d: %s", resp.StatusCode, extractMinimaxErrorBody(body)) } - // Parse response - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %w", err) - } + return parseChatCompletionResponse(body, chatModelConfig, modelUsage, func(body []byte, chatConfig *ChatConfig) (chatResponseParts, error) { + var result MinimaxChatResponse + if err := json.Unmarshal(body, &result); err != nil { + return chatResponseParts{}, fmt.Errorf("failed to unmarshal response: %w", err) + } - // MiniMax can embed an error (rate limit, etc.) inside a 200 body. - if errMsg := extractMinimaxAPIError(result); errMsg != "" { - return nil, fmt.Errorf("minimax API error: %s", errMsg) - } + if errMsg := extractMinimaxChatResponseError(&result); errMsg != "" { + return chatResponseParts{}, fmt.Errorf("minimax API error: %s", errMsg) + } + if len(result.Choices) == 0 { + return chatResponseParts{}, fmt.Errorf("no choices in response") + } - choices, ok := result["choices"].([]interface{}) - if !ok || len(choices) == 0 { - return nil, fmt.Errorf("no choices in response") - } + choice := &result.Choices[0] + content := "" + if choice.Message.Content != nil { + content = *choice.Message.Content + } - firstChoice, ok := choices[0].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("no choices in response") - } - - messageMap, ok := firstChoice["message"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("no message in response") - } - - content, _ := messageMap["content"].(string) - - var reasonContent string - if chatModelConfig != nil && chatModelConfig.Thinking != nil && *chatModelConfig.Thinking { - if rc, ok := messageMap["reasoning_content"].(string); ok { - reasonContent = rc + reasonContent := "" + if chatConfig != nil && chatConfig.Thinking != nil && *chatConfig.Thinking { + if choice.Message.ReasoningContent != nil { + reasonContent = *choice.Message.ReasoningContent + } if reasonContent != "" && reasonContent[0] == '\n' { reasonContent = reasonContent[1:] } } - } - var toolCalls []map[string]any - if tcs, ok := messageMap["tool_calls"].([]any); ok { - for _, tc := range tcs { - if tcMap, ok := tc.(map[string]any); ok { - toolCalls = append(toolCalls, tcMap) + totalTokens := result.Usage.TotalTokens + if totalTokens == 0 { + totalTokens = result.Usage.PromptTokens + result.Usage.CompletionTokens + } + var usage *TokenUsage + if totalTokens > 0 { + usage = &TokenUsage{ + PromptTokens: result.Usage.PromptTokens, + CompletionTokens: result.Usage.CompletionTokens, + TotalTokens: totalTokens, } } - } - chatResponse := &ChatResponse{ - Answer: &content, - ReasonContent: &reasonContent, - ToolCalls: toolCalls, - } - if pt, ct, tt := extractUsageFromMap(result); tt > 0 { - chatResponse.Usage = &TokenUsage{ - PromptTokens: pt, CompletionTokens: ct, TotalTokens: tt, - } - } - - return chatResponse, nil + return chatResponseParts{ + RequestID: result.ID, + Content: &content, + ReasonContent: &reasonContent, + ToolCalls: choice.Message.ToolCalls, + Usage: usage, + }, nil + }) } // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel) @@ -308,6 +346,14 @@ func (m *MinimaxModel) ChatStreamlyWithSender(ctx context.Context, modelName str sawTerminal := false accumulatedToolCalls := make(map[int]map[string]any) done, err := ParseSSEStream[map[string]interface{}](resp.Body, func(event map[string]interface{}) error { + tokenUsage, found, usageErr := decodeOpenAICompatibleStreamUsage(event) + if usageErr != nil { + return usageErr + } + if found { + applyStreamUsage(modelConfig, modelUsage, tokenUsage) + } + choices, ok := event["choices"].([]interface{}) if !ok || len(choices) == 0 { // MiniMax can send an error event (rate limit, etc.) diff --git a/internal/entity/models/mistral.go b/internal/entity/models/mistral.go index ae48e99e7a..60a14d7241 100644 --- a/internal/entity/models/mistral.go +++ b/internal/entity/models/mistral.go @@ -23,8 +23,12 @@ import ( "encoding/json" "fmt" "io" + "mime/multipart" "net/http" + "os" + "path/filepath" "ragflow/internal/common" + "strconv" "strings" "time" ) @@ -54,6 +58,33 @@ func (m *MistralModel) Name() string { return "mistral" } +type MistralChatResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []map[string]any `json:"tool_calls"` + ReasoningContent *string `json:"reasoning_content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + CompletionTokens int `json:"completion_tokens"` + NumCachedTokens int `json:"num_cached_tokens"` + PromptAudioSeconds int `json:"prompt_audio_seconds"` + PromptTokenDetails struct { + CachedTokens int `json:"cached_tokens"` + } + PromptTokens int `json:"prompt_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + // ChatWithMessages sends multiple messages with roles and returns the response. func (m *MistralModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) { if err := m.baseModel.APIConfigCheck(apiConfig); err != nil { @@ -103,37 +134,45 @@ func (m *MistralModel) ChatWithMessages(ctx context.Context, modelName string, m return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } - var result map[string]interface{} - if err = json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } + return parseChatCompletionResponse(body, chatModelConfig, modelUsage, func(body []byte, _ *ChatConfig) (chatResponseParts, error) { + var result MistralChatResponse + if err := json.Unmarshal(body, &result); err != nil { + return chatResponseParts{}, fmt.Errorf("failed to parse response: %w", err) + } + if len(result.Choices) == 0 { + return chatResponseParts{}, fmt.Errorf("no choices in response") + } - choices, ok := result["choices"].([]interface{}) - if !ok || len(choices) == 0 { - return nil, fmt.Errorf("no choices in response") - } + choice := &result.Choices[0] + content, reasonContent, err := extractMistralContent(choice.Message.Content) + if err != nil { + return chatResponseParts{}, err + } + if reasonContent == "" && choice.Message.ReasoningContent != nil { + reasonContent = *choice.Message.ReasoningContent + } - firstChoice, ok := choices[0].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid choice format") - } + totalTokens := result.Usage.TotalTokens + if totalTokens == 0 { + totalTokens = result.Usage.PromptTokens + result.Usage.CompletionTokens + } + var usage *TokenUsage + if totalTokens > 0 { + usage = &TokenUsage{ + PromptTokens: result.Usage.PromptTokens, + CompletionTokens: result.Usage.CompletionTokens, + TotalTokens: totalTokens, + } + } - messageMap, ok := firstChoice["message"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid message format") - } - - content, reasonContent, err := extractMistralContent(messageMap["content"]) - if err != nil { - return nil, err - } - toolCalls := extractToolCalls(messageMap) - - return &ChatResponse{ - Answer: &content, - ReasonContent: &reasonContent, - ToolCalls: toolCalls, - }, nil + return chatResponseParts{ + RequestID: result.ID, + Content: &content, + ReasonContent: &reasonContent, + ToolCalls: choice.Message.ToolCalls, + Usage: usage, + }, nil + }) } func extractMistralContent(raw interface{}) (string, string, error) { @@ -230,6 +269,14 @@ func (m *MistralModel) ChatStreamlyWithSender(ctx context.Context, modelName str sawTerminal := false accumulatedToolCalls := make(map[int]map[string]any) done, err := ParseSSEStream[map[string]interface{}](resp.Body, func(event map[string]interface{}) error { + tokenUsage, found, usageErr := decodeOpenAICompatibleStreamUsage(event) + if usageErr != nil { + return usageErr + } + if found { + applyStreamUsage(chatModelConfig, modelUsage, tokenUsage) + } + choices, ok := event["choices"].([]interface{}) if !ok || len(choices) == 0 { return nil @@ -280,6 +327,16 @@ type mistralEmbeddingData struct { Embedding []float64 `json:"embedding"` Object string `json:"object"` Index int `json:"index"` + Usage struct { + CompletionTokens int `json:"completion_tokens"` + NumCachedTokens int `json:"num_cached_tokens"` + PromptAudioSeconds int `json:"prompt_audio_seconds"` + PromptTokenDetails struct { + CachedTokens int `json:"cached_tokens"` + } + PromptTokens int `json:"prompt_tokens"` + TotalTokens int `json:"total_tokens"` + } } type mistralEmbeddingResponse struct { @@ -435,10 +492,7 @@ func (m *MistralModel) Balance(ctx context.Context, apiConfig *APIConfig) (map[s // CheckConnection runs a lightweight ListModels call to verify the API key. func (m *MistralModel) CheckConnection(ctx context.Context, apiConfig *APIConfig) error { _, err := m.ListModels(ctx, apiConfig) - if err != nil { - return err - } - return nil + return err } // Rerank calculates similarity scores between query and documents @@ -448,7 +502,127 @@ func (m *MistralModel) Rerank(ctx context.Context, modelName *string, query stri // TranscribeAudio transcribe audio func (m *MistralModel) TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) { - return nil, fmt.Errorf("%s, no such method", m.Name()) + if err := m.baseModel.APIConfigCheck(apiConfig); err != nil { + return nil, err + } + if modelName == nil || strings.TrimSpace(*modelName) == "" { + return nil, fmt.Errorf("model name is required") + } + + if file == nil || *file == "" { + return nil, fmt.Errorf("file is missing") + } + + resolvedBaseURL, err := m.baseModel.GetBaseURL(apiConfig) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s/%s", resolvedBaseURL, m.baseModel.URLSuffix.ASR) + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + // codeql[go/path-injection] False positive: *file is the audio file path the caller passes in to upload. The user (or operator-supplied pipeline) explicitly chose this path, and the OS access check enforces permissions anyway. + audioFile, err := os.Open(*file) + if err != nil { + return nil, fmt.Errorf("failed to open audio file: %w", err) + } + defer audioFile.Close() + + // create multipart file field + part, err := writer.CreateFormFile( + "file", + filepath.Base(*file), + ) + if err != nil { + return nil, fmt.Errorf("failed to create multipart file: %w", err) + } + + // copy file content + if _, err = io.Copy(part, audioFile); err != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", err) + } + + if err := writer.WriteField("model", strings.TrimSpace(*modelName)); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + if err = writeMistralFormField(writer, key, value); err != nil { + return nil, fmt.Errorf("failed to write field %s: %w", key, err) + } + } + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, longOpCallTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "POST", url, &body) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", strings.TrimSpace(*apiConfig.ApiKey))) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Accept", "application/json") + + resp, err := m.baseModel.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Mistral ASR API error: %s, body: %s", resp.Status, string(respBody)) + } + + var result struct { + Text string `json:"text"` + } + + if err = json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse Mistral ASR response: %w, body=%s", err, string(respBody)) + } + if result.Text == "" { + return nil, fmt.Errorf("Mistral ASR response missing text") + } + + return &ASRResponse{Text: result.Text}, nil +} + +func writeMistralFormField(writer *multipart.Writer, key string, value interface{}) error { + var fieldValue string + switch v := value.(type) { + case string: + fieldValue = v + case bool: + fieldValue = strconv.FormatBool(v) + case int: + fieldValue = strconv.Itoa(v) + case int64: + fieldValue = strconv.FormatInt(v, 10) + case float32: + fieldValue = strconv.FormatFloat(float64(v), 'f', -1, 32) + case float64: + fieldValue = strconv.FormatFloat(v, 'f', -1, 64) + default: + data, err := json.Marshal(v) + if err != nil { + return err + } + fieldValue = string(data) + } + return writer.WriteField(key, fieldValue) } func (m *MistralModel) TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { @@ -457,7 +631,84 @@ func (m *MistralModel) TranscribeAudioWithSender(ctx context.Context, modelName // AudioSpeech convert text to audio func (m *MistralModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) { - return nil, fmt.Errorf("%s, no such method", m.Name()) + if err := m.baseModel.APIConfigCheck(apiConfig); err != nil { + return nil, err + } + if modelName == nil || strings.TrimSpace(*modelName) == "" { + return nil, fmt.Errorf("model name is required") + } + if audioContent == nil || *audioContent == "" { + return nil, fmt.Errorf("text content is empty") + } + + resolvedBaseURL, err := m.baseModel.GetBaseURL(apiConfig) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s/%s", resolvedBaseURL, m.baseModel.URLSuffix.TTS) + + reqBody := map[string]interface{}{ + "model": strings.TrimSpace(*modelName), + "input": *audioContent, + } + if ttsConfig != nil && ttsConfig.Params != nil { + for key, value := range ttsConfig.Params { + reqBody[key] = value + } + } + if ttsConfig != nil && ttsConfig.Format != "" { + reqBody["response_format"] = ttsConfig.Format + } + reqBody["stream"] = false + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, longOpCallTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", strings.TrimSpace(*apiConfig.ApiKey))) + + resp, err := m.baseModel.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Mistral TTS API error: status %d, body: %s", resp.StatusCode, string(body)) + } + + var result struct { + AudioData string `json:"audio_data"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + // format HEX + audioBytes, err := base64.StdEncoding.DecodeString(result.AudioData) + if err != nil { + return nil, fmt.Errorf("failed to decode Mistral base64 audio: %w", err) + } + + return &TTSResponse{ + Audio: audioBytes, + }, nil } func (m *MistralModel) AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { diff --git a/internal/entity/models/mistral_test.go b/internal/entity/models/mistral_test.go index 38759ec581..6d93998270 100644 --- a/internal/entity/models/mistral_test.go +++ b/internal/entity/models/mistral_test.go @@ -5,6 +5,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync/atomic" "testing" @@ -53,6 +55,7 @@ func newMistralForTest(baseURL string) *MistralModel { Chat: "chat/completions", Models: "models", Embedding: "embeddings", + ASR: "audio/transcriptions", }, ) } @@ -411,6 +414,104 @@ func TestMistralRerankReturnsNoSuchMethod(t *testing.T) { } } +func TestMistralTranscribeAudio(t *testing.T) { + ctx := t.Context() + audioPath := filepath.Join(t.TempDir(), "sample.wav") + if err := os.WriteFile(audioPath, []byte("fake-audio"), 0o600); err != nil { + t.Fatalf("write audio file: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/audio/transcriptions" { + t.Errorf("path=%s, want /audio/transcriptions", r.URL.Path) + } + if r.Method != http.MethodPost { + t.Errorf("method=%s, want POST", r.Method) + } + if got := r.Header.Get("Authorization"); got != "Bearer test-key" { + t.Errorf("Authorization=%q, want Bearer test-key", got) + } + if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data;") { + t.Errorf("Content-Type=%q, want multipart/form-data", r.Header.Get("Content-Type")) + } + if got := r.Header.Get("Accept"); got != "application/json" { + t.Errorf("Accept=%q, want application/json", got) + } + + reader, err := r.MultipartReader() + if err != nil { + t.Fatalf("MultipartReader: %v", err) + } + fields := make(map[string]string) + var fileName, fileBody string + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart: %v", err) + } + data, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll part: %v", err) + } + if part.FormName() == "file" { + fileName = part.FileName() + fileBody = string(data) + continue + } + fields[part.FormName()] = string(data) + } + if fileName != "sample.wav" { + t.Errorf("file name=%q, want sample.wav", fileName) + } + if fileBody != "fake-audio" { + t.Errorf("file body=%q, want fake-audio", fileBody) + } + if fields["model"] != "voxtral-mini-latest" { + t.Errorf("model=%q, want voxtral-mini-latest", fields["model"]) + } + if fields["language"] != "en" { + t.Errorf("language=%q, want en", fields["language"]) + } + if fields["diarize"] != "true" { + t.Errorf("diarize=%q, want true", fields["diarize"]) + } + if fields["timestamp_granularities"] != `["segment","word"]` { + t.Errorf("timestamp_granularities=%q", fields["timestamp_granularities"]) + } + + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "language": "en", + "model": "voxtral-mini-latest", + "text": "hello world", + }) + })) + defer srv.Close() + + apiKey := " test-key " + modelName := " voxtral-mini-latest " + resp, err := newMistralForTest(srv.URL).TranscribeAudio( + ctx, + &modelName, + &audioPath, + &APIConfig{ApiKey: &apiKey}, + &ASRConfig{Params: map[string]interface{}{ + "language": "en", + "diarize": true, + "timestamp_granularities": []string{"segment", "word"}, + }}, + nil, + ) + if err != nil { + t.Fatalf("TranscribeAudio: %v", err) + } + if resp.Text != "hello world" { + t.Fatalf("Text=%q, want hello world", resp.Text) + } +} + func TestMistralUnsupportedDefaultsReturnNoSuchMethod(t *testing.T) { ctx := t.Context() m := newMistralForTest("http://unused") @@ -420,17 +521,9 @@ func TestMistralUnsupportedDefaultsReturnNoSuchMethod(t *testing.T) { name string call func() error }{ - {"TranscribeAudio", func() error { - _, err := m.TranscribeAudio(ctx, &modelName, nil, &APIConfig{}, nil, nil) - return err - }}, {"TranscribeAudioWithSender", func() error { return m.TranscribeAudioWithSender(ctx, &modelName, nil, &APIConfig{}, nil, nil, nil) }}, - {"AudioSpeech", func() error { - _, err := m.AudioSpeech(ctx, &modelName, nil, &APIConfig{}, nil, nil) - return err - }}, {"AudioSpeechWithSender", func() error { return m.AudioSpeechWithSender(ctx, &modelName, nil, &APIConfig{}, nil, nil, nil) }}, diff --git a/internal/entity/models/moonshot.go b/internal/entity/models/moonshot.go index d5d5c231e7..4e34d4f8c9 100644 --- a/internal/entity/models/moonshot.go +++ b/internal/entity/models/moonshot.go @@ -51,6 +51,29 @@ func (m *MoonshotModel) Name() string { return "moonshot" } +type MoonshotChatResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []map[string]any `json:"tool_calls"` + ReasoningContent *string `json:"reasoning_content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + CachedTokens int `json:"cached_tokens"` + } `json:"usage"` +} + func validateMoonshotModelName(modelName string) (string, error) { if strings.TrimSpace(modelName) == "" { return "", fmt.Errorf("model name is required") @@ -124,62 +147,53 @@ func (m *MoonshotModel) ChatWithMessages(ctx context.Context, modelName string, return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } - // Parse response - var result map[string]interface{} - if err = json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - - choices, ok := result["choices"].([]interface{}) - if !ok || len(choices) == 0 { - return nil, fmt.Errorf("no choices in response") - } - - firstChoice, ok := choices[0].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid choice format") - } - - messageMap, ok := firstChoice["message"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid message format") - } - - var content string - if c, ok := messageMap["content"].(string); ok { - content = c - } - - var reasonContent string - if chatModelConfig != nil && chatModelConfig.Thinking != nil && *chatModelConfig.Thinking { - reasonContent, _ = messageMap["reasoning_content"].(string) - // if first char of reasonContent is \n remove the \n - if reasonContent != "" && reasonContent[0] == '\n' { - reasonContent = reasonContent[1:] + return parseChatCompletionResponse(body, chatModelConfig, modelUsage, func(body []byte, chatConfig *ChatConfig) (chatResponseParts, error) { + var result MoonshotChatResponse + if err := json.Unmarshal(body, &result); err != nil { + return chatResponseParts{}, fmt.Errorf("failed to parse response: %w", err) } - } - var toolCalls []map[string]any - if tcs, ok := messageMap["tool_calls"].([]interface{}); ok { - for _, tc := range tcs { - if tcMap, ok := tc.(map[string]any); ok { - toolCalls = append(toolCalls, tcMap) + if len(result.Choices) == 0 { + return chatResponseParts{}, fmt.Errorf("no choices in response") + } + + choice := &result.Choices[0] + content := "" + if choice.Message.Content != nil { + content = *choice.Message.Content + } + + reasonContent := "" + if chatConfig != nil && chatConfig.Thinking != nil && *chatConfig.Thinking { + if choice.Message.ReasoningContent != nil { + reasonContent = *choice.Message.ReasoningContent + } + if reasonContent != "" && reasonContent[0] == '\n' { + reasonContent = reasonContent[1:] } } - } - chatResponse := &ChatResponse{ - Answer: &content, - ReasonContent: &reasonContent, - ToolCalls: toolCalls, - } - if pt, ct, tt := extractUsageFromMap(result); tt > 0 { - chatResponse.Usage = &TokenUsage{ - PromptTokens: pt, CompletionTokens: ct, TotalTokens: tt, + totalTokens := result.Usage.TotalTokens + if totalTokens == 0 { + totalTokens = result.Usage.PromptTokens + result.Usage.CompletionTokens + } + var usage *TokenUsage + if totalTokens > 0 { + usage = &TokenUsage{ + PromptTokens: result.Usage.PromptTokens, + CompletionTokens: result.Usage.CompletionTokens, + TotalTokens: totalTokens, + } } - } - return chatResponse, nil + return chatResponseParts{ + RequestID: result.ID, + Content: &content, + ReasonContent: &reasonContent, + ToolCalls: choice.Message.ToolCalls, + Usage: usage, + }, nil + }) } // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel) @@ -255,6 +269,14 @@ func (m *MoonshotModel) ChatStreamlyWithSender(ctx context.Context, modelName st } accumulatedToolCalls := make(map[int]map[string]any) done, err := ParseSSEStream[map[string]interface{}](resp.Body, func(event map[string]interface{}) error { + tokenUsage, found, usageErr := decodeOpenAICompatibleStreamUsage(event) + if usageErr != nil { + return usageErr + } + if found { + applyStreamUsage(chatModelConfig, modelUsage, tokenUsage) + } + choices, ok := event["choices"].([]interface{}) if !ok || len(choices) == 0 { return nil diff --git a/internal/entity/models/moonshot_test.go b/internal/entity/models/moonshot_test.go index bb6b563bc7..483bc6c970 100644 --- a/internal/entity/models/moonshot_test.go +++ b/internal/entity/models/moonshot_test.go @@ -184,6 +184,71 @@ func TestMoonshotChatSupportsTools(t *testing.T) { } } +func TestMoonshotChatParsesCompletionSchema(t *testing.T) { + ctx := t.Context() + srv := newMoonshotServer(t, func(t *testing.T, _ *http.Request, _ map[string]interface{}, w http.ResponseWriter) { + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "cmpl_1", + "object": "chat.completion", + "created": 1774598400, + "model": "kimi-k2.6", + "choices": []map[string]interface{}{{ + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": nil, + "reasoning_content": "\nreason", + "tool_calls": []map[string]interface{}{{ + "id": "call_1", + "type": "function", + "function": map[string]interface{}{ + "name": "lookup", + "arguments": `{"q":"moonshot"}`, + }, + }}, + }, + "finish_reason": "tool_calls", + }}, + "usage": map[string]interface{}{ + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "cached_tokens": 3, + }, + }) + }) + defer srv.Close() + + apiKey := "test-key" + thinking := true + resp, err := newMoonshotForTest(srv.URL).ChatWithMessages( + ctx, + "kimi-k2.6", + []Message{{Role: "user", Content: "ping"}}, + &APIConfig{ApiKey: &apiKey}, + &ChatConfig{Thinking: &thinking}, nil, + ) + if err != nil { + t.Fatalf("ChatWithMessages: %v", err) + } + if resp.Answer == nil || *resp.Answer != "" { + t.Errorf("Answer=%v, want empty", resp.Answer) + } + if resp.ReasonContent == nil || *resp.ReasonContent != "reason" { + t.Errorf("ReasonContent=%v, want reason", resp.ReasonContent) + } + if len(resp.ToolCalls) != 1 { + t.Fatalf("ToolCalls=%#v, want one", resp.ToolCalls) + } + function, ok := resp.ToolCalls[0]["function"].(map[string]any) + if !ok || function["name"] != "lookup" || function["arguments"] != `{"q":"moonshot"}` { + t.Errorf("function=%#v", resp.ToolCalls[0]["function"]) + } + if resp.Usage == nil || resp.Usage.PromptTokens != 10 || resp.Usage.CompletionTokens != 5 || resp.Usage.TotalTokens != 15 { + t.Errorf("Usage=%#v, want 10/5/15", resp.Usage) + } +} + func TestMoonshotStreamForcesStreaming(t *testing.T) { ctx := t.Context() srv := newMoonshotServer(t, func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) {