From b1ef5d365f5d579e0b93cd08c89c6263fe8d4774 Mon Sep 17 00:00:00 2001 From: Jake Armstrong <65635253+jakearmstrong59@users.noreply.github.com> Date: Thu, 21 May 2026 21:19:38 -1000 Subject: [PATCH] Go: implement ASR in OpenRouter driver (#15067) ### What problem does this PR solve? Fixes #15066 OpenRouter now exposes an official speech-to-text endpoint at `POST /api/v1/audio/transcriptions`, but the Go model driver still returned `openrouter, no such method` from `TranscribeAudio`. This left OpenRouter ASR models unavailable through the Go API server even though the provider already has OpenRouter audio support for TTS. Related provider-tracking context: #14736 ### Type of change - [x] New Feature (non-breaking change which adds functionality) Co-authored-by: Jin Hai --- conf/models/openrouter.json | 12 +- internal/entity/models/openrouter.go | 99 +++++++++- internal/entity/models/openrouter_test.go | 220 ++++++++++++++++++++++ 3 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 internal/entity/models/openrouter_test.go diff --git a/conf/models/openrouter.json b/conf/models/openrouter.json index 33d0bdadbd..3fb81d332b 100644 --- a/conf/models/openrouter.json +++ b/conf/models/openrouter.json @@ -9,7 +9,8 @@ "embedding": "embeddings", "rerank": "rerank", "balance": "credits", - "tts": "audio/speech" + "tts": "audio/speech", + "asr": "audio/transcriptions" }, "class": "openrouter", "models": [ @@ -52,6 +53,13 @@ "model_types": [ "tts" ] + }, + { + "name": "openai/whisper-large-v3", + "max_tokens": 131072, + "model_types": [ + "asr" + ] } ] -} \ No newline at end of file +} diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go index 03d3c85252..b57d6956fb 100644 --- a/internal/entity/models/openrouter.go +++ b/internal/entity/models/openrouter.go @@ -3,10 +3,13 @@ package models import ( "bufio" "bytes" + "encoding/base64" "encoding/json" "fmt" "io" "net/http" + "os" + "path/filepath" "ragflow/internal/common" "strings" "time" @@ -531,9 +534,103 @@ func (o *OpenRouterModel) Rerank(modelName *string, query string, documents []st return &rerankResponse, nil } +type openRouterTranscriptionResponse struct { + Text string `json:"text"` +} + +func openRouterAudioFormat(file string, asrConfig *ASRConfig) string { + if asrConfig != nil && asrConfig.Params != nil { + if format, ok := asrConfig.Params["format"]; ok && format != nil { + if value := strings.TrimPrefix(fmt.Sprint(format), "."); value != "" { + return value + } + } + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(file)), ".") + if ext == "" { + return "wav" + } + return ext +} + // TranscribeAudio transcribe audio func (o *OpenRouterModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) { - return nil, fmt.Errorf("%s, no such method", o.Name()) + if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { + return nil, fmt.Errorf("OpenRouter API key is missing") + } + if modelName == nil || *modelName == "" { + return nil, fmt.Errorf("model name is required") + } + if file == nil || *file == "" { + return nil, fmt.Errorf("file is missing") + } + if o.URLSuffix.ASR == "" { + return nil, fmt.Errorf("OpenRouter ASR url suffix is missing") + } + + audio, err := os.ReadFile(*file) + if err != nil { + return nil, fmt.Errorf("failed to read audio file: %w", err) + } + + region := "default" + if apiConfig.Region != nil && *apiConfig.Region != "" { + region = *apiConfig.Region + } + + reqBody := map[string]interface{}{ + "model": *modelName, + "input_audio": map[string]interface{}{ + "data": base64.StdEncoding.EncodeToString(audio), + "format": openRouterAudioFormat(*file, asrConfig), + }, + } + + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + switch key { + case "format", "model", "input_audio": + continue + } + reqBody[key] = value + } + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + url := fmt.Sprintf("%s/%s", strings.TrimSuffix(o.BaseURL[region], "/"), o.URLSuffix.ASR) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + + resp, err := o.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("OpenRouter ASR API error: %s, body: %s", resp.Status, string(body)) + } + + var result openRouterTranscriptionResponse + if err = json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to parse transcription response: %w", err) + } + + return &ASRResponse{Text: result.Text}, nil } func (z *OpenRouterModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error { diff --git a/internal/entity/models/openrouter_test.go b/internal/entity/models/openrouter_test.go new file mode 100644 index 0000000000..7868ebebf1 --- /dev/null +++ b/internal/entity/models/openrouter_test.go @@ -0,0 +1,220 @@ +package models + +import ( + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func newOpenRouterServer(t *testing.T, expectedPath string, handler func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != expectedPath { + t.Errorf("path=%s, want %s", r.URL.Path, expectedPath) + return + } + if got := r.Header.Get("Authorization"); got != "Bearer test-key" { + t.Errorf("Authorization=%q, want Bearer test-key", got) + return + } + if got := r.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") { + t.Errorf("Content-Type=%q, want application/json", got) + return + } + + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read body: %v", err) + return + } + + var body map[string]interface{} + if len(raw) > 0 { + if err := json.Unmarshal(raw, &body); err != nil { + t.Errorf("unmarshal body: %v\nraw=%s", err, string(raw)) + return + } + } + handler(t, r, body, w) + })) +} + +func newOpenRouterForTest(baseURL string) *OpenRouterModel { + return NewOpenRouterModel( + map[string]string{"default": baseURL}, + URLSuffix{ + Chat: "chat/completions", + Models: "models", + ASR: "audio/transcriptions", + TTS: "audio/speech", + Balance: "credits", + }, + ) +} + +func writeOpenRouterAudioFile(t *testing.T, name string, content []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write audio file: %v", err) + } + return path +} + +func TestOpenRouterAudioFormatUsesConfiguredValue(t *testing.T) { + if got := openRouterAudioFormat("sample.wav", &ASRConfig{Params: map[string]interface{}{"format": ".mp3"}}); got != "mp3" { + t.Errorf("format=%q, want mp3", got) + } + if got := openRouterAudioFormat("sample.wav", &ASRConfig{Params: map[string]interface{}{"format": 123}}); got != "123" { + t.Errorf("format=%q, want 123", got) + } +} + +func TestOpenRouterTranscribeAudioHappyPath(t *testing.T) { + audio := []byte("RIFF test audio") + srv := newOpenRouterServer(t, "/audio/transcriptions", func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) { + if r.Method != http.MethodPost { + t.Errorf("method=%s, want POST", r.Method) + } + if body["model"] != "openai/whisper-large-v3" { + t.Errorf("model=%v", body["model"]) + } + if body["language"] != "en" { + t.Errorf("language=%v", body["language"]) + } + if body["temperature"] != 0.2 { + t.Errorf("temperature=%v", body["temperature"]) + } + + inputAudio, ok := body["input_audio"].(map[string]interface{}) + if !ok { + t.Errorf("input_audio=%T, want object", body["input_audio"]) + return + } + if inputAudio["data"] != base64.StdEncoding.EncodeToString(audio) { + t.Errorf("input_audio.data=%v", inputAudio["data"]) + } + if inputAudio["format"] != "wav" { + t.Errorf("input_audio.format=%v, want wav", inputAudio["format"]) + } + if _, ok := body["format"]; ok { + t.Errorf("format should only be sent inside input_audio: %v", body["format"]) + } + if inputAudio["data"] == "bad-data" { + t.Errorf("input_audio should not be overwritten by params") + } + + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "text": "hello from audio", + }) + }) + defer srv.Close() + + apiKey := "test-key" + modelName := "openai/whisper-large-v3" + file := writeOpenRouterAudioFile(t, "sample.wav", audio) + resp, err := newOpenRouterForTest(srv.URL).TranscribeAudio( + &modelName, + &file, + &APIConfig{ApiKey: &apiKey}, + &ASRConfig{Params: map[string]interface{}{ + "format": "wav", + "language": "en", + "temperature": 0.2, + "model": "wrong-model", + "input_audio": map[string]interface{}{"data": "bad-data", "format": "bad"}, + }}, + ) + if err != nil { + t.Fatalf("TranscribeAudio: %v", err) + } + if resp.Text != "hello from audio" { + t.Errorf("Text=%q", resp.Text) + } +} + +func TestOpenRouterTranscribeAudioInfersFormat(t *testing.T) { + srv := newOpenRouterServer(t, "/audio/transcriptions", func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) { + inputAudio, ok := body["input_audio"].(map[string]interface{}) + if !ok { + t.Errorf("input_audio=%T, want object", body["input_audio"]) + return + } + if inputAudio["format"] != "mp3" { + t.Errorf("input_audio.format=%v, want mp3", inputAudio["format"]) + } + _ = json.NewEncoder(w).Encode(map[string]interface{}{"text": "ok"}) + }) + defer srv.Close() + + apiKey := "test-key" + modelName := "openai/whisper-large-v3" + file := writeOpenRouterAudioFile(t, "sample.mp3", []byte("audio")) + _, err := newOpenRouterForTest(srv.URL).TranscribeAudio(&modelName, &file, &APIConfig{ApiKey: &apiKey}, nil) + if err != nil { + t.Fatalf("TranscribeAudio: %v", err) + } +} + +func TestOpenRouterTranscribeAudioValidatesInputs(t *testing.T) { + modelName := "openai/whisper-large-v3" + apiKey := "test-key" + file := "sample.wav" + + tests := []struct { + name string + modelName *string + file *string + apiConfig *APIConfig + want string + }{ + {name: "api key", modelName: &modelName, file: &file, apiConfig: &APIConfig{}, want: "OpenRouter API key is missing"}, + {name: "model", file: &file, apiConfig: &APIConfig{ApiKey: &apiKey}, want: "model name is required"}, + {name: "file", modelName: &modelName, apiConfig: &APIConfig{ApiKey: &apiKey}, want: "file is missing"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := newOpenRouterForTest("http://unused").TranscribeAudio(tt.modelName, tt.file, tt.apiConfig, nil) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("err=%v, want %q", err, tt.want) + } + }) + } +} + +func TestOpenRouterTranscribeAudioValidatesASRSuffix(t *testing.T) { + apiKey := "test-key" + modelName := "openai/whisper-large-v3" + file := "sample.wav" + model := NewOpenRouterModel(map[string]string{"default": "http://unused"}, URLSuffix{}) + + _, err := model.TranscribeAudio(&modelName, &file, &APIConfig{ApiKey: &apiKey}, nil) + if err == nil || !strings.Contains(err.Error(), "OpenRouter ASR url suffix is missing") { + t.Fatalf("err=%v", err) + } +} + +func TestOpenRouterTranscribeAudioHTTPError(t *testing.T) { + srv := newOpenRouterServer(t, "/audio/transcriptions", func(t *testing.T, r *http.Request, body map[string]interface{}, w http.ResponseWriter) { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + }) + defer srv.Close() + + apiKey := "test-key" + modelName := "openai/whisper-large-v3" + file := writeOpenRouterAudioFile(t, "sample.wav", []byte("audio")) + _, err := newOpenRouterForTest(srv.URL).TranscribeAudio(&modelName, &file, &APIConfig{ApiKey: &apiKey}, nil) + if err == nil || + !strings.Contains(err.Error(), "OpenRouter ASR API error") || + !strings.Contains(err.Error(), "401 Unauthorized") || + !strings.Contains(err.Error(), "unauthorized") { + t.Fatalf("err=%v", err) + } +}