diff --git a/conf/models/funasr.json b/conf/models/funasr.json new file mode 100644 index 0000000000..ed7497c833 --- /dev/null +++ b/conf/models/funasr.json @@ -0,0 +1,91 @@ +{ + "name": "FunASR", + "url": { + "default": "http://localhost:8000/v1" + }, + "url_suffix": { + "asr": "audio/transcriptions", + "models": "models" + }, + "class": "local", + "models": [ + { + "name": "fun-asr-nano", + "model_types": [ + "asr" + ] + }, + { + "name": "Fun-ASR-MLT-Nano", + "model_types": [ + "asr" + ] + }, + { + "name": "SenseVoiceSmall", + "model_types": [ + "asr" + ] + }, + { + "name": "paraformer", + "model_types": [ + "asr" + ] + }, + { + "name": "Paraformer-zh-streaming", + "model_types": [ + "asr" + ] + }, + { + "name": "Qwen3-ASR", + "model_types": [ + "asr" + ] + }, + { + "name": "GLM-ASR-Nano", + "model_types": [ + "asr" + ] + }, + { + "name": "Whisper-large-v3", + "model_types": [ + "asr" + ] + }, + { + "name": "Whisper-large-v3-turbo", + "model_types": [ + "asr" + ] + }, + { + "name": "ct-punc", + "model_types": [ + "asr" + ] + }, + { + "name": "fsmn-vad", + "model_types": [ + "asr" + ] + }, + { + "name": "cam++", + "model_types": [ + "asr" + ] + }, + { + "name": "emotion2vec+large", + "model_types": [ + "asr" + ] + } + ] +} \ No newline at end of file diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go index 62726d6780..0742a5569e 100644 --- a/internal/entity/models/factory.go +++ b/internal/entity/models/factory.go @@ -155,6 +155,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string return NewQiniuModel(baseURL, urlSuffix), nil case "xiaomi": return NewXiaomiModel(baseURL, urlSuffix), nil + case "funasr": + return NewFunASRModel(baseURL, urlSuffix), nil default: return NewDummyModel(baseURL, urlSuffix), nil } diff --git a/internal/entity/models/funasr.go b/internal/entity/models/funasr.go new file mode 100644 index 0000000000..be81620b01 --- /dev/null +++ b/internal/entity/models/funasr.go @@ -0,0 +1,276 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package models + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "ragflow/internal/common" + "strconv" + "strings" +) + +type FunASR struct { + baseModel BaseModel +} + +func NewFunASRModel(baseURL map[string]string, urlSuffix URLSuffix) *FunASR { + return &FunASR{ + baseModel: BaseModel{ + BaseURL: baseURL, + URLSuffix: urlSuffix, + httpClient: NewDriverHTTPClient(), + }, + } +} + +func (f *FunASR) NewInstance(baseURL map[string]string) ModelDriver { + return NewFunASRModel(baseURL, f.baseModel.URLSuffix) +} + +func (f *FunASR) Name() string { + return "funasr" +} + +func (f *FunASR) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { + return fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) { + if err := f.baseModel.APIConfigCheck(apiConfig); err != nil { + return nil, err + } + + if file == nil || *file == "" { + return nil, fmt.Errorf("file is missing") + } + + resolvedBaseURL, err := f.baseModel.GetBaseURL(apiConfig) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s/%s", resolvedBaseURL, f.baseModel.URLSuffix.ASR) + + // multipart body + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + audioFile, err := os.Open(*file) + if err != nil { + return nil, fmt.Errorf("failed to open audio file: %w", err) + } + defer audioFile.Close() + + // create multipart file field + part, err := writer.CreateFormFile( + "file", + filepath.Base(*file), + ) + if err != nil { + return nil, fmt.Errorf("failed to create multipart file: %w", err) + } + + // copy file content + if _, err = io.Copy(part, audioFile); err != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", err) + } + + // model field + if err := writer.WriteField("model", *modelName); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + // extra params + if asrConfig != nil && asrConfig.Params != nil { + for key, value := range asrConfig.Params { + + var val string + + switch v := value.(type) { + case string: + val = v + case bool: + val = strconv.FormatBool(v) + case int: + val = strconv.Itoa(v) + case int64: + val = strconv.FormatInt(v, 10) + case float32: + val = strconv.FormatFloat(float64(v), 'f', -1, 32) + case float64: + val = strconv.FormatFloat(v, 'f', -1, 64) + default: + val = fmt.Sprintf("%v", v) + } + + if err = writer.WriteField(key, val); err != nil { + return nil, fmt.Errorf("failed to write field %s: %w", key, err) + } + } + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + // build request + ctx, cancel := context.WithTimeout(context.Background(), longOpCallTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "POST", url, &body) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Accept", "application/json") + + // send request + resp, err := f.baseModel.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("FunASR error: %s - %s", resp.Status, string(respBody)) + } + + var result struct { + Text string `json:"text"` + } + + if err = json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody)) + } + + return &ASRResponse{Text: result.Text}, nil +} + +func (f *FunASR) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { + return fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error { + return fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig, modelUsage *common.ModelUsage) (*OCRFileResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig, modelUsage *common.ModelUsage) (*ParseFileResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) ListModels(apiConfig *APIConfig) ([]ListModelResponse, error) { + if err := f.baseModel.APIConfigCheck(apiConfig); err != nil { + return nil, err + } + apiKey := strings.TrimSpace(*apiConfig.ApiKey) + + resolvedBaseURL, err := f.baseModel.GetBaseURL(apiConfig) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s/%s", resolvedBaseURL, f.baseModel.URLSuffix.Models) + + ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + req.Header.Set("Accept", "application/json") + + resp, err := f.baseModel.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + var modelList ModelList + if err = json.Unmarshal(body, &modelList); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + if modelList.Models == nil { + return nil, fmt.Errorf("invalid models list format") + } + models := ParseListModel(modelList) + if len(models) == 0 { + return nil, fmt.Errorf("invalid models list format") + } + return models, nil +} + +func (f *FunASR) Balance(apiConfig *APIConfig) (map[string]interface{}, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) CheckConnection(apiConfig *APIConfig) error { + _, err := f.ListModels(apiConfig) + return err +} + +func (f *FunASR) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +} + +func (f *FunASR) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) { + return nil, fmt.Errorf("%s no such method", f.Name()) +}