mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-23 17:06:42 +08:00
Go: add funASR provider (#17171)
### Summary
As title
verif ied from CLI
```
RAGFlow(api/default)> asr with 'paraformer@test@funasr' audio './internal/test.wav' param '{"language": "en"}'
+----------------------------------------------------------------------------------------------------------------------+
| text |
+----------------------------------------------------------------------------------------------------------------------+
| The examination and testimony of the experts enabled the commission to conclude that five shots may have been fired. |
+----------------------------------------------------------------------------------------------------------------------+
```
---
<img width="2876" height="1377" alt="image"
src="https://github.com/user-attachments/assets/4cee62c6-1c71-43ce-b398-4d9fbff33c3c"
/>
```bash
INFO: 127.0.0.1:51910 - "POST /v1/audio/transcriptions HTTP/1.1" 200 OK██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 2.19it/s]
INFO: 127.0.0.1:60934 - "GET /v1/models HTTP/1.1" 200 OK%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 2.10it/s]
```
---------
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
Haruko386
91
conf/models/funasr.json
Normal file
91
conf/models/funasr.json
Normal file
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"name": "FunASR",
|
||||
"url": {
|
||||
"default": "http://localhost:8000/v1"
|
||||
},
|
||||
"url_suffix": {
|
||||
"asr": "audio/transcriptions",
|
||||
"models": "models"
|
||||
},
|
||||
"class": "local",
|
||||
"models": [
|
||||
{
|
||||
"name": "fun-asr-nano",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Fun-ASR-MLT-Nano",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "SenseVoiceSmall",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "paraformer",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Paraformer-zh-streaming",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Qwen3-ASR",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "GLM-ASR-Nano",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Whisper-large-v3",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Whisper-large-v3-turbo",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ct-punc",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "fsmn-vad",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cam++",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "emotion2vec+large",
|
||||
"model_types": [
|
||||
"asr"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -155,6 +155,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
|
||||
return NewQiniuModel(baseURL, urlSuffix), nil
|
||||
case "xiaomi":
|
||||
return NewXiaomiModel(baseURL, urlSuffix), nil
|
||||
case "funasr":
|
||||
return NewFunASRModel(baseURL, urlSuffix), nil
|
||||
default:
|
||||
return NewDummyModel(baseURL, urlSuffix), nil
|
||||
}
|
||||
|
||||
276
internal/entity/models/funasr.go
Normal file
276
internal/entity/models/funasr.go
Normal file
@@ -0,0 +1,276 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type FunASR struct {
|
||||
baseModel BaseModel
|
||||
}
|
||||
|
||||
func NewFunASRModel(baseURL map[string]string, urlSuffix URLSuffix) *FunASR {
|
||||
return &FunASR{
|
||||
baseModel: BaseModel{
|
||||
BaseURL: baseURL,
|
||||
URLSuffix: urlSuffix,
|
||||
httpClient: NewDriverHTTPClient(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FunASR) NewInstance(baseURL map[string]string) ModelDriver {
|
||||
return NewFunASRModel(baseURL, f.baseModel.URLSuffix)
|
||||
}
|
||||
|
||||
func (f *FunASR) Name() string {
|
||||
return "funasr"
|
||||
}
|
||||
|
||||
func (f *FunASR) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
|
||||
return fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) {
|
||||
if err := f.baseModel.APIConfigCheck(apiConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if file == nil || *file == "" {
|
||||
return nil, fmt.Errorf("file is missing")
|
||||
}
|
||||
|
||||
resolvedBaseURL, err := f.baseModel.GetBaseURL(apiConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := fmt.Sprintf("%s/%s", resolvedBaseURL, f.baseModel.URLSuffix.ASR)
|
||||
|
||||
// multipart body
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
|
||||
audioFile, err := os.Open(*file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open audio file: %w", err)
|
||||
}
|
||||
defer audioFile.Close()
|
||||
|
||||
// create multipart file field
|
||||
part, err := writer.CreateFormFile(
|
||||
"file",
|
||||
filepath.Base(*file),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create multipart file: %w", err)
|
||||
}
|
||||
|
||||
// copy file content
|
||||
if _, err = io.Copy(part, audioFile); err != nil {
|
||||
return nil, fmt.Errorf("failed to copy audio data: %w", err)
|
||||
}
|
||||
|
||||
// model field
|
||||
if err := writer.WriteField("model", *modelName); err != nil {
|
||||
return nil, fmt.Errorf("failed to write model field: %w", err)
|
||||
}
|
||||
|
||||
// extra params
|
||||
if asrConfig != nil && asrConfig.Params != nil {
|
||||
for key, value := range asrConfig.Params {
|
||||
|
||||
var val string
|
||||
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
val = v
|
||||
case bool:
|
||||
val = strconv.FormatBool(v)
|
||||
case int:
|
||||
val = strconv.Itoa(v)
|
||||
case int64:
|
||||
val = strconv.FormatInt(v, 10)
|
||||
case float32:
|
||||
val = strconv.FormatFloat(float64(v), 'f', -1, 32)
|
||||
case float64:
|
||||
val = strconv.FormatFloat(v, 'f', -1, 64)
|
||||
default:
|
||||
val = fmt.Sprintf("%v", v)
|
||||
}
|
||||
|
||||
if err = writer.WriteField(key, val); err != nil {
|
||||
return nil, fmt.Errorf("failed to write field %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err = writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
// build request
|
||||
ctx, cancel := context.WithTimeout(context.Background(), longOpCallTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, &body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// send request
|
||||
resp, err := f.baseModel.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("FunASR error: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(respBody, &result); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w, body=%s", err, string(respBody))
|
||||
}
|
||||
|
||||
return &ASRResponse{Text: result.Text}, nil
|
||||
}
|
||||
|
||||
func (f *FunASR) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
|
||||
return fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
|
||||
return fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig, modelUsage *common.ModelUsage) (*OCRFileResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) ParseFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig, modelUsage *common.ModelUsage) (*ParseFileResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) ListModels(apiConfig *APIConfig) ([]ListModelResponse, error) {
|
||||
if err := f.baseModel.APIConfigCheck(apiConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
apiKey := strings.TrimSpace(*apiConfig.ApiKey)
|
||||
|
||||
resolvedBaseURL, err := f.baseModel.GetBaseURL(apiConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := fmt.Sprintf("%s/%s", resolvedBaseURL, f.baseModel.URLSuffix.Models)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), nonStreamCallTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := f.baseModel.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var modelList ModelList
|
||||
if err = json.Unmarshal(body, &modelList); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
if modelList.Models == nil {
|
||||
return nil, fmt.Errorf("invalid models list format")
|
||||
}
|
||||
models := ParseListModel(modelList)
|
||||
if len(models) == 0 {
|
||||
return nil, fmt.Errorf("invalid models list format")
|
||||
}
|
||||
return models, nil
|
||||
}
|
||||
|
||||
func (f *FunASR) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) CheckConnection(apiConfig *APIConfig) error {
|
||||
_, err := f.ListModels(apiConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *FunASR) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
|
||||
func (f *FunASR) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
|
||||
return nil, fmt.Errorf("%s no such method", f.Name())
|
||||
}
|
||||
Reference in New Issue
Block a user