zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-15 20:27:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(stt): add FunASR / SenseVoice provider (#16473)

Browse Source 
### Summary

Adds FunASR as a self-hosted speech-to-text provider through its
OpenAI-compatible `/v1/audio/transcriptions` endpoint.

This is a focused replacement for #15526 by @Rene0422 and relates to
#15448. The unrelated Markdown parser changes from the previous branch
are intentionally removed so this PR contains only the FunASR provider
integration.

- register FunASR as a `SPEECH2TEXT` factory;
- add `FunASRSeq2txt` with `sensevoice` and `http://localhost:8000/v1`
defaults, an optional API key, URL normalization, and inherited
transcription handling;
- wire FunASR into the current local-provider schema with a prefilled
local URL and official documentation link;
- discover the server's `/v1/models` dynamically and expose every
returned model as speech-to-text in the model picker;
- use RAGFlow's existing default provider icon fallback instead of
referencing a missing `funasr` asset;
- list FunASR in the supported-provider documentation;
- add focused backend and frontend regression tests.

### Validation

- focused backend pytest suite -> `7 passed`
- real CPU `funasr-server` + RAGFlow provider smoke test -> discovered
`fun-asr-nano`, `sensevoice`, and `paraformer`; transcribed a real WAV
as `我现在在录一段测试音频` (`10` tokens, `0.504s`)
- `ruff check` and `ruff format --check` on the changed Python files
- `python3 -m py_compile` on the provider and its test
- JSON parse and a semantic assertion for exactly one enabled FunASR
`SPEECH2TEXT` factory
- focused frontend Jest test -> `2 passed`
- ESLint and Prettier on all changed TypeScript files
- `npm run build` -> production build succeeded (`14,181` modules
transformed)
- `git diff --check`

### Deployment

Run FunASR separately and point the RAGFlow provider at it:

```bash
pip install funasr
funasr-server --device cuda --model sensevoice
```

The API key remains optional because the stock local server does not
require authentication. A key can still be supplied when the endpoint is
protected by a gateway.

---------

Signed-off-by: LauraGPT <LauraGPT@users.noreply.github.com>
Co-authored-by: LauraGPT <LauraGPT@users.noreply.github.com>
This commit is contained in:
zhifu gao
2026-07-15 19:02:05 +08:00
committed by GitHub
parent 2223a514de
commit 06e36d24f4
11 changed files with 233 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
conf/llm_factories.json
View File

@@ -1929,6 +1929,13 @@
"status": "1",
"llm": []
},
{
"name": "FunASR",
"logo": "",
"tags": "SPEECH2TEXT",
"status": "1",
"llm": []
},
{
"name": "DeepSeek",
"logo": "",

1
docs/guides/models/supported_models.mdx
View File

@@ -28,6 +28,7 @@ A complete list of model providers supported by RAGFlow, which will continue to
| Cohere | `https://cohere.com` |
| DeepSeek | `https://www.deepseek.com` |
| Fish Audio | `https://fish.audio` |
| FunASR | `https://github.com/modelscope/FunASR` |
| FuturMix | `https://futurmix.ai` |
| Gemini | `https://gemini.google.com` |
| Google Cloud | `https://cloud.google.com` |

23
rag/llm/model_meta.py
View File

@@ -458,6 +458,29 @@ class OpenAIAPICompatible(Base):
return model_list
class FunASR(Base):
_FACTORY_NAME = "FunASR"
def _format_model_list(self, raw_model_list):
models = raw_model_list.get("data") if isinstance(raw_model_list, dict) else None
if not isinstance(models, list):
return []
model_list = []
for model in models:
if not isinstance(model, dict) or not model.get("id"):
continue
model_list.append(
{
"name": model["id"],
"model_types": [LLMType.ASR.value],
"features": [],
"max_tokens": 8192,
}
)
return model_list
class VLLM(OpenAIAPICompatible):
_FACTORY_NAME = "VLLM"

13
rag/llm/sequence2txt_model.py
View File

@@ -408,3 +408,16 @@ class NewAPISeq2txt(GPTSeq2txt):
raise ValueError("url cannot be None")
model_name = model_name.split("___")[0]
super().__init__(key, model_name=model_name, base_url=base_url, **kwargs)
class FunASRSeq2txt(GPTSeq2txt):
"""FunASR speech-to-text provider for its OpenAI-compatible API."""
_FACTORY_NAME = "FunASR"
def __init__(self, key, model_name="sensevoice", base_url="http://localhost:8000/v1", **kwargs):
"""Initialize a client for a FunASR OpenAI-compatible endpoint."""
if not base_url:
base_url = "http://localhost:8000/v1"
super().__init__(key=key or "funasr", model_name=model_name, base_url=base_url, **kwargs)
logging.info("[FunASR] Speech2Text initialized with model %s at %s", model_name, self.base_url)

55
test/unit_test/rag/llm/test_model_meta_funasr.py Normal file
View File

@@ -0,0 +1,55 @@
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common.constants import LLMType
from rag.llm.model_meta import FunASR
pytestmark = pytest.mark.p2
def test_funasr_formats_server_models_as_speech_to_text():
provider = FunASR(api_key="", base_url="http://localhost:8000/v1")
models = provider._format_model_list(
{
"object": "list",
"data": [
{"id": "fun-asr-nano", "object": "model"},
{"id": "sensevoice", "object": "model"},
{"object": "model"},
],
}
)
assert models == [
{"name": "fun-asr-nano", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192},
{"name": "sensevoice", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192},
]
def test_funasr_rejects_malformed_model_lists():
provider = FunASR(api_key="", base_url="http://localhost:8000/v1")
assert provider._format_model_list({}) == []
assert provider._format_model_list({"data": "sensevoice"}) == []
def test_funasr_uses_openai_compatible_models_endpoint():
assert FunASR(api_key="", base_url="http://localhost:8000/v1")._get_model_list_url() == "http://localhost:8000/v1/models"
assert FunASR(api_key="", base_url="http://localhost:8000")._get_model_list_url() == "http://localhost:8000/v1/models"

74
test/unit_test/rag/llm/test_sequence2txt_funasr.py Normal file
View File

@@ -0,0 +1,74 @@
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from rag.llm.sequence2txt_model import FunASRSeq2txt
pytestmark = pytest.mark.p2
@patch("rag.llm.sequence2txt_model.OpenAI")
def test_funasr_defaults_to_local_sensevoice(mock_openai):
provider = FunASRSeq2txt(key="")
mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1")
assert provider._FACTORY_NAME == "FunASR"
assert provider.model_name == "sensevoice"
assert provider.base_url == "http://localhost:8000/v1"
@patch("rag.llm.sequence2txt_model.OpenAI")
def test_funasr_forwards_custom_connection_settings(mock_openai):
provider = FunASRSeq2txt(
key="local-secret",
model_name="paraformer",
base_url="http://funasr.internal:9000",
)
mock_openai.assert_called_once_with(api_key="local-secret", base_url="http://funasr.internal:9000/v1")
assert provider.model_name == "paraformer"
@patch("rag.llm.sequence2txt_model.OpenAI")
def test_funasr_empty_base_url_uses_local_default(mock_openai):
FunASRSeq2txt(key=None, base_url="")
mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1")
@patch("rag.llm.sequence2txt_model.OpenAI")
def test_funasr_transcription_uses_openai_compatible_endpoint(mock_openai, tmp_path):
audio_path = tmp_path / "sample.wav"
audio_path.write_bytes(b"RIFF-test-audio")
client = MagicMock()
client.audio.transcriptions.create.return_value = SimpleNamespace(text=" hello from FunASR ")
mock_openai.return_value = client
provider = FunASRSeq2txt(key="", model_name="sensevoice")
with patch("rag.llm.sequence2txt_model.num_tokens_from_string", return_value=4):
text, token_count = provider.transcription(audio_path)
assert text == "hello from FunASR"
assert token_count == 4
call = client.audio.transcriptions.create.call_args
assert call.kwargs["model"] == "sensevoice"
assert call.kwargs["file"].closed

1
web/src/constants/llm.ts
View File

@@ -74,6 +74,7 @@ export enum LLMFactory {
RAGcon = 'RAGcon',
Perplexity = 'Perplexity',
NewAPI = 'New API',
FunASR = 'FunASR',
}
// Please lowercase the file name

1
web/src/pages/user-setting/constants.tsx
View File

@@ -40,6 +40,7 @@ export const LocalLlmFactories = [
LLMFactory.ModelScope,
LLMFactory.VLLM,
LLMFactory.RAGcon,
LLMFactory.FunASR,
];
export enum TenantRole {

1
web/src/pages/user-setting/setting-model/provider-schema/constants.ts
View File

@@ -38,6 +38,7 @@ export const LIST_MODEL_PROVIDERS = new Set<string>([
LLMFactory.VolcEngine,
LLMFactory.Xinference,
LLMFactory.LocalAI,
LLMFactory.FunASR,
LLMFactory.BaiduYiYan,
LLMFactory.NewAPI,

38
web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.test.ts Normal file
View File

@@ -0,0 +1,38 @@
import { LLMFactory } from '@/constants/llm';
import { LIST_MODEL_PROVIDERS } from '../constants';
import { LocalLlmConfigs } from './local-llm-configs';
jest.mock('@/components/dynamic-form', () => ({
FormFieldType: {
Password: 'password',
Switch: 'switch',
Text: 'text',
},
}));
describe('FunASR local provider configuration', () => {
it('uses the model picker backed by the FunASR models endpoint', () => {
expect(LIST_MODEL_PROVIDERS.has(LLMFactory.FunASR)).toBe(true);
});
it('registers an optional-key local endpoint with the FunASR default URL', () => {
const config = LocalLlmConfigs[LLMFactory.FunASR];
expect(config).toMatchObject({
llmFactory: LLMFactory.FunASR,
title: 'FunASR',
docLink: 'https://github.com/modelscope/FunASR',
});
expect(
config.fields.find((field) => field.name === 'base_url'),
).toMatchObject({
required: true,
defaultValue: 'http://localhost:8000/v1',
});
expect(
config.fields.find((field) => field.name === 'api_key'),
).toMatchObject({
required: false,
});
});
});

19
web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts
View File

@@ -42,6 +42,25 @@ export const LocalLlmConfigs: Record<string, ProviderConfig> = {
undefined,
'https://inference.readthedocs.io/en/latest/user_guide',
),
[LLMFactory.FunASR]: buildLocalConfig(
LLMFactory.FunASR,
'FunASR',
['speech2text'],
undefined,
false,
[
{
name: 'base_url',
label: 'addLlmBaseUrl',
type: 'inputSelect',
required: true,
defaultValue: 'http://localhost:8000/v1',
placeholder: 'baseUrlNameMessage',
shouldRender: 'hideWhenInstanceExists',
},
],
'https://github.com/modelscope/FunASR',
),
[LLMFactory.ModelScope]: buildLocalConfig(
LLMFactory.ModelScope,
'ModelScope',