From 06e36d24f4bea0460e4a32fe4671e27cd11c4c80 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Wed, 15 Jul 2026 19:02:05 +0800 Subject: [PATCH] feat(stt): add FunASR / SenseVoice provider (#16473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Adds FunASR as a self-hosted speech-to-text provider through its OpenAI-compatible `/v1/audio/transcriptions` endpoint. This is a focused replacement for #15526 by @Rene0422 and relates to #15448. The unrelated Markdown parser changes from the previous branch are intentionally removed so this PR contains only the FunASR provider integration. - register FunASR as a `SPEECH2TEXT` factory; - add `FunASRSeq2txt` with `sensevoice` and `http://localhost:8000/v1` defaults, an optional API key, URL normalization, and inherited transcription handling; - wire FunASR into the current local-provider schema with a prefilled local URL and official documentation link; - discover the server's `/v1/models` dynamically and expose every returned model as speech-to-text in the model picker; - use RAGFlow's existing default provider icon fallback instead of referencing a missing `funasr` asset; - list FunASR in the supported-provider documentation; - add focused backend and frontend regression tests. ### Validation - focused backend pytest suite -> `7 passed` - real CPU `funasr-server` + RAGFlow provider smoke test -> discovered `fun-asr-nano`, `sensevoice`, and `paraformer`; transcribed a real WAV as `我现在在录一段测试音频` (`10` tokens, `0.504s`) - `ruff check` and `ruff format --check` on the changed Python files - `python3 -m py_compile` on the provider and its test - JSON parse and a semantic assertion for exactly one enabled FunASR `SPEECH2TEXT` factory - focused frontend Jest test -> `2 passed` - ESLint and Prettier on all changed TypeScript files - `npm run build` -> production build succeeded (`14,181` modules transformed) - `git diff --check` ### Deployment Run FunASR separately and point the RAGFlow provider at it: ```bash pip install funasr funasr-server --device cuda --model sensevoice ``` The API key remains optional because the stock local server does not require authentication. A key can still be supplied when the endpoint is protected by a gateway. --------- Signed-off-by: LauraGPT Co-authored-by: LauraGPT --- conf/llm_factories.json | 7 ++ docs/guides/models/supported_models.mdx | 1 + rag/llm/model_meta.py | 23 ++++++ rag/llm/sequence2txt_model.py | 13 ++++ .../rag/llm/test_model_meta_funasr.py | 55 ++++++++++++++ .../rag/llm/test_sequence2txt_funasr.py | 74 +++++++++++++++++++ web/src/constants/llm.ts | 1 + web/src/pages/user-setting/constants.tsx | 1 + .../provider-schema/constants.ts | 1 + .../field-config/local-llm-configs.test.ts | 38 ++++++++++ .../field-config/local-llm-configs.ts | 19 +++++ 11 files changed, 233 insertions(+) create mode 100644 test/unit_test/rag/llm/test_model_meta_funasr.py create mode 100644 test/unit_test/rag/llm/test_sequence2txt_funasr.py create mode 100644 web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.test.ts diff --git a/conf/llm_factories.json b/conf/llm_factories.json index c1734a13d9..46fbd10db6 100644 --- a/conf/llm_factories.json +++ b/conf/llm_factories.json @@ -1929,6 +1929,13 @@ "status": "1", "llm": [] }, + { + "name": "FunASR", + "logo": "", + "tags": "SPEECH2TEXT", + "status": "1", + "llm": [] + }, { "name": "DeepSeek", "logo": "", diff --git a/docs/guides/models/supported_models.mdx b/docs/guides/models/supported_models.mdx index dc3071ecf2..fa4b7a355a 100644 --- a/docs/guides/models/supported_models.mdx +++ b/docs/guides/models/supported_models.mdx @@ -28,6 +28,7 @@ A complete list of model providers supported by RAGFlow, which will continue to | Cohere | `https://cohere.com` | | DeepSeek | `https://www.deepseek.com` | | Fish Audio | `https://fish.audio` | +| FunASR | `https://github.com/modelscope/FunASR` | | FuturMix | `https://futurmix.ai` | | Gemini | `https://gemini.google.com` | | Google Cloud | `https://cloud.google.com` | diff --git a/rag/llm/model_meta.py b/rag/llm/model_meta.py index 82f4a657f4..5aeb25e7f4 100644 --- a/rag/llm/model_meta.py +++ b/rag/llm/model_meta.py @@ -458,6 +458,29 @@ class OpenAIAPICompatible(Base): return model_list +class FunASR(Base): + _FACTORY_NAME = "FunASR" + + def _format_model_list(self, raw_model_list): + models = raw_model_list.get("data") if isinstance(raw_model_list, dict) else None + if not isinstance(models, list): + return [] + + model_list = [] + for model in models: + if not isinstance(model, dict) or not model.get("id"): + continue + model_list.append( + { + "name": model["id"], + "model_types": [LLMType.ASR.value], + "features": [], + "max_tokens": 8192, + } + ) + return model_list + + class VLLM(OpenAIAPICompatible): _FACTORY_NAME = "VLLM" diff --git a/rag/llm/sequence2txt_model.py b/rag/llm/sequence2txt_model.py index d21586a21e..993e7a8248 100644 --- a/rag/llm/sequence2txt_model.py +++ b/rag/llm/sequence2txt_model.py @@ -408,3 +408,16 @@ class NewAPISeq2txt(GPTSeq2txt): raise ValueError("url cannot be None") model_name = model_name.split("___")[0] super().__init__(key, model_name=model_name, base_url=base_url, **kwargs) + + +class FunASRSeq2txt(GPTSeq2txt): + """FunASR speech-to-text provider for its OpenAI-compatible API.""" + + _FACTORY_NAME = "FunASR" + + def __init__(self, key, model_name="sensevoice", base_url="http://localhost:8000/v1", **kwargs): + """Initialize a client for a FunASR OpenAI-compatible endpoint.""" + if not base_url: + base_url = "http://localhost:8000/v1" + super().__init__(key=key or "funasr", model_name=model_name, base_url=base_url, **kwargs) + logging.info("[FunASR] Speech2Text initialized with model %s at %s", model_name, self.base_url) diff --git a/test/unit_test/rag/llm/test_model_meta_funasr.py b/test/unit_test/rag/llm/test_model_meta_funasr.py new file mode 100644 index 0000000000..87e9a64e61 --- /dev/null +++ b/test/unit_test/rag/llm/test_model_meta_funasr.py @@ -0,0 +1,55 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from common.constants import LLMType +from rag.llm.model_meta import FunASR + + +pytestmark = pytest.mark.p2 + + +def test_funasr_formats_server_models_as_speech_to_text(): + provider = FunASR(api_key="", base_url="http://localhost:8000/v1") + + models = provider._format_model_list( + { + "object": "list", + "data": [ + {"id": "fun-asr-nano", "object": "model"}, + {"id": "sensevoice", "object": "model"}, + {"object": "model"}, + ], + } + ) + + assert models == [ + {"name": "fun-asr-nano", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192}, + {"name": "sensevoice", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192}, + ] + + +def test_funasr_rejects_malformed_model_lists(): + provider = FunASR(api_key="", base_url="http://localhost:8000/v1") + + assert provider._format_model_list({}) == [] + assert provider._format_model_list({"data": "sensevoice"}) == [] + + +def test_funasr_uses_openai_compatible_models_endpoint(): + assert FunASR(api_key="", base_url="http://localhost:8000/v1")._get_model_list_url() == "http://localhost:8000/v1/models" + assert FunASR(api_key="", base_url="http://localhost:8000")._get_model_list_url() == "http://localhost:8000/v1/models" diff --git a/test/unit_test/rag/llm/test_sequence2txt_funasr.py b/test/unit_test/rag/llm/test_sequence2txt_funasr.py new file mode 100644 index 0000000000..39fbbd4607 --- /dev/null +++ b/test/unit_test/rag/llm/test_sequence2txt_funasr.py @@ -0,0 +1,74 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from rag.llm.sequence2txt_model import FunASRSeq2txt + + +pytestmark = pytest.mark.p2 + + +@patch("rag.llm.sequence2txt_model.OpenAI") +def test_funasr_defaults_to_local_sensevoice(mock_openai): + provider = FunASRSeq2txt(key="") + + mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1") + assert provider._FACTORY_NAME == "FunASR" + assert provider.model_name == "sensevoice" + assert provider.base_url == "http://localhost:8000/v1" + + +@patch("rag.llm.sequence2txt_model.OpenAI") +def test_funasr_forwards_custom_connection_settings(mock_openai): + provider = FunASRSeq2txt( + key="local-secret", + model_name="paraformer", + base_url="http://funasr.internal:9000", + ) + + mock_openai.assert_called_once_with(api_key="local-secret", base_url="http://funasr.internal:9000/v1") + assert provider.model_name == "paraformer" + + +@patch("rag.llm.sequence2txt_model.OpenAI") +def test_funasr_empty_base_url_uses_local_default(mock_openai): + FunASRSeq2txt(key=None, base_url="") + + mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1") + + +@patch("rag.llm.sequence2txt_model.OpenAI") +def test_funasr_transcription_uses_openai_compatible_endpoint(mock_openai, tmp_path): + audio_path = tmp_path / "sample.wav" + audio_path.write_bytes(b"RIFF-test-audio") + + client = MagicMock() + client.audio.transcriptions.create.return_value = SimpleNamespace(text=" hello from FunASR ") + mock_openai.return_value = client + provider = FunASRSeq2txt(key="", model_name="sensevoice") + + with patch("rag.llm.sequence2txt_model.num_tokens_from_string", return_value=4): + text, token_count = provider.transcription(audio_path) + + assert text == "hello from FunASR" + assert token_count == 4 + call = client.audio.transcriptions.create.call_args + assert call.kwargs["model"] == "sensevoice" + assert call.kwargs["file"].closed diff --git a/web/src/constants/llm.ts b/web/src/constants/llm.ts index 9e3ebc162d..b7e33919dd 100644 --- a/web/src/constants/llm.ts +++ b/web/src/constants/llm.ts @@ -74,6 +74,7 @@ export enum LLMFactory { RAGcon = 'RAGcon', Perplexity = 'Perplexity', NewAPI = 'New API', + FunASR = 'FunASR', } // Please lowercase the file name diff --git a/web/src/pages/user-setting/constants.tsx b/web/src/pages/user-setting/constants.tsx index 1e32df7536..8f66c00150 100644 --- a/web/src/pages/user-setting/constants.tsx +++ b/web/src/pages/user-setting/constants.tsx @@ -40,6 +40,7 @@ export const LocalLlmFactories = [ LLMFactory.ModelScope, LLMFactory.VLLM, LLMFactory.RAGcon, + LLMFactory.FunASR, ]; export enum TenantRole { diff --git a/web/src/pages/user-setting/setting-model/provider-schema/constants.ts b/web/src/pages/user-setting/setting-model/provider-schema/constants.ts index 6508bf7e1a..76bc4784de 100644 --- a/web/src/pages/user-setting/setting-model/provider-schema/constants.ts +++ b/web/src/pages/user-setting/setting-model/provider-schema/constants.ts @@ -38,6 +38,7 @@ export const LIST_MODEL_PROVIDERS = new Set([ LLMFactory.VolcEngine, LLMFactory.Xinference, LLMFactory.LocalAI, + LLMFactory.FunASR, LLMFactory.BaiduYiYan, LLMFactory.NewAPI, diff --git a/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.test.ts b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.test.ts new file mode 100644 index 0000000000..9bcc67f0bc --- /dev/null +++ b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.test.ts @@ -0,0 +1,38 @@ +import { LLMFactory } from '@/constants/llm'; +import { LIST_MODEL_PROVIDERS } from '../constants'; +import { LocalLlmConfigs } from './local-llm-configs'; + +jest.mock('@/components/dynamic-form', () => ({ + FormFieldType: { + Password: 'password', + Switch: 'switch', + Text: 'text', + }, +})); + +describe('FunASR local provider configuration', () => { + it('uses the model picker backed by the FunASR models endpoint', () => { + expect(LIST_MODEL_PROVIDERS.has(LLMFactory.FunASR)).toBe(true); + }); + + it('registers an optional-key local endpoint with the FunASR default URL', () => { + const config = LocalLlmConfigs[LLMFactory.FunASR]; + + expect(config).toMatchObject({ + llmFactory: LLMFactory.FunASR, + title: 'FunASR', + docLink: 'https://github.com/modelscope/FunASR', + }); + expect( + config.fields.find((field) => field.name === 'base_url'), + ).toMatchObject({ + required: true, + defaultValue: 'http://localhost:8000/v1', + }); + expect( + config.fields.find((field) => field.name === 'api_key'), + ).toMatchObject({ + required: false, + }); + }); +}); diff --git a/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts index c098f8214a..1167b82728 100644 --- a/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts +++ b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts @@ -42,6 +42,25 @@ export const LocalLlmConfigs: Record = { undefined, 'https://inference.readthedocs.io/en/latest/user_guide', ), + [LLMFactory.FunASR]: buildLocalConfig( + LLMFactory.FunASR, + 'FunASR', + ['speech2text'], + undefined, + false, + [ + { + name: 'base_url', + label: 'addLlmBaseUrl', + type: 'inputSelect', + required: true, + defaultValue: 'http://localhost:8000/v1', + placeholder: 'baseUrlNameMessage', + shouldRender: 'hideWhenInstanceExists', + }, + ], + 'https://github.com/modelscope/FunASR', + ), [LLMFactory.ModelScope]: buildLocalConfig( LLMFactory.ModelScope, 'ModelScope',