mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-17 05:07:23 +08:00
feat(stt): add FunASR / SenseVoice provider (#16473)
### Summary Adds FunASR as a self-hosted speech-to-text provider through its OpenAI-compatible `/v1/audio/transcriptions` endpoint. This is a focused replacement for #15526 by @Rene0422 and relates to #15448. The unrelated Markdown parser changes from the previous branch are intentionally removed so this PR contains only the FunASR provider integration. - register FunASR as a `SPEECH2TEXT` factory; - add `FunASRSeq2txt` with `sensevoice` and `http://localhost:8000/v1` defaults, an optional API key, URL normalization, and inherited transcription handling; - wire FunASR into the current local-provider schema with a prefilled local URL and official documentation link; - discover the server's `/v1/models` dynamically and expose every returned model as speech-to-text in the model picker; - use RAGFlow's existing default provider icon fallback instead of referencing a missing `funasr` asset; - list FunASR in the supported-provider documentation; - add focused backend and frontend regression tests. ### Validation - focused backend pytest suite -> `7 passed` - real CPU `funasr-server` + RAGFlow provider smoke test -> discovered `fun-asr-nano`, `sensevoice`, and `paraformer`; transcribed a real WAV as `我现在在录一段测试音频` (`10` tokens, `0.504s`) - `ruff check` and `ruff format --check` on the changed Python files - `python3 -m py_compile` on the provider and its test - JSON parse and a semantic assertion for exactly one enabled FunASR `SPEECH2TEXT` factory - focused frontend Jest test -> `2 passed` - ESLint and Prettier on all changed TypeScript files - `npm run build` -> production build succeeded (`14,181` modules transformed) - `git diff --check` ### Deployment Run FunASR separately and point the RAGFlow provider at it: ```bash pip install funasr funasr-server --device cuda --model sensevoice ``` The API key remains optional because the stock local server does not require authentication. A key can still be supplied when the endpoint is protected by a gateway. --------- Signed-off-by: LauraGPT <LauraGPT@users.noreply.github.com> Co-authored-by: LauraGPT <LauraGPT@users.noreply.github.com>
This commit is contained in:
55
test/unit_test/rag/llm/test_model_meta_funasr.py
Normal file
55
test/unit_test/rag/llm/test_model_meta_funasr.py
Normal file
@@ -0,0 +1,55 @@
|
||||
#
|
||||
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import pytest
|
||||
|
||||
from common.constants import LLMType
|
||||
from rag.llm.model_meta import FunASR
|
||||
|
||||
|
||||
pytestmark = pytest.mark.p2
|
||||
|
||||
|
||||
def test_funasr_formats_server_models_as_speech_to_text():
|
||||
provider = FunASR(api_key="", base_url="http://localhost:8000/v1")
|
||||
|
||||
models = provider._format_model_list(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "fun-asr-nano", "object": "model"},
|
||||
{"id": "sensevoice", "object": "model"},
|
||||
{"object": "model"},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert models == [
|
||||
{"name": "fun-asr-nano", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192},
|
||||
{"name": "sensevoice", "model_types": [LLMType.ASR.value], "features": [], "max_tokens": 8192},
|
||||
]
|
||||
|
||||
|
||||
def test_funasr_rejects_malformed_model_lists():
|
||||
provider = FunASR(api_key="", base_url="http://localhost:8000/v1")
|
||||
|
||||
assert provider._format_model_list({}) == []
|
||||
assert provider._format_model_list({"data": "sensevoice"}) == []
|
||||
|
||||
|
||||
def test_funasr_uses_openai_compatible_models_endpoint():
|
||||
assert FunASR(api_key="", base_url="http://localhost:8000/v1")._get_model_list_url() == "http://localhost:8000/v1/models"
|
||||
assert FunASR(api_key="", base_url="http://localhost:8000")._get_model_list_url() == "http://localhost:8000/v1/models"
|
||||
74
test/unit_test/rag/llm/test_sequence2txt_funasr.py
Normal file
74
test/unit_test/rag/llm/test_sequence2txt_funasr.py
Normal file
@@ -0,0 +1,74 @@
|
||||
#
|
||||
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from rag.llm.sequence2txt_model import FunASRSeq2txt
|
||||
|
||||
|
||||
pytestmark = pytest.mark.p2
|
||||
|
||||
|
||||
@patch("rag.llm.sequence2txt_model.OpenAI")
|
||||
def test_funasr_defaults_to_local_sensevoice(mock_openai):
|
||||
provider = FunASRSeq2txt(key="")
|
||||
|
||||
mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1")
|
||||
assert provider._FACTORY_NAME == "FunASR"
|
||||
assert provider.model_name == "sensevoice"
|
||||
assert provider.base_url == "http://localhost:8000/v1"
|
||||
|
||||
|
||||
@patch("rag.llm.sequence2txt_model.OpenAI")
|
||||
def test_funasr_forwards_custom_connection_settings(mock_openai):
|
||||
provider = FunASRSeq2txt(
|
||||
key="local-secret",
|
||||
model_name="paraformer",
|
||||
base_url="http://funasr.internal:9000",
|
||||
)
|
||||
|
||||
mock_openai.assert_called_once_with(api_key="local-secret", base_url="http://funasr.internal:9000/v1")
|
||||
assert provider.model_name == "paraformer"
|
||||
|
||||
|
||||
@patch("rag.llm.sequence2txt_model.OpenAI")
|
||||
def test_funasr_empty_base_url_uses_local_default(mock_openai):
|
||||
FunASRSeq2txt(key=None, base_url="")
|
||||
|
||||
mock_openai.assert_called_once_with(api_key="funasr", base_url="http://localhost:8000/v1")
|
||||
|
||||
|
||||
@patch("rag.llm.sequence2txt_model.OpenAI")
|
||||
def test_funasr_transcription_uses_openai_compatible_endpoint(mock_openai, tmp_path):
|
||||
audio_path = tmp_path / "sample.wav"
|
||||
audio_path.write_bytes(b"RIFF-test-audio")
|
||||
|
||||
client = MagicMock()
|
||||
client.audio.transcriptions.create.return_value = SimpleNamespace(text=" hello from FunASR ")
|
||||
mock_openai.return_value = client
|
||||
provider = FunASRSeq2txt(key="", model_name="sensevoice")
|
||||
|
||||
with patch("rag.llm.sequence2txt_model.num_tokens_from_string", return_value=4):
|
||||
text, token_count = provider.transcription(audio_path)
|
||||
|
||||
assert text == "hello from FunASR"
|
||||
assert token_count == 4
|
||||
call = client.audio.transcriptions.create.call_args
|
||||
assert call.kwargs["model"] == "sensevoice"
|
||||
assert call.kwargs["file"].closed
|
||||
Reference in New Issue
Block a user