mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-18 05:37:24 +08:00
621 lines
24 KiB
Python
621 lines
24 KiB
Python
#
|
|
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
import base64
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
from abc import ABC
|
|
import tempfile
|
|
import logging
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
from openai import OpenAI
|
|
from openai.lib.azure import AzureOpenAI
|
|
|
|
from common.token_utils import num_tokens_from_string
|
|
from rag.utils.url_utils import ensure_v1
|
|
|
|
|
|
class Base(ABC):
|
|
def __init__(self, key, model_name, **kwargs):
|
|
"""
|
|
Abstract base class constructor.
|
|
Parameters are not stored; initialization is left to subclasses.
|
|
"""
|
|
pass
|
|
|
|
def transcription(self, audio_path, **kwargs):
|
|
with open(audio_path, "rb") as audio_file:
|
|
transcription = self.client.audio.transcriptions.create(model=self.model_name, file=audio_file)
|
|
return transcription.text.strip(), num_tokens_from_string(transcription.text.strip())
|
|
|
|
@staticmethod
|
|
def _generate_test_wav(duration_seconds=0.5, sample_rate=16000):
|
|
"""Generate a minimal silent WAV file as bytes (pure stdlib, no dependencies)."""
|
|
n_samples = int(sample_rate * duration_seconds)
|
|
data_size = n_samples * 2 # 16-bit mono = 2 bytes per sample
|
|
header = struct.pack(
|
|
"<4sI4s4sIHHIIHH4sI",
|
|
b"RIFF",
|
|
36 + data_size,
|
|
b"WAVE",
|
|
b"fmt ",
|
|
16,
|
|
1,
|
|
1,
|
|
sample_rate,
|
|
sample_rate * 2,
|
|
2,
|
|
16,
|
|
b"data",
|
|
data_size,
|
|
)
|
|
return header + b"\x00" * data_size
|
|
|
|
def check_available(self) -> tuple[bool, str]:
|
|
"""Check if the ASR model is available by transcribing a minimal test WAV."""
|
|
try:
|
|
wav_data = self._generate_test_wav()
|
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
|
f.write(wav_data)
|
|
temp_path = f.name
|
|
try:
|
|
text, _ = self.transcription(temp_path)
|
|
if text.find("**ERROR**") >= 0:
|
|
return False, text.replace("**ERROR**: ", "").strip()
|
|
return True, ""
|
|
finally:
|
|
if os.path.exists(temp_path):
|
|
os.unlink(temp_path)
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
def audio2base64(self, audio):
|
|
if isinstance(audio, bytes):
|
|
return base64.b64encode(audio).decode("utf-8")
|
|
if isinstance(audio, io.BytesIO):
|
|
return base64.b64encode(audio.getvalue()).decode("utf-8")
|
|
raise TypeError("The input audio file should be in binary format.")
|
|
|
|
|
|
class GPTSeq2txt(Base):
|
|
_FACTORY_NAME = "OpenAI"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="https://api.openai.com/v1", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://api.openai.com/v1"
|
|
self.base_url = ensure_v1(base_url)
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
self.model_name = model_name
|
|
|
|
|
|
class StepFunSeq2txt(GPTSeq2txt):
|
|
_FACTORY_NAME = "StepFun"
|
|
|
|
def __init__(self, key, model_name="step-asr", lang="Chinese", base_url="https://api.stepfun.com/v1", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://api.stepfun.com/v1"
|
|
super().__init__(key, model_name=model_name, base_url=base_url, **kwargs)
|
|
|
|
|
|
class FuturMixSeq2txt(GPTSeq2txt):
|
|
_FACTORY_NAME = "FuturMix"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="https://futurmix.ai/v1", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://futurmix.ai/v1"
|
|
super().__init__(key, model_name=model_name, base_url=base_url, **kwargs)
|
|
logging.info("[FuturMix] Speech2Text initialized with model %s", model_name)
|
|
|
|
|
|
class QWenSeq2txt(Base):
|
|
_FACTORY_NAME = "Tongyi-Qianwen"
|
|
_FUN_ASR_FLASH_PREFIX = "fun-asr-flash"
|
|
_FUN_ASR_BASE64_MAX_SIZE = 10 * 1024 * 1024
|
|
_DASHSCOPE_API_BASE = "https://dashscope.aliyuncs.com/api/v1"
|
|
_AUDIO_MIME_FORMATS = {
|
|
"audio/mpeg": "mp3",
|
|
"audio/mp3": "mp3",
|
|
"audio/wav": "wav",
|
|
"audio/wave": "wav",
|
|
"audio/x-wav": "wav",
|
|
}
|
|
|
|
def __init__(self, key, model_name="qwen-audio-asr", base_url=None, **kwargs):
|
|
import dashscope
|
|
|
|
dashscope.api_key = key
|
|
self.api_key = key
|
|
self.model_name = model_name
|
|
self.base_url = (base_url or self._DASHSCOPE_API_BASE).rstrip("/")
|
|
|
|
def transcription(self, audio_path):
|
|
# Fun-ASR-Flash uses DashScope's workspace-scoped native multimodal
|
|
# endpoint and payload instead of MultiModalConversation.
|
|
if self.model_name.startswith(self._FUN_ASR_FLASH_PREFIX):
|
|
return self._transcribe_fun_asr_flash(audio_path)
|
|
|
|
return self._transcribe_qwen_audio(audio_path)
|
|
|
|
def _transcribe_qwen_audio(self, audio_path):
|
|
import dashscope
|
|
|
|
if audio_path.startswith("http"):
|
|
audio_input = audio_path
|
|
else:
|
|
audio_input = f"file://{audio_path}"
|
|
|
|
messages = [{"role": "system", "content": [{"text": ""}]}, {"role": "user", "content": [{"audio": audio_input}]}]
|
|
|
|
resp = dashscope.MultiModalConversation.call(model=self.model_name, messages=messages, result_format="message", asr_options={"enable_lid": True, "enable_itn": False})
|
|
|
|
try:
|
|
text = resp["output"]["choices"][0]["message"].content[0]["text"]
|
|
except Exception as e:
|
|
text = "**ERROR**: " + str(e)
|
|
return text, num_tokens_from_string(text)
|
|
|
|
@classmethod
|
|
def _fun_asr_audio_format(cls, audio_path):
|
|
"""Derive the Fun-ASR audio format from a data URI, URL, or path."""
|
|
|
|
if audio_path.startswith("data:"):
|
|
mime_type = audio_path[5:].split(";", 1)[0].lower()
|
|
if not mime_type.startswith("audio/"):
|
|
raise ValueError(f"Unsupported audio data URI MIME type: {mime_type or 'missing'}")
|
|
audio_format = cls._AUDIO_MIME_FORMATS.get(mime_type, mime_type.split("/", 1)[1].removeprefix("x-"))
|
|
else:
|
|
path = urlparse(audio_path).path if audio_path.startswith(("http://", "https://")) else audio_path
|
|
audio_format = os.path.splitext(path)[1].lower().lstrip(".")
|
|
|
|
if audio_format == "wave":
|
|
audio_format = "wav"
|
|
if not audio_format:
|
|
raise ValueError("Cannot determine audio format; use a URL/path extension or an audio data URI MIME type")
|
|
return audio_format
|
|
|
|
@classmethod
|
|
def _validate_fun_asr_base64_size(cls, encoded_size):
|
|
if encoded_size > cls._FUN_ASR_BASE64_MAX_SIZE:
|
|
raise ValueError("Fun-ASR-Flash Base64 audio exceeds the 10 MB encoded-input limit; provide a publicly accessible URL (for example, OSS) instead")
|
|
|
|
def _fun_asr_flash_request(self, audio_path, *, stream=False):
|
|
audio_format = self._fun_asr_audio_format(audio_path)
|
|
|
|
if audio_path.startswith(("http://", "https://")):
|
|
audio_input = audio_path
|
|
elif audio_path.startswith("data:"):
|
|
_, separator, encoded_audio = audio_path.partition(",")
|
|
if not separator:
|
|
raise ValueError("Invalid audio data URI: missing Base64 payload")
|
|
self._validate_fun_asr_base64_size(len(encoded_audio.encode("utf-8")))
|
|
audio_input = audio_path
|
|
else:
|
|
file_size = os.path.getsize(audio_path)
|
|
encoded_size = 4 * ((file_size + 2) // 3)
|
|
self._validate_fun_asr_base64_size(encoded_size)
|
|
mime_type = "audio/mpeg" if audio_format == "mp3" else f"audio/{audio_format}"
|
|
with open(audio_path, "rb") as audio_file:
|
|
audio_input = f"data:{mime_type};base64,{base64.b64encode(audio_file.read()).decode('utf-8')}"
|
|
|
|
api_base = self.base_url
|
|
if api_base.endswith("/compatible-mode/v1"):
|
|
api_base = api_base[: -len("/compatible-mode/v1")] + "/api/v1"
|
|
url = f"{api_base}/services/aigc/multimodal-generation/generation"
|
|
payload = {
|
|
"model": self.model_name,
|
|
"input": {
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": [{"type": "input_audio", "input_audio": {"data": audio_input}}],
|
|
}
|
|
]
|
|
},
|
|
# sample_rate is optional in the Fun-ASR-Flash API. Omitting it
|
|
# avoids declaring incorrect metadata for remote or compressed audio.
|
|
"parameters": {"format": audio_format},
|
|
}
|
|
headers = {
|
|
"Authorization": f"Bearer {self.api_key}",
|
|
"Content-Type": "application/json",
|
|
"X-DashScope-SSE": "enable" if stream else "disable",
|
|
}
|
|
return url, headers, payload
|
|
|
|
def _transcribe_fun_asr_flash(self, audio_path):
|
|
try:
|
|
url, headers, payload = self._fun_asr_flash_request(audio_path)
|
|
|
|
response = requests.post(url, headers=headers, json=payload, timeout=60)
|
|
response.raise_for_status()
|
|
result = response.json()
|
|
text = result.get("text") or result.get("output", {}).get("text")
|
|
if not text:
|
|
raise ValueError("Missing transcription text in Fun-ASR-Flash response")
|
|
text = text.strip()
|
|
return text, num_tokens_from_string(text)
|
|
except Exception as e:
|
|
logging.exception("Fun-ASR-Flash transcription failed for model %s", self.model_name)
|
|
return "**ERROR**: " + str(e), 0
|
|
|
|
def _stream_fun_asr_flash(self, audio_path):
|
|
response = None
|
|
try:
|
|
url, headers, payload = self._fun_asr_flash_request(audio_path, stream=True)
|
|
response = requests.post(url, headers=headers, json=payload, timeout=60, stream=True)
|
|
response.raise_for_status()
|
|
|
|
full = ""
|
|
for line in response.iter_lines(decode_unicode=True):
|
|
if not line or not line.startswith("data:"):
|
|
continue
|
|
event_data = line[5:].strip()
|
|
if not event_data or event_data == "[DONE]":
|
|
continue
|
|
result = json.loads(event_data)
|
|
text = result.get("text") or result.get("output", {}).get("text")
|
|
if not text:
|
|
continue
|
|
full = text.strip()
|
|
yield {"event": "delta", "text": full}
|
|
|
|
if not full:
|
|
raise ValueError("Missing transcription text in Fun-ASR-Flash stream")
|
|
yield {"event": "final", "text": full}
|
|
except Exception as e:
|
|
logging.exception("Fun-ASR-Flash streaming transcription failed for model %s", self.model_name)
|
|
yield {"event": "error", "text": "**ERROR**: " + str(e)}
|
|
finally:
|
|
if response is not None:
|
|
response.close()
|
|
|
|
def stream_transcription(self, audio_path):
|
|
if self.model_name.startswith(self._FUN_ASR_FLASH_PREFIX):
|
|
yield from self._stream_fun_asr_flash(audio_path)
|
|
return
|
|
|
|
import dashscope
|
|
|
|
if audio_path.startswith("http"):
|
|
audio_input = audio_path
|
|
else:
|
|
audio_input = f"file://{audio_path}"
|
|
|
|
messages = [{"role": "system", "content": [{"text": ""}]}, {"role": "user", "content": [{"audio": audio_input}]}]
|
|
|
|
stream = dashscope.MultiModalConversation.call(model=self.model_name, messages=messages, result_format="message", stream=True, asr_options={"enable_lid": True, "enable_itn": False})
|
|
|
|
full = ""
|
|
for chunk in stream:
|
|
try:
|
|
piece = chunk["output"]["choices"][0]["message"].content[0]["text"]
|
|
full = piece
|
|
yield {"event": "delta", "text": piece}
|
|
except Exception as e:
|
|
yield {"event": "error", "text": str(e)}
|
|
|
|
yield {"event": "final", "text": full}
|
|
|
|
|
|
class AzureSeq2txt(Base):
|
|
_FACTORY_NAME = "Azure-OpenAI"
|
|
|
|
def __init__(self, key, model_name, lang="Chinese", **kwargs):
|
|
self.base_url = ensure_v1(kwargs["base_url"])
|
|
self.client = AzureOpenAI(api_key=key, azure_endpoint=self.base_url, api_version="2024-02-01")
|
|
self.model_name = model_name
|
|
self.lang = lang
|
|
|
|
|
|
class XinferenceSeq2txt(Base):
|
|
_FACTORY_NAME = "Xinference"
|
|
|
|
def __init__(self, key, model_name="whisper-small", **kwargs):
|
|
self.base_url = ensure_v1(kwargs["base_url"]) if kwargs.get("base_url") else None
|
|
self.model_name = model_name
|
|
self.key = key
|
|
|
|
def transcription(self, audio, language="zh", prompt=None, response_format="json", temperature=0.7):
|
|
if isinstance(audio, str):
|
|
with open(audio, "rb") as audio_file:
|
|
audio_data = audio_file.read()
|
|
audio_file_name = audio.split("/")[-1]
|
|
else:
|
|
audio_data = audio
|
|
audio_file_name = "audio.wav"
|
|
|
|
payload = {"model": self.model_name, "language": language, "prompt": prompt, "response_format": response_format, "temperature": temperature}
|
|
|
|
files = {"file": (audio_file_name, audio_data, "audio/wav")}
|
|
|
|
try:
|
|
response = requests.post(f"{self.base_url}/v1/audio/transcriptions", files=files, data=payload, timeout=60)
|
|
response.raise_for_status()
|
|
result = response.json()
|
|
|
|
if "text" in result:
|
|
transcription_text = result["text"].strip()
|
|
return transcription_text, num_tokens_from_string(transcription_text)
|
|
else:
|
|
return "**ERROR**: Failed to retrieve transcription.", 0
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
return f"**ERROR**: {str(e)}", 0
|
|
|
|
|
|
class TencentCloudSeq2txt(Base):
|
|
_FACTORY_NAME = "Tencent Cloud"
|
|
|
|
def __init__(self, key, model_name="16k_zh", base_url="https://asr.tencentcloudapi.com"):
|
|
from tencentcloud.asr.v20190614 import asr_client
|
|
from tencentcloud.common import credential
|
|
|
|
key = json.loads(key)
|
|
sid = key.get("tencent_cloud_sid", "")
|
|
sk = key.get("tencent_cloud_sk", "")
|
|
cred = credential.Credential(sid, sk)
|
|
self.client = asr_client.AsrClient(cred, "")
|
|
self.model_name = model_name
|
|
|
|
def check_available(self) -> tuple[bool, str]:
|
|
"""Tencent Cloud ASR transcription expects raw bytes, not a file path."""
|
|
try:
|
|
wav_data = self._generate_test_wav()
|
|
text, _ = self.transcription(wav_data)
|
|
if text.find("**ERROR**") >= 0:
|
|
return False, text.replace("**ERROR**: ", "").strip()
|
|
return True, ""
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
def transcription(self, audio, max_retries=60, retry_interval=5):
|
|
import time
|
|
|
|
from tencentcloud.asr.v20190614 import models
|
|
from tencentcloud.common.exception.tencent_cloud_sdk_exception import (
|
|
TencentCloudSDKException,
|
|
)
|
|
|
|
b64 = self.audio2base64(audio)
|
|
try:
|
|
# dispatch disk
|
|
req = models.CreateRecTaskRequest()
|
|
params = {
|
|
"EngineModelType": self.model_name,
|
|
"ChannelNum": 1,
|
|
"ResTextFormat": 0,
|
|
"SourceType": 1,
|
|
"Data": b64,
|
|
}
|
|
req.from_json_string(json.dumps(params))
|
|
resp = self.client.CreateRecTask(req)
|
|
|
|
# loop query
|
|
req = models.DescribeTaskStatusRequest()
|
|
params = {"TaskId": resp.Data.TaskId}
|
|
req.from_json_string(json.dumps(params))
|
|
retries = 0
|
|
while retries < max_retries:
|
|
resp = self.client.DescribeTaskStatus(req)
|
|
if resp.Data.StatusStr == "success":
|
|
text = re.sub(r"\[\d+:\d+\.\d+,\d+:\d+\.\d+\]\s*", "", resp.Data.Result).strip()
|
|
return text, num_tokens_from_string(text)
|
|
elif resp.Data.StatusStr == "failed":
|
|
return (
|
|
"**ERROR**: Failed to retrieve speech recognition results.",
|
|
0,
|
|
)
|
|
else:
|
|
time.sleep(retry_interval)
|
|
retries += 1
|
|
return "**ERROR**: Max retries exceeded. Task may still be processing.", 0
|
|
|
|
except TencentCloudSDKException as e:
|
|
return "**ERROR**: " + str(e), 0
|
|
except Exception as e:
|
|
return "**ERROR**: " + str(e), 0
|
|
|
|
|
|
class GPUStackSeq2txt(Base):
|
|
_FACTORY_NAME = "GPUStack"
|
|
|
|
def __init__(self, key, model_name, base_url):
|
|
if not base_url:
|
|
raise ValueError("url cannot be None")
|
|
if base_url.split("/")[-1] != "v1":
|
|
base_url = os.path.join(base_url, "v1")
|
|
self.base_url = base_url
|
|
self.model_name = model_name
|
|
self.key = key
|
|
|
|
def check_available(self) -> tuple[bool, str]:
|
|
"""GPUStack ASR transcription endpoint is not yet implemented."""
|
|
return False, "GPUStack ASR transcription is not yet implemented"
|
|
|
|
|
|
class GiteeSeq2txt(Base):
|
|
_FACTORY_NAME = "GiteeAI"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="https://ai.gitee.com/v1/", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://ai.gitee.com/v1/"
|
|
self.base_url = ensure_v1(base_url)
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
self.model_name = model_name
|
|
|
|
|
|
class DeepInfraSeq2txt(Base):
|
|
_FACTORY_NAME = "DeepInfra"
|
|
|
|
def __init__(self, key, model_name, base_url="https://api.deepinfra.com/v1/openai", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://api.deepinfra.com/v1/openai"
|
|
self.base_url = ensure_v1(base_url)
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
self.model_name = model_name
|
|
|
|
|
|
class CometAPISeq2txt(Base):
|
|
_FACTORY_NAME = "CometAPI"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="https://api.cometapi.com/v1", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://api.cometapi.com/v1"
|
|
self.base_url = ensure_v1(base_url)
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
self.model_name = model_name
|
|
|
|
|
|
class DeerAPISeq2txt(Base):
|
|
_FACTORY_NAME = "DeerAPI"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="https://api.deerapi.com/v1", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://api.deerapi.com/v1"
|
|
self.base_url = ensure_v1(base_url)
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
self.model_name = model_name
|
|
|
|
|
|
class ZhipuSeq2txt(Base):
|
|
_FACTORY_NAME = "ZHIPU-AI"
|
|
|
|
def __init__(self, key, model_name="glm-asr", base_url="https://open.bigmodel.cn/api/paas/v4", **kwargs):
|
|
if not base_url:
|
|
base_url = "https://open.bigmodel.cn/api/paas/v4"
|
|
self.base_url = base_url
|
|
self.api_key = key
|
|
self.model_name = model_name
|
|
self.gen_conf = kwargs.get("gen_conf", {})
|
|
self.stream = kwargs.get("stream", False)
|
|
|
|
def _convert_to_wav(self, input_path):
|
|
ext = os.path.splitext(input_path)[1].lower()
|
|
if ext in [".wav", ".mp3"]:
|
|
return input_path
|
|
fd, out_path = tempfile.mkstemp(suffix=".wav")
|
|
os.close(fd)
|
|
try:
|
|
import ffmpeg
|
|
import imageio_ffmpeg as ffmpeg_exe
|
|
|
|
ffmpeg_path = ffmpeg_exe.get_ffmpeg_exe()
|
|
(ffmpeg.input(input_path).output(out_path, ar=16000, ac=1).overwrite_output().run(cmd=ffmpeg_path, quiet=True))
|
|
return out_path
|
|
except Exception as e:
|
|
raise RuntimeError(f"audio convert failed: {e}")
|
|
|
|
def transcription(self, audio_path):
|
|
payload = {
|
|
"model": self.model_name,
|
|
"temperature": str(self.gen_conf.get("temperature", 0.75)) or "0.75",
|
|
"stream": self.stream,
|
|
}
|
|
|
|
headers = {"Authorization": f"Bearer {self.api_key}"}
|
|
converted = self._convert_to_wav(audio_path)
|
|
|
|
with open(converted, "rb") as audio_file:
|
|
files = {"file": audio_file}
|
|
|
|
try:
|
|
response = requests.post(
|
|
url=f"{self.base_url}/audio/transcriptions",
|
|
data=payload,
|
|
files=files,
|
|
headers=headers,
|
|
timeout=60,
|
|
)
|
|
body = response.json()
|
|
if response.status_code == 200:
|
|
full_content = body["text"]
|
|
return full_content, num_tokens_from_string(full_content)
|
|
else:
|
|
error = body["error"]
|
|
return f"**ERROR**: code: {error['code']}, message: {error['message']}", 0
|
|
except Exception as e:
|
|
return "**ERROR**: " + str(e), 0
|
|
|
|
|
|
class RAGconSeq2txt(Base):
|
|
"""
|
|
RAGcon Sequence2Text Provider - routes through LiteLLM proxy
|
|
|
|
Speech-to-text models routed through LiteLLM.
|
|
Default Base URL: https://connect.ragcon.com/v1
|
|
"""
|
|
|
|
_FACTORY_NAME = "RAGcon"
|
|
|
|
def __init__(self, key, model_name, base_url=None, lang="English", **kwargs):
|
|
# Use provided base_url or fallback to default
|
|
if not base_url:
|
|
base_url = "https://connect.ragcon.com/v1"
|
|
|
|
self.base_url = ensure_v1(base_url)
|
|
self.model_name = model_name
|
|
self.key = key
|
|
self.lang = lang
|
|
|
|
self.client = OpenAI(api_key=key, base_url=self.base_url)
|
|
|
|
def transcription(self, audio_path, **kwargs):
|
|
"""
|
|
Transcribe audio file using RAGcon's OpenAI-compatible API.
|
|
Uses Whisper's automatic language detection for German and English audio.
|
|
|
|
Args:
|
|
audio_path: Path to the audio file
|
|
**kwargs: Additional parameters (currently unused but maintained for compatibility)
|
|
|
|
Returns:
|
|
tuple: (transcribed_text, token_count)
|
|
"""
|
|
with open(audio_path, "rb") as audio_file:
|
|
# Call RAGcon API - Whisper will auto-detect language
|
|
transcription = self.client.audio.transcriptions.create(model=self.model_name, file=audio_file)
|
|
|
|
# Return text and token count
|
|
text = transcription.text.strip()
|
|
return text, num_tokens_from_string(text)
|
|
|
|
|
|
class NewAPISeq2txt(GPTSeq2txt):
|
|
_FACTORY_NAME = "New API"
|
|
|
|
def __init__(self, key, model_name="whisper-1", base_url="", **kwargs):
|
|
if not base_url:
|
|
raise ValueError("url cannot be None")
|
|
model_name = model_name.split("___")[0]
|
|
super().__init__(key, model_name=model_name, base_url=base_url, **kwargs)
|
|
|
|
|
|
class FunASRSeq2txt(GPTSeq2txt):
|
|
"""FunASR speech-to-text provider for its OpenAI-compatible API."""
|
|
|
|
_FACTORY_NAME = "FunASR"
|
|
|
|
def __init__(self, key, model_name="sensevoice", base_url="http://localhost:8000/v1", **kwargs):
|
|
"""Initialize a client for a FunASR OpenAI-compatible endpoint."""
|
|
if not base_url:
|
|
base_url = "http://localhost:8000/v1"
|
|
super().__init__(key=key or "funasr", model_name=model_name, base_url=base_url, **kwargs)
|
|
logging.info("[FunASR] Speech2Text initialized with model %s at %s", model_name, self.base_url)
|