Fix: verify model (#16951)

api/apps/services/provider_api_service.py

@@ -25,7 +25,7 @@ from api.db.services.tenant_model_provider_service import TenantModelProviderSer
 from api.db.services.tenant_model_instance_service import TenantModelInstanceService
 from api.db.services.tenant_model_service import TenantModelService
 from api.utils.model_utils import get_model_type_human, calculate_model_type
-from rag.llm import ChatModel, EmbeddingModel, ModelMeta, OcrModel, RerankModel, TTSModel
+from rag.llm import ChatModel, CvModel, EmbeddingModel, ModelMeta, OcrModel, RerankModel, Seq2txtModel, TTSModel
 
 
 def _to_int(v, default=500):
@@ -651,7 +651,7 @@ async def verify_api_key(provider_id_or_name: str, api_key: str | dict, base_url
 
     model_verify_result = {}
     # test if api key works
-    chat_passed, embd_passed, rerank_passed, ocr_passed, tts_passed = False, False, False, False, False
+    chat_passed, embd_passed, rerank_passed, ocr_passed, tts_passed, asr_passed, vlm_passed = False, False, False, False, False, False, False
     timeout_seconds = int(os.environ.get("LLM_TIMEOUT_SECONDS", 10))
     extra = {"provider": provider_name}
     msg = ""
@@ -782,11 +782,62 @@ async def verify_api_key(provider_id_or_name: str, api_key: str | dict, base_url
                 )
                 model_verify_result[llm["llm_name"]] = ModelVerifyStatusEnum.FAIL.value
                 msg += f"\nFail to access model({provider_name}/{llm['llm_name']})." + str(e)
-        if any([embd_passed, chat_passed, rerank_passed, ocr_passed, tts_passed]):
+        elif not vlm_passed and LLMType.VISION.value in model_types:
+            if provider_name not in CvModel:
+                unsupported_msg = f"Image to text model from {provider_name} is not supported yet."
+                logging.warning(unsupported_msg)
+                msg += f"\n{unsupported_msg}"
+                continue
+            from rag.utils.base64_image import test_image
+
+            mdl = CvModel[provider_name](key=api_key_str, model_name=llm["llm_name"], base_url=base_url)
+            try:
+                image_data = test_image
+                m, tc = await asyncio.wait_for(
+                    asyncio.to_thread(mdl.describe, image_data),
+                    timeout=timeout_seconds,
+                )
+                if not tc and m.find("**ERROR**:") >= 0:
+                    raise Exception(m)
+                vlm_passed = True
+                model_verify_result[llm["llm_name"]] = ModelVerifyStatusEnum.SUCCESS.value
+            except Exception as e:
+                logging.exception(
+                    "Fail to access vision model for provider=%s model=%s",
+                    provider_name,
+                    llm["llm_name"],
+                )
+                model_verify_result[llm["llm_name"]] = ModelVerifyStatusEnum.FAIL.value
+                msg += f"\nFail to access model({provider_name}/{llm['llm_name']})." + str(e)
+        elif not asr_passed and LLMType.ASR.value in model_types:
+            if provider_name not in Seq2txtModel:
+                unsupported_msg = f"Speech model from {provider_name} is not supported yet."
+                logging.warning(unsupported_msg)
+                msg += f"\n{unsupported_msg}"
+                continue
+            mdl = Seq2txtModel[provider_name](key=api_key_str, model_name=llm["llm_name"], base_url=base_url)
+            try:
+                ok, reason = await asyncio.wait_for(
+                    asyncio.to_thread(mdl.check_available),
+                    timeout=timeout_seconds,
+                )
+                if not ok:
+                    raise RuntimeError(reason or "Model not available")
+                asr_passed = True
+                model_verify_result[llm["llm_name"]] = ModelVerifyStatusEnum.SUCCESS.value
+            except Exception as e:
+                logging.exception(
+                    "Fail to access ASR model for provider=%s model=%s",
+                    provider_name,
+                    llm["llm_name"],
+                )
+                model_verify_result[llm["llm_name"]] = ModelVerifyStatusEnum.FAIL.value
+                msg += f"\nFail to access model({provider_name}/{llm['llm_name']})." + str(e)
+        if any([embd_passed, chat_passed, rerank_passed, ocr_passed, tts_passed, vlm_passed, asr_passed]):
             msg = ""
             break
 
-    success = any([embd_passed, chat_passed, rerank_passed, ocr_passed, tts_passed])
+    success = any([embd_passed, chat_passed, rerank_passed, ocr_passed, tts_passed, vlm_passed, asr_passed])
     return success, "success" if success else msg, model_verify_result
 
 

rag/llm/sequence2txt_model.py

@@ -18,6 +18,7 @@ import io
 import json
 import os
 import re
+import struct
 from abc import ABC
 import tempfile
 import logging
@@ -44,6 +45,47 @@ class Base(ABC):
             transcription = self.client.audio.transcriptions.create(model=self.model_name, file=audio_file)
         return transcription.text.strip(), num_tokens_from_string(transcription.text.strip())
 
+    @staticmethod
+    def _generate_test_wav(duration_seconds=0.5, sample_rate=16000):
+        """Generate a minimal silent WAV file as bytes (pure stdlib, no dependencies)."""
+        n_samples = int(sample_rate * duration_seconds)
+        data_size = n_samples * 2  # 16-bit mono = 2 bytes per sample
+        header = struct.pack(
+            "<4sI4s4sIHHIIHH4sI",
+            b"RIFF",
+            36 + data_size,
+            b"WAVE",
+            b"fmt ",
+            16,
+            1,
+            1,
+            sample_rate,
+            sample_rate * 2,
+            2,
+            16,
+            b"data",
+            data_size,
+        )
+        return header + b"\x00" * data_size
+
+    def check_available(self) -> tuple[bool, str]:
+        """Check if the ASR model is available by transcribing a minimal test WAV."""
+        try:
+            wav_data = self._generate_test_wav()
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+                f.write(wav_data)
+                temp_path = f.name
+            try:
+                text, _ = self.transcription(temp_path)
+                if text.find("**ERROR**") >= 0:
+                    return False, text.replace("**ERROR**: ", "").strip()
+                return True, ""
+            finally:
+                if os.path.exists(temp_path):
+                    os.unlink(temp_path)
+        except Exception as e:
+            return False, str(e)
+
     def audio2base64(self, audio):
         if isinstance(audio, bytes):
             return base64.b64encode(audio).decode("utf-8")
@@ -332,6 +374,17 @@ class TencentCloudSeq2txt(Base):
         self.client = asr_client.AsrClient(cred, "")
         self.model_name = model_name
 
+    def check_available(self) -> tuple[bool, str]:
+        """Tencent Cloud ASR transcription expects raw bytes, not a file path."""
+        try:
+            wav_data = self._generate_test_wav()
+            text, _ = self.transcription(wav_data)
+            if text.find("**ERROR**") >= 0:
+                return False, text.replace("**ERROR**: ", "").strip()
+            return True, ""
+        except Exception as e:
+            return False, str(e)
+
     def transcription(self, audio, max_retries=60, retry_interval=5):
         import time
 
@@ -392,6 +445,10 @@ class GPUStackSeq2txt(Base):
         self.model_name = model_name
         self.key = key
 
+    def check_available(self) -> tuple[bool, str]:
+        """GPUStack ASR transcription endpoint is not yet implemented."""
+        return False, "GPUStack ASR transcription is not yet implemented"
+
 
 class GiteeSeq2txt(Base):
     _FACTORY_NAME = "GiteeAI"