From 3bfad1f00ef29db4e3246e3ae082c8924ab7c6b5 Mon Sep 17 00:00:00 2001 From: euvre <93761161+euvre@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:42:55 +0800 Subject: [PATCH] fix: correct model type mappings and improve system setting persistence (#16501) --- admin/server/auth.py | 2 +- agent/component/llm.py | 4 +- api/apps/llm_app.py | 6 +- api/apps/restful_apis/chat_api.py | 6 +- api/apps/restful_apis/openai_api.py | 2 +- api/apps/restful_apis/user_api.py | 2 +- api/apps/services/models_api_service.py | 4 +- api/db/db_models.py | 43 ++++++++++++- api/db/init_data.py | 2 +- api/db/joint_services/tenant_model_service.py | 8 +-- api/db/joint_services/user_account_service.py | 2 +- api/db/services/dialog_service.py | 12 ++-- api/db/services/tenant_llm_service.py | 14 ++--- api/utils/model_utils.py | 11 +++- common/constants.py | 8 +-- common/settings.py | 14 ++--- deepdoc/parser/figure_parser.py | 8 +-- deepdoc/parser/mineru_parser.py | 2 +- internal/dao/tenant.go | 4 +- internal/service/tenant.go | 12 ++-- rag/app/audio.py | 2 +- rag/app/naive.py | 10 +-- rag/app/picture.py | 4 +- rag/flow/parser/parser.py | 10 +-- rag/flow/parser/utils.py | 4 +- rag/llm/cv_model.py | 2 +- rag/llm/model_meta.py | 24 ++++---- rag/prompts/generator.py | 8 +-- test/testcases/restful_api/test_chats.py | 4 +- .../test_user_tenant_routes_unit.py | 4 +- .../test_chat_sdk_routes_unit.py | 4 +- .../test_session_sdk_routes_unit.py | 12 ++-- .../test_chunk_app/test_chunk_routes_unit.py | 4 +- .../test_llm_app/test_llm_list_unit.py | 8 +-- .../test_user_app/test_user_app_unit.py | 2 +- .../api/apps/services/test_delete_datasets.py | 16 ++--- ...ls_api_service_list_tenant_added_models.py | 4 +- tools/scripts/mysql_migration.py | 8 ++- web/src/components/llm-select/next.tsx | 8 +-- web/src/components/model-tree-select.tsx | 6 +- web/src/components/ui/modal/modal.tsx | 2 +- web/src/constants/llm.ts | 4 +- web/src/pages/agent/form/agent-form/index.tsx | 11 +--- .../setting-model/layout/system-setting.tsx | 61 ++++++++++++++++--- 44 files changed, 238 insertions(+), 150 deletions(-) diff --git a/admin/server/auth.py b/admin/server/auth.py index 6b140ac34f..2a9e5cf8f0 100644 --- a/admin/server/auth.py +++ b/admin/server/auth.py @@ -121,7 +121,7 @@ def add_tenant_for_admin(user_info: dict, role: str): "embd_id": settings.EMBEDDING_MDL, "asr_id": settings.ASR_MDL, "parser_ids": settings.PARSERS, - "img2txt_id": settings.IMAGE2TEXT_MDL, + "img2txt_id": settings.VISION_MDL, "rerank_id": settings.RERANK_MDL, } usr_tenant = {"tenant_id": user_info["id"], "user_id": user_info["id"], "invited_by": user_info["id"], "role": role} diff --git a/agent/component/llm.py b/agent/component/llm.py index 4b836f61fc..f50bd22edd 100644 --- a/agent/component/llm.py +++ b/agent/component/llm.py @@ -319,8 +319,8 @@ class LLM(ComponentBase): max(0, prev_img_count + len(sys_file_imgs) - len(self.imgs)), ) model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id) - if self.imgs and LLMType.IMAGE2TEXT.value in model_types: - model_type = LLMType.IMAGE2TEXT.value + if self.imgs and LLMType.VISION.value in model_types: + model_type = LLMType.VISION.value elif LLMType.CHAT.value in model_types: model_type = LLMType.CHAT.value else: diff --git a/api/apps/llm_app.py b/api/apps/llm_app.py index 8f52101f16..f83c61f250 100644 --- a/api/apps/llm_app.py +++ b/api/apps/llm_app.py @@ -62,7 +62,7 @@ def factories(): f["model_types"] = list( mdl_types.get( f["name"], - [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK, LLMType.IMAGE2TEXT, LLMType.SPEECH2TEXT, LLMType.TTS, LLMType.OCR], + [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK, LLMType.VISION, LLMType.ASR, LLMType.TTS, LLMType.OCR], ) ) @@ -377,7 +377,7 @@ async def add_llm(): except Exception as e: msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e) - case LLMType.IMAGE2TEXT.value: + case LLMType.VISION.value: from rag.utils.base64_image import test_image assert factory in CvModel, f"Image to text model from {factory} is not supported yet." @@ -419,7 +419,7 @@ async def add_llm(): raise RuntimeError(reason or "Model not available") except Exception as e: msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e) - case LLMType.SPEECH2TEXT.value: + case LLMType.ASR.value: assert factory in Seq2txtModel, f"Speech model from {factory} is not supported yet." try: mdl = Seq2txtModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url) diff --git a/api/apps/restful_apis/chat_api.py b/api/apps/restful_apis/chat_api.py index b219e1fd30..9398676a30 100644 --- a/api/apps/restful_apis/chat_api.py +++ b/api/apps/restful_apis/chat_api.py @@ -270,9 +270,9 @@ async def _validate_llm_id(llm_id, tenant_id, llm_setting=None): conf_model_type = (llm_setting or {}).get("model_type") if isinstance(conf_model_type, str): - model_type = conf_model_type if conf_model_type in {"chat", "image2text"} else "chat" + model_type = conf_model_type if conf_model_type in {"chat", "vision"} else "chat" elif isinstance(conf_model_type, list): - model_type = "image2text" if "image2text" in conf_model_type else "chat" + model_type = "vision" if "vision" in conf_model_type else "chat" else: model_type = "chat" try: @@ -1064,7 +1064,7 @@ async def transcription(): await uploaded.save(temp_audio_path) try: - default_asr_model_config = get_tenant_default_model_by_type(current_user.id, LLMType.SPEECH2TEXT) + default_asr_model_config = get_tenant_default_model_by_type(current_user.id, LLMType.ASR) except Exception as e: return get_data_error_result(message=str(e)) diff --git a/api/apps/restful_apis/openai_api.py b/api/apps/restful_apis/openai_api.py index 8f345563b5..1b881747d6 100644 --- a/api/apps/restful_apis/openai_api.py +++ b/api/apps/restful_apis/openai_api.py @@ -36,7 +36,7 @@ def _validate_llm_id(llm_id, tenant_id, llm_setting=None): return None model_type = (llm_setting or {}).get("model_type") - if model_type not in {"chat", "image2text"}: + if model_type not in {"chat", "vision"}: model_type = "chat" try: diff --git a/api/apps/restful_apis/user_api.py b/api/apps/restful_apis/user_api.py index 833fc451b4..bbb3676ffc 100644 --- a/api/apps/restful_apis/user_api.py +++ b/api/apps/restful_apis/user_api.py @@ -433,7 +433,7 @@ def user_register(user_id, user): "embd_id": settings.EMBEDDING_MDL, "asr_id": settings.ASR_MDL, "parser_ids": settings.PARSERS, - "img2txt_id": settings.IMAGE2TEXT_MDL, + "img2txt_id": settings.VISION_MDL, "rerank_id": settings.RERANK_MDL, } usr_tenant = { diff --git a/api/apps/services/models_api_service.py b/api/apps/services/models_api_service.py index 8c37bd6d44..b6fcf1492c 100644 --- a/api/apps/services/models_api_service.py +++ b/api/apps/services/models_api_service.py @@ -50,8 +50,8 @@ MODEL_TAG_TO_TYPE = { "chat": "chat", "embedding": "embedding", "rerank": "rerank", - "asr": "speech2text", - "vision": "image2text", + "asr": "asr", + "vision": "vision", "tts": "tts", "ocr": "ocr", } diff --git a/api/db/db_models.py b/api/db/db_models.py index ca4ba8a5a1..b8bff59dd4 100644 --- a/api/db/db_models.py +++ b/api/db/db_models.py @@ -1460,7 +1460,7 @@ class TenantModel(DataBaseModel): model_name = CharField(max_length=128, null=True, index=False, help_text="Model name") provider_id = CharField(max_length=32, null=False, index=False) instance_id = CharField(max_length=32, null=False, index=True) - model_type = IntegerField(null=False, default=1, index=True, help_text="Bit flags (LSB->MSB): 1=chat, 2=embedding, 4=speech2text, 8=image2text, 16=rerank, 32=tts, 64=ocr") + model_type = IntegerField(null=False, default=1, index=True, help_text="Bit flags (LSB->MSB): 1=chat, 2=embedding, 4=asr, 8=vision, 16=rerank, 32=tts, 64=ocr") status = CharField(max_length=32, default="active", index=False) extra = CharField(max_length=1024, default="{}", index=False) @@ -1833,3 +1833,44 @@ def migrate_db(): logging.disable(logging.NOTSET) # this is after re-enabling logging to allow logging changed user emails migrate_add_unique_email(migrator) + migrate_model_type_names() + + +def migrate_model_type_names(): + """Rename legacy model_type string values to the canonical asr/vision names. + + Previously the code used speech2text / image2text. LLMType now emits asr / + vision, and the backend compares model_type strings directly. This idempotent + data migration updates persisted rows in llm and tenant_llm before the new + enum values are used at runtime. + """ + RENAME_MAP = { + "speech2text": "asr", + "image2text": "vision", + } + tables = ["llm", "tenant_llm"] + for table in tables: + if not DB.table_exists(table): + continue + for old_name, new_name in RENAME_MAP.items(): + try: + cursor = DB.execute_sql( + "UPDATE {} SET model_type = %s WHERE model_type = %s".format(table), + (new_name, old_name), + ) + if cursor.rowcount: + logging.info( + "Migrated %s rows in %s.model_type from %s to %s", + cursor.rowcount, + table, + old_name, + new_name, + ) + except Exception as ex: + logging.warning( + "Failed to migrate model_type values in %s (from %s to %s): %s", + table, + old_name, + new_name, + ex, + ) diff --git a/api/db/init_data.py b/api/db/init_data.py index c755565a39..d42304a733 100644 --- a/api/db/init_data.py +++ b/api/db/init_data.py @@ -65,7 +65,7 @@ def init_superuser(nickname=DEFAULT_SUPERUSER_NICKNAME, email=DEFAULT_SUPERUSER_ "embd_id": settings.EMBEDDING_MDL, "asr_id": settings.ASR_MDL, "parser_ids": settings.PARSERS, - "img2txt_id": settings.IMAGE2TEXT_MDL, + "img2txt_id": settings.VISION_MDL, "rerank_id": settings.RERANK_MDL, } usr_tenant = {"tenant_id": user_info["id"], "user_id": user_info["id"], "invited_by": user_info["id"], "role": role} diff --git a/api/db/joint_services/tenant_model_service.py b/api/db/joint_services/tenant_model_service.py index eead80a10d..90796dfa4e 100644 --- a/api/db/joint_services/tenant_model_service.py +++ b/api/db/joint_services/tenant_model_service.py @@ -171,10 +171,10 @@ def get_tenant_default_model_by_type(tenant_id: str, model_type: str | enum.Enum case LLMType.EMBEDDING.value: model_id = tenant.tenant_embd_id model_name = tenant.embd_id - case LLMType.SPEECH2TEXT.value: + case LLMType.ASR.value: model_id = tenant.tenant_asr_id model_name = tenant.asr_id - case LLMType.IMAGE2TEXT.value: + case LLMType.VISION.value: model_id = tenant.tenant_img2txt_id model_name = tenant.img2txt_id case LLMType.CHAT.value: @@ -403,8 +403,8 @@ _MODEL_NAME_TO_ID_FIELD_MAP: dict[str, tuple[str, str]] = { "llm_id": (LLMType.CHAT, "tenant_llm_id"), "embd_id": (LLMType.EMBEDDING, "tenant_embd_id"), "rerank_id": (LLMType.RERANK, "tenant_rerank_id"), - "asr_id": (LLMType.SPEECH2TEXT, "tenant_asr_id"), - "img2txt_id": (LLMType.IMAGE2TEXT, "tenant_img2txt_id"), + "asr_id": (LLMType.ASR, "tenant_asr_id"), + "img2txt_id": (LLMType.VISION, "tenant_img2txt_id"), "tts_id": (LLMType.TTS, "tenant_tts_id"), } diff --git a/api/db/joint_services/user_account_service.py b/api/db/joint_services/user_account_service.py index 55e3eb5498..32b95781b8 100644 --- a/api/db/joint_services/user_account_service.py +++ b/api/db/joint_services/user_account_service.py @@ -67,7 +67,7 @@ def create_new_user(user_info: dict) -> dict: "embd_id": settings.EMBEDDING_MDL, "asr_id": settings.ASR_MDL, "parser_ids": settings.PARSERS, - "img2txt_id": settings.IMAGE2TEXT_MDL, + "img2txt_id": settings.VISION_MDL, "rerank_id": settings.RERANK_MDL, } usr_tenant = { diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py index 8987f25581..8572211a1a 100644 --- a/api/db/services/dialog_service.py +++ b/api/db/services/dialog_service.py @@ -299,19 +299,19 @@ async def async_chat_solo(dialog, messages, stream=True, session_id=None): if "chat" in llm_types: model_config = get_model_config_by_id(dialog.tenant_id, LLMType.CHAT, dialog.tenant_llm_id) else: - model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) except LookupError: llm_types = resolve_model_type(dialog.tenant_id, dialog.llm_id) if "chat" in llm_types: model_config = resolve_model_config(dialog.tenant_id, LLMType.CHAT, dialog.llm_id) else: - model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) else: llm_types = resolve_model_type(dialog.tenant_id, dialog.llm_id) if "chat" in llm_types: model_config = resolve_model_config(dialog.tenant_id, LLMType.CHAT, dialog.llm_id) else: - model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) else: model_config = get_tenant_default_model_by_type(dialog.tenant_id, LLMType.CHAT) @@ -587,19 +587,19 @@ async def async_chat(dialog, messages, stream=True, **kwargs): if "chat" in llm_types: llm_model_config = get_model_config_by_id(dialog.tenant_id, LLMType.CHAT, dialog.tenant_llm_id) else: - llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) except LookupError: llm_types = resolve_model_type(dialog.tenant_id, dialog.llm_id) if "chat" in llm_types: llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.CHAT, dialog.llm_id) else: - llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) else: llm_types = resolve_model_type(dialog.tenant_id, dialog.llm_id) if "chat" in llm_types: llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.CHAT, dialog.llm_id) else: - llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id) + llm_model_config = resolve_model_config(dialog.tenant_id, LLMType.VISION, dialog.llm_id) else: llm_model_config = get_tenant_default_model_by_type(dialog.tenant_id, LLMType.CHAT) diff --git a/api/db/services/tenant_llm_service.py b/api/db/services/tenant_llm_service.py index f9caa2100d..511ee1a950 100644 --- a/api/db/services/tenant_llm_service.py +++ b/api/db/services/tenant_llm_service.py @@ -135,9 +135,9 @@ class TenantLLMService(CommonService): if llm_type == LLMType.EMBEDDING.value: mdlnm = tenant.embd_id if not llm_name else llm_name - elif llm_type == LLMType.SPEECH2TEXT.value: + elif llm_type == LLMType.ASR.value: mdlnm = tenant.asr_id if not llm_name else llm_name - elif llm_type == LLMType.IMAGE2TEXT.value: + elif llm_type == LLMType.VISION.value: mdlnm = tenant.img2txt_id if not llm_name else llm_name elif llm_type == LLMType.CHAT.value: mdlnm = tenant.llm_id if not llm_name else llm_name @@ -198,7 +198,7 @@ class TenantLLMService(CommonService): return None return RerankModel[model_config["llm_factory"]](api_key, model_config["llm_name"], base_url=model_config["api_base"]) - elif model_config["model_type"] == LLMType.IMAGE2TEXT.value: + elif model_config["model_type"] == LLMType.VISION.value: if model_config["llm_factory"] not in CvModel: logging.error("Factory not in cv model. Supported factories: %s", list(CvModel.keys())) return None @@ -210,9 +210,9 @@ class TenantLLMService(CommonService): return None return ChatModel[model_config["llm_factory"]](api_key, model_config["llm_name"], base_url=model_config["api_base"], **kwargs) - elif model_config["model_type"] == LLMType.SPEECH2TEXT.value: + elif model_config["model_type"] == LLMType.ASR.value: if model_config["llm_factory"] not in Seq2txtModel: - logging.error("Factory not in speech2text model. Supported factories: %s", list(Seq2txtModel.keys())) + logging.error("Factory not in asr model. Supported factories: %s", list(Seq2txtModel.keys())) return None return Seq2txtModel[model_config["llm_factory"]](key=api_key, model_name=model_config["llm_name"], lang=lang, base_url=model_config["api_base"]) elif model_config["model_type"] == LLMType.TTS.value: @@ -248,8 +248,8 @@ class TenantLLMService(CommonService): llm_map = { LLMType.EMBEDDING.value: tenant.embd_id if not llm_name else llm_name, - LLMType.SPEECH2TEXT.value: tenant.asr_id, - LLMType.IMAGE2TEXT.value: tenant.img2txt_id, + LLMType.ASR.value: tenant.asr_id, + LLMType.VISION.value: tenant.img2txt_id, LLMType.CHAT.value: tenant.llm_id if not llm_name else llm_name, LLMType.RERANK.value: tenant.rerank_id if not llm_name else llm_name, LLMType.TTS.value: tenant.tts_id if not llm_name else llm_name, diff --git a/api/utils/model_utils.py b/api/utils/model_utils.py index f28dc37926..f2a462de3b 100644 --- a/api/utils/model_utils.py +++ b/api/utils/model_utils.py @@ -22,12 +22,19 @@ def get_model_type_human(model_type: int) -> List[str]: return [mt.name.lower() for mt in ModelTypeBinary if model_type & mt.value] +_LEGACY_MODEL_TYPE_ALIASES = { + "speech2text": "asr", + "image2text": "vision", +} + + def calculate_model_type(model_type_name_list: List[str]|str) -> int: model_type = 0 if isinstance(model_type_name_list, str): model_type_name_list = [model_type_name_list] type_value_map = {mt.name.lower(): mt.value for mt in ModelTypeBinary} for mt in model_type_name_list: - if mt in type_value_map: - model_type |= type_value_map[mt] + normalized_mt = _LEGACY_MODEL_TYPE_ALIASES.get(mt, mt) + if normalized_mt in type_value_map: + model_type |= type_value_map[normalized_mt] return model_type diff --git a/common/constants.py b/common/constants.py index 3c1dd63993..b171263b91 100644 --- a/common/constants.py +++ b/common/constants.py @@ -86,8 +86,8 @@ class ActiveEnum(Enum): class LLMType(StrEnum): CHAT = "chat" EMBEDDING = "embedding" - SPEECH2TEXT = "speech2text" - IMAGE2TEXT = "image2text" + ASR = "asr" + VISION = "vision" RERANK = "rerank" TTS = "tts" OCR = "ocr" @@ -96,8 +96,8 @@ class LLMType(StrEnum): class ModelTypeBinary(Enum): CHAT = 0b0000001 # 1 << 0 = 1 EMBEDDING = 0b0000010 # 1 << 1 = 2 - SPEECH2TEXT = 0b0000100 # 1 << 2 = 4 - IMAGE2TEXT = 0b0001000 # 1 << 3 = 8 + ASR = 0b0000100 # 1 << 2 = 4 + VISION = 0b0001000 # 1 << 3 = 8 RERANK = 0b0010000 # 1 << 4 = 16 TTS = 0b0100000 # 1 << 5 = 32 OCR = 0b1000000 # 1 << 6 = 64 diff --git a/common/settings.py b/common/settings.py index d5270eb506..a28d09a0eb 100644 --- a/common/settings.py +++ b/common/settings.py @@ -54,14 +54,14 @@ CHAT_MDL = "" EMBEDDING_MDL = "" RERANK_MDL = "" ASR_MDL = "" -IMAGE2TEXT_MDL = "" +VISION_MDL = "" CHAT_CFG = "" EMBEDDING_CFG = "" RERANK_CFG = "" ASR_CFG = "" -IMAGE2TEXT_CFG = "" +VISION_CFG = "" API_KEY = None PARSERS = None HOST_IP = None @@ -259,19 +259,19 @@ def init_settings(): "parsers", "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag" ) - global CHAT_MDL, EMBEDDING_MDL, RERANK_MDL, ASR_MDL, IMAGE2TEXT_MDL + global CHAT_MDL, EMBEDDING_MDL, RERANK_MDL, ASR_MDL, VISION_MDL chat_entry = _parse_model_entry(llm_default_models.get("chat_model", CHAT_MDL)) embedding_entry = _parse_model_entry(llm_default_models.get("embedding_model", EMBEDDING_MDL)) rerank_entry = _parse_model_entry(llm_default_models.get("rerank_model", RERANK_MDL)) asr_entry = _parse_model_entry(llm_default_models.get("asr_model", ASR_MDL)) - image2text_entry = _parse_model_entry(llm_default_models.get("image2text_model", IMAGE2TEXT_MDL)) + vision_entry = _parse_model_entry(llm_default_models.get("vision_model", VISION_MDL)) - global CHAT_CFG, EMBEDDING_CFG, RERANK_CFG, ASR_CFG, IMAGE2TEXT_CFG + global CHAT_CFG, EMBEDDING_CFG, RERANK_CFG, ASR_CFG, VISION_CFG CHAT_CFG = _resolve_per_model_config(chat_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) EMBEDDING_CFG = _resolve_per_model_config(embedding_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) RERANK_CFG = _resolve_per_model_config(rerank_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) ASR_CFG = _resolve_per_model_config(asr_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) - IMAGE2TEXT_CFG = _resolve_per_model_config(image2text_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) + VISION_CFG = _resolve_per_model_config(vision_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL) CHAT_MDL = CHAT_CFG.get("model", "") or "" EMBEDDING_MDL = EMBEDDING_CFG.get("model", "") or "" @@ -280,7 +280,7 @@ def init_settings(): EMBEDDING_MDL = os.getenv("TEI_MODEL", EMBEDDING_MDL or "BAAI/bge-small-en-v1.5") RERANK_MDL = RERANK_CFG.get("model", "") or "" ASR_MDL = ASR_CFG.get("model", "") or "" - IMAGE2TEXT_MDL = IMAGE2TEXT_CFG.get("model", "") or "" + VISION_MDL = VISION_CFG.get("model", "") or "" global HOST_IP, HOST_PORT HOST_IP = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") diff --git a/deepdoc/parser/figure_parser.py b/deepdoc/parser/figure_parser.py index d37a5e7309..b684df07ce 100644 --- a/deepdoc/parser/figure_parser.py +++ b/deepdoc/parser/figure_parser.py @@ -50,7 +50,7 @@ def vision_figure_parser_docx_wrapper(sections, tbls, callback=None, **kwargs): if not sections: return tbls try: - vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.VISION) vision_model = LLMBundle(kwargs["tenant_id"], vision_model_config) callback(0.7, "Visual model detected. Attempting to enhance figure extraction...") except Exception: @@ -71,7 +71,7 @@ def vision_figure_parser_figure_xlsx_wrapper(images, callback=None, **kwargs): if not images: return [] try: - vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.VISION) vision_model = LLMBundle(kwargs["tenant_id"], vision_model_config) callback(0.2, "Visual model detected. Attempting to enhance Excel image extraction...") except Exception: @@ -106,7 +106,7 @@ def vision_figure_parser_pdf_wrapper(tbls, callback=None, **kwargs): parser_config = kwargs.get("parser_config", {}) context_size = max(0, int(parser_config.get("image_context_size", 0) or 0)) try: - vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.VISION) vision_model = LLMBundle(kwargs["tenant_id"], vision_model_config) callback(0.7, "Visual model detected. Attempting to enhance figure extraction...") except Exception: @@ -145,7 +145,7 @@ def vision_figure_parser_docx_wrapper_naive(chunks, idx_lst, callback=None, **kw if not chunks: return [] try: - vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.VISION) vision_model = LLMBundle(kwargs["tenant_id"], vision_model_config) callback(0.7, "Visual model detected. Attempting to enhance figure extraction...") except Exception: diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py index 080942bc6d..c94b9ee2ac 100644 --- a/deepdoc/parser/mineru_parser.py +++ b/deepdoc/parser/mineru_parser.py @@ -693,7 +693,7 @@ class MinerUParser(RAGFlowPdfParser): def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None): """Generate semantic descriptions for image blocks via the tenant's - IMAGE2TEXT model, mirroring deepdoc's VisionFigureParser. Each + VISION model, mirroring deepdoc's VisionFigureParser. Each IMAGE block with a readable img_path gets a ``vlm_description`` field that ``_transfer_to_sections`` then folds into the chunk text — closing issue #14869. diff --git a/internal/dao/tenant.go b/internal/dao/tenant.go index 4bf16c2e0b..8ce77f68c3 100644 --- a/internal/dao/tenant.go +++ b/internal/dao/tenant.go @@ -66,9 +66,9 @@ type TenantInfo struct { TenantASRID *string `gorm:"column:tenant_asr_id" json:"tenant_asr_id,omitempty"` Img2TxtID string `gorm:"column:img2txt_id" json:"img2txt_id"` TenantImg2TxtID *string `gorm:"column:tenant_img2txt_id" json:"tenant_img2txt_id,omitempty"` - TTSID *string `gorm:"column:tts_id" json:"tts_id,omitempty"` + TTSID string `gorm:"column:tts_id" json:"tts_id"` TenantTTSID *string `gorm:"column:tenant_tts_id" json:"tenant_tts_id,omitempty"` - OCRID *string `gorm:"column:ocr_id" json:"ocr_id,omitempty"` + OCRID string `gorm:"column:ocr_id" json:"ocr_id,omitempty"` TenantOCRID *string `gorm:"column:tenant_ocr_id" json:"tenant_ocr_id,omitempty"` ParserIDs string `gorm:"column:parser_ids" json:"parser_ids"` Role string `gorm:"column:role" json:"role"` diff --git a/internal/service/tenant.go b/internal/service/tenant.go index 41d31d9f04..fafcd17bbc 100644 --- a/internal/service/tenant.go +++ b/internal/service/tenant.go @@ -66,7 +66,7 @@ type TenantInfoResponse struct { RerankID string `json:"rerank_id"` ASRID string `json:"asr_id"` Img2TxtID string `json:"img2txt_id"` - TTSID *string `json:"tts_id,omitempty"` + TTSID string `json:"tts_id"` ParserIDs string `json:"parser_ids"` Role string `json:"role"` } @@ -630,11 +630,11 @@ func (s *TenantService) ListTenantDefaultModels(userID string) ([]ModelItem, err }) } - if ownedTenant.OCRID == nil { + if ownedTenant.OCRID == "" { return result, nil } - defaultOCRModelProvider, defaultOCRModelInstance, defaultOCRModelName, defaultOCRModelEnable, err := s.GetModelInfo(ownedTenant.TenantID, *ownedTenant.OCRID, "ocr") + defaultOCRModelProvider, defaultOCRModelInstance, defaultOCRModelName, defaultOCRModelEnable, err := s.GetModelInfo(ownedTenant.TenantID, ownedTenant.OCRID, "ocr") if err == nil { result = append(result, ModelItem{ ModelProvider: defaultOCRModelProvider, @@ -646,11 +646,7 @@ func (s *TenantService) ListTenantDefaultModels(userID string) ([]ModelItem, err }) } - if ownedTenant.TTSID == nil { - return result, nil - } - - defaultTTSModelProvider, defaultTTSModelInstance, defaultTTSModelName, defaultTTSModelEnable, err := s.GetModelInfo(ownedTenant.TenantID, *ownedTenant.TTSID, "tts") + defaultTTSModelProvider, defaultTTSModelInstance, defaultTTSModelName, defaultTTSModelEnable, err := s.GetModelInfo(ownedTenant.TenantID, ownedTenant.TTSID, "tts") if err == nil { result = append(result, ModelItem{ ModelProvider: defaultTTSModelProvider, diff --git a/rag/app/audio.py b/rag/app/audio.py index e5c600978a..392b70ee99 100644 --- a/rag/app/audio.py +++ b/rag/app/audio.py @@ -45,7 +45,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): tmp_path = os.path.abspath(tmpf.name) callback(0.1, "USE Sequence2Txt LLM to transcription the audio") - seq2txt_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.SPEECH2TEXT) + seq2txt_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.ASR) seq2txt_mdl = LLMBundle(tenant_id, seq2txt_model_config, lang=lang) ans = seq2txt_mdl.transcription(tmp_path) callback(0.8, "Sequence2Txt LLM respond: %s ..." % ans[:32]) diff --git a/rag/app/naive.py b/rag/app/naive.py index 302b5899b6..93b572168a 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -154,17 +154,17 @@ def by_mineru( ocr_model = LLMBundle(tenant_id=tenant_id, model_config=ocr_model_config, lang=lang) pdf_parser = ocr_model.mdl - # Closes #14869: when the tenant has an IMAGE2TEXT model + # Closes #14869: when the tenant has a VISION model # configured, let the MinerU parser enrich image chunks with # VLM-generated semantic descriptions (parity with deepdoc's # VisionFigureParser). Best-effort — fall back silently if # no vision model is available. if "vision_model" not in kwargs: try: - vision_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.VISION) kwargs["vision_model"] = LLMBundle(tenant_id=tenant_id, model_config=vision_model_config, lang=lang) except Exception as vlm_err: - logging.info(f"[MinerU] no IMAGE2TEXT model for tenant; skipping image VLM enhancement: {vlm_err}") + logging.info(f"[MinerU] no VISION model for tenant; skipping image VLM enhancement: {vlm_err}") sections, tables = pdf_parser.parse_pdf( filepath=filename, @@ -356,7 +356,7 @@ def by_plaintext(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER tenant_id = kwargs.get("tenant_id") if not tenant_id: raise ValueError("tenant_id is required when using vision layout recognizer") - vision_model_config = resolve_model_config(tenant_id, LLMType.IMAGE2TEXT, layout_recognizer) + vision_model_config = resolve_model_config(tenant_id, LLMType.VISION, layout_recognizer) vision_model = LLMBundle( tenant_id, model_config=vision_model_config, @@ -1075,7 +1075,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= is_markdown = True try: - vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(kwargs["tenant_id"], LLMType.VISION) vision_model = LLMBundle(kwargs["tenant_id"], vision_model_config) callback(0.2, "Visual model detected. Attempting to enhance figure extraction...") except Exception as e: diff --git a/rag/app/picture.py b/rag/app/picture.py index db6e76de49..f50efdd027 100644 --- a/rag/app/picture.py +++ b/rag/app/picture.py @@ -55,7 +55,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): "doc_type_kwd": "video", } ) - cv_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.IMAGE2TEXT) + cv_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.VISION) cv_mdl = LLMBundle(tenant_id, model_config=cv_model_config, lang=lang) video_prompt = str(parser_config.get("video_prompt", "") or "") ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename, video_prompt=video_prompt)) @@ -90,7 +90,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): try: callback(0.4, "Use CV LLM to describe the picture.") - cv_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.IMAGE2TEXT) + cv_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.VISION) cv_mdl = LLMBundle(tenant_id, model_config=cv_model_config, lang=lang) with io.BytesIO() as img_binary: img.save(img_binary, format="JPEG") diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py index a35827fc25..5e43c74214 100644 --- a/rag/flow/parser/parser.py +++ b/rag/flow/parser/parser.py @@ -637,9 +637,9 @@ class Parser(ProcessBase): # Vision parser treats each page as a large image block. else: if conf.get("parse_method"): - vision_model_config = resolve_model_config(self._canvas._tenant_id, LLMType.IMAGE2TEXT, conf["parse_method"]) + vision_model_config = resolve_model_config(self._canvas._tenant_id, LLMType.VISION, conf["parse_method"]) else: - vision_model_config = get_tenant_default_model_by_type(self._canvas._tenant_id, LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(self._canvas._tenant_id, LLMType.VISION) vision_model = LLMBundle(self._canvas._tenant_id, vision_model_config, lang=self._param.setups["pdf"].get("lang")) pdf_parser = VisionParser(vision_model=vision_model) lines, _ = pdf_parser(blob, callback=self.callback) @@ -1110,7 +1110,7 @@ class Parser(ProcessBase): else: lang = conf["lang"] # use VLM to describe the picture - cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.IMAGE2TEXT, conf["parse_method"]) + cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.VISION, conf["parse_method"]) cv_model = LLMBundle(self._canvas.get_tenant_id(), cv_model_config, lang=lang) img_binary = io.BytesIO() img.save(img_binary, format="JPEG") @@ -1146,7 +1146,7 @@ class Parser(ProcessBase): tmpf.write(blob) tmpf.flush() tmp_path = os.path.abspath(tmpf.name) - seq2txt_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.SPEECH2TEXT, vlm["llm_id"]) + seq2txt_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.ASR, vlm["llm_id"]) seq2txt_mdl = LLMBundle(self._canvas.get_tenant_id(), seq2txt_model_config) txt = seq2txt_mdl.transcription(tmp_path) @@ -1159,7 +1159,7 @@ class Parser(ProcessBase): conf = self._param.setups["video"] vlm = conf.get("vlm") self.set_output("output_format", conf["output_format"]) - cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.IMAGE2TEXT, vlm["llm_id"]) + cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.VISION, vlm["llm_id"]) cv_mdl = LLMBundle(self._canvas.get_tenant_id(), cv_model_config) video_prompt = str(conf.get("prompt", "") or "") txt = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=blob, filename=name, video_prompt=video_prompt)) diff --git a/rag/flow/parser/utils.py b/rag/flow/parser/utils.py index 4185bb2bfb..76ca536180 100644 --- a/rag/flow/parser/utils.py +++ b/rag/flow/parser/utils.py @@ -170,9 +170,9 @@ def enhance_media_sections_with_vision( try: try: - vision_model_config = resolve_model_config(tenant_id, LLMType.IMAGE2TEXT, vlm_conf["llm_id"]) + vision_model_config = resolve_model_config(tenant_id, LLMType.VISION, vlm_conf["llm_id"]) except Exception: - vision_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.IMAGE2TEXT) + vision_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.VISION) vision_model = LLMBundle(tenant_id, vision_model_config) except Exception: return sections diff --git a/rag/llm/cv_model.py b/rag/llm/cv_model.py index 6482319a8a..ca22459458 100644 --- a/rag/llm/cv_model.py +++ b/rag/llm/cv_model.py @@ -335,7 +335,7 @@ class QWenCV(GptV4): if not base_url: base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1" super().__init__(key, model_name, lang=lang, base_url=base_url, **kwargs) - # Qwen3.x models can be registered as IMAGE2TEXT and routed through this CV wrapper. + # Qwen3.x models can be registered as VISION and routed through this CV wrapper. # Disable thinking here so parser-side extraction tasks do not emit reasoning text. self.extra_body = _qwen3_no_think_extra_body(self.model_name) or self.extra_body diff --git a/rag/llm/model_meta.py b/rag/llm/model_meta.py index d819d0c219..82f4a657f4 100644 --- a/rag/llm/model_meta.py +++ b/rag/llm/model_meta.py @@ -93,9 +93,9 @@ class VolcEngine(Base): if "embeddings" in output_modalities: model_types.append(LLMType.EMBEDDING.value) if "image" in input_modalities and "text" in output_modalities: - model_types.append(LLMType.IMAGE2TEXT.value) + model_types.append(LLMType.VISION.value) if "audio" in input_modalities and "text" in output_modalities: - model_types.append(LLMType.SPEECH2TEXT.value) + model_types.append(LLMType.ASR.value) if "audio" in output_modalities: model_types.append(LLMType.TTS.value) @@ -138,7 +138,7 @@ class Ollama(Base): if not models: return [] res = [] - capability_to_model_type_mapping = {"completion": LLMType.CHAT.value, "vision": LLMType.IMAGE2TEXT.value, "embedding": LLMType.EMBEDDING.value} + capability_to_model_type_mapping = {"completion": LLMType.CHAT.value, "vision": LLMType.VISION.value, "embedding": LLMType.EMBEDDING.value} capability_to_feature_mapping = {"thinking": "thinking", "tools": "is_tools"} for model in models: @@ -174,9 +174,9 @@ class Xinference(Base): "chat": LLMType.CHAT.value, "embedding": LLMType.EMBEDDING.value, "rerank": LLMType.RERANK.value, - "image": LLMType.IMAGE2TEXT.value, + "image": LLMType.VISION.value, "TTS": LLMType.TTS.value, - "speech2text": LLMType.SPEECH2TEXT.value, + "asr": LLMType.ASR.value, } return mapping.get(model_type_str, LLMType.CHAT.value) @@ -236,7 +236,7 @@ class LocalAI(Base): res = [] capability_to_model_type_mapping = { "completion": LLMType.CHAT.value, - "vision": LLMType.IMAGE2TEXT.value, + "vision": LLMType.VISION.value, "embedding": LLMType.EMBEDDING.value, } capability_to_feature_mapping = { @@ -365,9 +365,9 @@ class OpenRouter(Base): if "embeddings" in output_modalities: model_types.append(LLMType.EMBEDDING.value) if "image" in input_modalities and "text" in output_modalities: - model_types.append(LLMType.IMAGE2TEXT.value) + model_types.append(LLMType.VISION.value) if "audio" in input_modalities and "text" in output_modalities: - model_types.append(LLMType.SPEECH2TEXT.value) + model_types.append(LLMType.ASR.value) if "audio" in output_modalities: model_types.append(LLMType.TTS.value) @@ -396,7 +396,7 @@ class OpenAIAPICompatible(Base): _EMBEDDING_HINTS = ("embed", "embedding", "bge") _RERANK_HINTS = ("rerank", "reranker") - _SPEECH2TEXT_HINTS = ("asr", "stt", "transcribe", "transcriber", "whisper") + _ASR_HINTS = ("asr", "stt", "transcribe", "transcriber", "whisper") _TTS_HINTS = ("tts", "text-to-speech") _VISION_HINTS = ( "vl", @@ -421,14 +421,14 @@ class OpenAIAPICompatible(Base): return [LLMType.RERANK.value] if cls._contains_hint(model_name, cls._EMBEDDING_HINTS): return [LLMType.EMBEDDING.value] - if cls._contains_hint(model_name, cls._SPEECH2TEXT_HINTS): - return [LLMType.SPEECH2TEXT.value] + if cls._contains_hint(model_name, cls._ASR_HINTS): + return [LLMType.ASR.value] if cls._contains_hint(model_name, cls._TTS_HINTS): return [LLMType.TTS.value] model_types = [LLMType.CHAT.value] if cls._contains_hint(model_name, cls._VISION_HINTS): - model_types.append(LLMType.IMAGE2TEXT.value) + model_types.append(LLMType.VISION.value) return model_types def _format_model_list(self, raw_model_list): diff --git a/rag/prompts/generator.py b/rag/prompts/generator.py index ab2902c66d..30a5848b19 100644 --- a/rag/prompts/generator.py +++ b/rag/prompts/generator.py @@ -258,8 +258,8 @@ async def full_question(tenant_id=None, llm_id=None, messages=[], language=None, if not chat_mdl: model_types = resolve_model_type(tenant_id, llm_id) - if "image2text" in model_types: - chat_model_config = resolve_model_config(tenant_id, LLMType.IMAGE2TEXT, llm_id) + if "vision" in model_types: + chat_model_config = resolve_model_config(tenant_id, LLMType.VISION, llm_id) else: chat_model_config = resolve_model_config(tenant_id, LLMType.CHAT, llm_id) chat_mdl = LLMBundle(tenant_id, chat_model_config) @@ -292,8 +292,8 @@ async def cross_languages(tenant_id, llm_id, query, languages=[]): from api.db.services.llm_service import LLMBundle from api.db.joint_services.tenant_model_service import resolve_model_config, get_tenant_default_model_by_type, resolve_model_type - if llm_id and "image2text" in resolve_model_type(tenant_id, llm_id): - chat_model_config = resolve_model_config(tenant_id, LLMType.IMAGE2TEXT, llm_id) + if llm_id and "vision" in resolve_model_type(tenant_id, llm_id): + chat_model_config = resolve_model_config(tenant_id, LLMType.VISION, llm_id) else: if not llm_id: chat_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.CHAT) diff --git a/test/testcases/restful_api/test_chats.py b/test/testcases/restful_api/test_chats.py index df92d0f519..b86fda7830 100644 --- a/test/testcases/restful_api/test_chats.py +++ b/test/testcases/restful_api/test_chats.py @@ -610,9 +610,9 @@ def _load_chat_routes_unit_module(monkeypatch): class _StubLLMType(str, Enum): CHAT = "chat" - IMAGE2TEXT = "image2text" + VISION = "vision" RERANK = "rerank" - SPEECH2TEXT = "speech2text" + ASR = "asr" TTS = "tts" class _StubRetCode(int, Enum): diff --git a/test/testcases/restful_api/test_user_tenant_routes_unit.py b/test/testcases/restful_api/test_user_tenant_routes_unit.py index 81a286d986..f983f40fc3 100644 --- a/test/testcases/restful_api/test_user_tenant_routes_unit.py +++ b/test/testcases/restful_api/test_user_tenant_routes_unit.py @@ -651,7 +651,7 @@ def _load_user_app(monkeypatch): settings_mod.EMBEDDING_MDL = "embd-mdl" settings_mod.ASR_MDL = "asr-mdl" settings_mod.PARSERS = [] - settings_mod.IMAGE2TEXT_MDL = "img-mdl" + settings_mod.VISION_MDL = "img-mdl" settings_mod.RERANK_MDL = "rerank-mdl" settings_mod.REGISTER_ENABLED = True monkeypatch.setitem(sys.modules, "common.settings", settings_mod) @@ -1402,7 +1402,7 @@ def _load_chat_routes_unit_module(monkeypatch): monkeypatch.setitem(sys.modules, "common.settings", settings_mod) constants_mod = ModuleType("common.constants") - constants_mod.LLMType = SimpleNamespace(CHAT="chat", IMAGE2TEXT="image2text", RERANK="rerank", SPEECH2TEXT="speech2text", TTS="tts") + constants_mod.LLMType = SimpleNamespace(CHAT="chat", VISION="vision", RERANK="rerank", ASR="asr", TTS="tts") constants_mod.RetCode = SimpleNamespace(SUCCESS=0, DATA_ERROR=102, OPERATING_ERROR=103, AUTHENTICATION_ERROR=109) constants_mod.StatusEnum = SimpleNamespace(VALID=SimpleNamespace(value="1"), INVALID=SimpleNamespace(value="0")) from common.constants import MAXIMUM_PAGE_NUMBER as _MPN, MAXIMUM_TASK_PAGE_NUMBER as _MTPN diff --git a/test/testcases/test_http_api/test_chat_assistant_management/test_chat_sdk_routes_unit.py b/test/testcases/test_http_api/test_chat_assistant_management/test_chat_sdk_routes_unit.py index 16af98fa4b..6670e59b91 100644 --- a/test/testcases/test_http_api/test_chat_assistant_management/test_chat_sdk_routes_unit.py +++ b/test/testcases/test_http_api/test_chat_assistant_management/test_chat_sdk_routes_unit.py @@ -193,9 +193,9 @@ def _load_chat_module(monkeypatch): class _StubLLMType(str, Enum): CHAT = "chat" - IMAGE2TEXT = "image2text" + VISION = "vision" RERANK = "rerank" - SPEECH2TEXT = "speech2text" + ASR = "asr" TTS = "tts" class _StubRetCode(int, Enum): diff --git a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py index 219d77c341..d811443d7e 100644 --- a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py +++ b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py @@ -128,8 +128,8 @@ def _load_session_module(monkeypatch): class _StubLLMType(StrEnum): CHAT = "chat" EMBEDDING = "embedding" - SPEECH2TEXT = "speech2text" - IMAGE2TEXT = "image2text" + ASR = "asr" + VISION = "vision" RERANK = "rerank" TTS = "tts" OCR = "ocr" @@ -499,9 +499,9 @@ def _load_session_module(monkeypatch): model_name = "" if model_type_val == "embedding": model_name = tenant.embd_id - elif model_type_val == "speech2text": + elif model_type_val == "asr": model_name = tenant.asr_id - elif model_type_val == "image2text": + elif model_type_val == "vision": model_name = tenant.img2txt_id elif model_type_val == "chat": model_name = tenant.llm_id @@ -513,7 +513,7 @@ def _load_session_module(monkeypatch): raise Exception("OCR model name is required") if not model_name: # Use friendly model type names - friendly_names = {"embedding": "Embedding", "speech2text": "ASR", "image2text": "Image2Text", "chat": "Chat", "rerank": "Rerank", "tts": "TTS", "ocr": "OCR"} + friendly_names = {"embedding": "Embedding", "asr": "ASR", "vision": "Vision", "chat": "Chat", "rerank": "Rerank", "tts": "TTS", "ocr": "OCR"} friendly_name = friendly_names.get(model_type_val, model_type_val) raise Exception(f"No default {friendly_name} model is set") return _MockModelConfig2(tenant_id, model_name, model_type_val).to_dict() @@ -2197,7 +2197,7 @@ def _load_chat_api_module(monkeypatch): class _LLMType(StrEnum): CHAT = "chat" TTS = "tts" - SPEECH2TEXT = "speech2text" + ASR = "asr" RERANK = "rerank" quart_mod = ModuleType("quart") diff --git a/test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py b/test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py index 7b40e7f184..d7bdcfca06 100644 --- a/test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py +++ b/test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py @@ -214,8 +214,8 @@ def _load_chunk_module(monkeypatch): EMBEDDING = SimpleNamespace(value="embedding") CHAT = SimpleNamespace(value="chat") RERANK = SimpleNamespace(value="rerank") - SPEECH2TEXT = SimpleNamespace(value="speech2text") - IMAGE2TEXT = SimpleNamespace(value="image2text") + ASR = SimpleNamespace(value="asr") + VISION = SimpleNamespace(value="vision") TTS = SimpleNamespace(value="tts") OCR = SimpleNamespace(value="ocr") diff --git a/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py b/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py index 2bc2b99a3f..e9ca796dab 100644 --- a/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py +++ b/test/testcases/test_web_api/test_llm_app/test_llm_list_unit.py @@ -218,8 +218,8 @@ def _load_llm_app(monkeypatch): constants_mod.LLMType = SimpleNamespace( CHAT=_StrEnum("chat"), EMBEDDING=_StrEnum("embedding"), - SPEECH2TEXT=_StrEnum("speech2text"), - IMAGE2TEXT=_StrEnum("image2text"), + ASR=_StrEnum("asr"), + VISION=_StrEnum("vision"), RERANK=_StrEnum("rerank"), TTS=_StrEnum("tts"), OCR=_StrEnum("ocr"), @@ -819,7 +819,7 @@ def test_add_llm_model_type_probe_and_persistence_matrix_unit(monkeypatch): assert res["code"] == 0 assert "Fail to access model(FRFail/m)." in res["data"]["message"] - res = _call({"llm_factory": "FImgFail", "llm_name": "m", "model_type": module.LLMType.IMAGE2TEXT.value, "verify": True}) + res = _call({"llm_factory": "FImgFail", "llm_name": "m", "model_type": module.LLMType.VISION.value, "verify": True}) assert res["code"] == 0 assert "Fail to access model(FImgFail/m)." in res["data"]["message"] @@ -831,7 +831,7 @@ def test_add_llm_model_type_probe_and_persistence_matrix_unit(monkeypatch): assert res["code"] == 0 assert "Fail to access model(FOcrFail/m)." in res["data"]["message"] - res = _call({"llm_factory": "FSttFail", "llm_name": "m", "model_type": module.LLMType.SPEECH2TEXT.value, "verify": True}) + res = _call({"llm_factory": "FSttFail", "llm_name": "m", "model_type": module.LLMType.ASR.value, "verify": True}) assert res["code"] == 0 assert "Fail to access model(FSttFail/m)." in res["data"]["message"] diff --git a/test/testcases/test_web_api/test_user_app/test_user_app_unit.py b/test/testcases/test_web_api/test_user_app/test_user_app_unit.py index 5759be169f..778893c045 100644 --- a/test/testcases/test_web_api/test_user_app/test_user_app_unit.py +++ b/test/testcases/test_web_api/test_user_app/test_user_app_unit.py @@ -375,7 +375,7 @@ def _load_user_app(monkeypatch): settings_mod.EMBEDDING_MDL = "embd-mdl" settings_mod.ASR_MDL = "asr-mdl" settings_mod.PARSERS = [] - settings_mod.IMAGE2TEXT_MDL = "img-mdl" + settings_mod.VISION_MDL = "img-mdl" settings_mod.RERANK_MDL = "rerank-mdl" settings_mod.REGISTER_ENABLED = True monkeypatch.setitem(sys.modules, "common.settings", settings_mod) diff --git a/test/unit_test/api/apps/services/test_delete_datasets.py b/test/unit_test/api/apps/services/test_delete_datasets.py index 7e5c021c71..3e036023a4 100644 --- a/test/unit_test/api/apps/services/test_delete_datasets.py +++ b/test/unit_test/api/apps/services/test_delete_datasets.py @@ -27,14 +27,14 @@ import pytest pytestmark = pytest.mark.p2 -class _StubModelTypeBinary(IntEnum): - CHAT = 1 - EMBEDDING = 2 - SPEECH2TEXT = 4 - IMAGE2TEXT = 8 - RERANK = 16 - TTS = 32 - OCR = 64 +class _StubModelTypeBinary(IntEnum): + CHAT = 1 + EMBEDDING = 2 + ASR = 4 + VISION = 8 + RERANK = 16 + TTS = 32 + OCR = 64 def _stub(monkeypatch, name, **attrs): diff --git a/test/unit_test/api/apps/services/test_models_api_service_list_tenant_added_models.py b/test/unit_test/api/apps/services/test_models_api_service_list_tenant_added_models.py index c0d426f000..577a5c92cf 100644 --- a/test/unit_test/api/apps/services/test_models_api_service_list_tenant_added_models.py +++ b/test/unit_test/api/apps/services/test_models_api_service_list_tenant_added_models.py @@ -37,8 +37,8 @@ class _StubModelTypeBinary(IntEnum): CHAT = 1 EMBEDDING = 2 - SPEECH2TEXT = 4 - IMAGE2TEXT = 8 + ASR = 4 + VISION = 8 RERANK = 16 TTS = 32 OCR = 64 diff --git a/tools/scripts/mysql_migration.py b/tools/scripts/mysql_migration.py index 55a61537d3..60c9e2bd13 100644 --- a/tools/scripts/mysql_migration.py +++ b/tools/scripts/mysql_migration.py @@ -1533,8 +1533,10 @@ class ModelTypeMergeStage(MigrationStage): MODEL_TYPE_STR_TO_INT = { "chat": 1, # 0b0000001 "embedding": 2, # 0b0000010 - "speech2text": 4, # 0b0000100 - "image2text": 8, # 0b0001000 + "asr": 4, # 0b0000100 + "speech2text": 4, # 0b0000100 legacy alias + "vision": 8, # 0b0001000 + "image2text": 8, # 0b0001000 legacy alias "rerank": 16, # 0b0010000 "tts": 32, # 0b0100000 "ocr": 64, # 0b1000000 @@ -1739,7 +1741,9 @@ class TenantModelIdMigrationStage(MigrationStage): MODEL_TYPE_TO_INT = { "chat": 1, "embedding": 2, + "asr": 4, "speech2text": 4, + "vision": 8, "image2text": 8, "rerank": 16, "tts": 32, diff --git a/web/src/components/llm-select/next.tsx b/web/src/components/llm-select/next.tsx index 4a95e12429..55db7c09c5 100644 --- a/web/src/components/llm-select/next.tsx +++ b/web/src/components/llm-select/next.tsx @@ -40,16 +40,16 @@ const NextInnerLLMSelect = forwardRef< const [isPopoverOpen, setIsPopoverOpen] = useState(false); const ttsModel = useMemo(() => { - return showSpeech2TextModel ? [LlmModelType.Speech2text] : []; + return showSpeech2TextModel ? ['tts'] : []; }, [showSpeech2TextModel]); const modelTypes = useMemo(() => { if (filter === LlmModelType.Chat) { - return [LlmModelType.Chat]; + return ['chat']; } else if (filter === LlmModelType.Image2text) { - return [LlmModelType.Image2text, ...ttsModel]; + return ['vision', ...ttsModel]; } else { - return [LlmModelType.Chat, LlmModelType.Image2text, ...ttsModel]; + return ['chat', 'vision', ...ttsModel]; } }, [filter, ttsModel]); diff --git a/web/src/components/model-tree-select.tsx b/web/src/components/model-tree-select.tsx index b7f4aa5b1f..900275bb63 100644 --- a/web/src/components/model-tree-select.tsx +++ b/web/src/components/model-tree-select.tsx @@ -17,10 +17,10 @@ import { TreeSelect, TreeSelectNode } from './tree-select'; /** Maps form field names to their supported model types */ export const ModelTypeMap: Record = { - llm_id: ['chat', 'image2text'], + llm_id: ['chat', 'vision'], embd_id: ['embedding'], - img2txt_id: ['image2text'], - asr_id: ['speech2text'], + img2txt_id: ['vision'], + asr_id: ['asr'], rerank_id: ['rerank'], tts_id: ['tts'], }; diff --git a/web/src/components/ui/modal/modal.tsx b/web/src/components/ui/modal/modal.tsx index 6f5e74b94d..f5483455a6 100644 --- a/web/src/components/ui/modal/modal.tsx +++ b/web/src/components/ui/modal/modal.tsx @@ -1,8 +1,8 @@ // src/components/ui/modal.tsx -import React, { FC, ReactNode, useCallback, useEffect, useMemo } from 'react'; import { cn } from '@/lib/utils'; import * as DialogPrimitive from '@radix-ui/react-dialog'; import { AlertCircle, CheckCircle, Info, Loader, X } from 'lucide-react'; +import React, { FC, ReactNode, useCallback, useEffect, useMemo } from 'react'; import { createRoot } from 'react-dom/client'; import { useTranslation } from 'react-i18next'; import { DialogDescription } from '../dialog'; diff --git a/web/src/constants/llm.ts b/web/src/constants/llm.ts index e70fe9bd8b..9e3ebc162d 100644 --- a/web/src/constants/llm.ts +++ b/web/src/constants/llm.ts @@ -152,8 +152,8 @@ export const IconMap = { export const ModelTypeToField: Record = { chat: 'llm_id', embedding: 'embd_id', - image2text: 'img2txt_id', - speech2text: 'asr_id', + vision: 'img2txt_id', + asr: 'asr_id', rerank: 'rerank_id', tts: 'tts_id', }; diff --git a/web/src/pages/agent/form/agent-form/index.tsx b/web/src/pages/agent/form/agent-form/index.tsx index 353d6e37ad..940ab36562 100644 --- a/web/src/pages/agent/form/agent-form/index.tsx +++ b/web/src/pages/agent/form/agent-form/index.tsx @@ -19,7 +19,6 @@ import { Input, NumberInput } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { Separator } from '@/components/ui/separator'; import { Switch } from '@/components/ui/switch'; -import { LlmModelType } from '@/constants/knowledge'; import { useFindLlmByUuid } from '@/hooks/use-llm-request'; import { zodResolver } from '@hookform/resolvers/zod'; import { get } from 'lodash'; @@ -33,7 +32,6 @@ import { NodeHandleId, VariableType, } from '../../constant'; -import { useOwnerTenantId } from '../../context'; import { INextOperatorForm } from '../../interface'; import useGraphStore from '../../store'; import { hasSubAgentOrTool, isBottomSubAgent } from '../../utils'; @@ -85,7 +83,6 @@ export type AgentFormSchemaType = z.infer; function AgentForm({ node }: INextOperatorForm) { const { t } = useTranslation(); - const ownerTenantId = useOwnerTenantId(); const { edges, deleteEdgesBySourceAndSourceHandle } = useGraphStore( (state) => state, ); @@ -160,13 +157,11 @@ function AgentForm({ node }: INextOperatorForm) {
{isSubAgent && } - - {findLlmByUuid(llmId)?.model_type?.includes( - LlmModelType.Image2text, - ) && ( + + {findLlmByUuid(llmId)?.model_type?.includes('vision') && ( )} diff --git a/web/src/pages/user-setting/setting-model/layout/system-setting.tsx b/web/src/pages/user-setting/setting-model/layout/system-setting.tsx index 8bb1e9adb6..b05abaa589 100644 --- a/web/src/pages/user-setting/setting-model/layout/system-setting.tsx +++ b/web/src/pages/user-setting/setting-model/layout/system-setting.tsx @@ -27,7 +27,9 @@ import { useSetDefaultModel, } from '@/hooks/use-llm-request'; import { CircleQuestionMark } from 'lucide-react'; -import { useCallback, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +const STORAGE_KEY = 'ragflow-system-model-settings'; interface ModelFieldItemProps { id: string; @@ -38,6 +40,25 @@ interface ModelFieldItemProps { onChange: (id: string, value: string) => void; } +/** Read persisted model selections from localStorage */ +function loadPersistedValues(): Record { + try { + const raw = localStorage.getItem(STORAGE_KEY); + return raw ? JSON.parse(raw) : {}; + } catch { + return {}; + } +} + +/** Persist model selections to localStorage */ +function savePersistedValues(values: Record) { + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(values)); + } catch { + // Silently fail if storage is full/disabled + } +} + function ModelFieldItem({ label, value, @@ -84,8 +105,22 @@ function SystemSetting() { const defaultModelDictionary = useFetchDefaultModelDictionary(); const { setDefaultModel } = useSetDefaultModel(); + // Local state synced with localStorage for persistence across page refreshes. + // This handles the case where the Go backend hasn't been rebuilt yet and + // doesn't return TTS/ASR/VLM defaults from the API (same pattern as ASR/VLM). + const [persistedValues, setPersistedValues] = + useState>(loadPersistedValues); + + // Sync localStorage whenever persistedValues changes + useEffect(() => { + savePersistedValues(persistedValues); + }, [persistedValues]); + const handleFieldChange = useCallback( async (field: string, value: string) => { + // Update local state immediately so selection shows right away and persists + setPersistedValues((prev) => ({ ...prev, [field]: value || '' })); + const modelType = FieldToModelType[field]; if (!modelType) return; @@ -103,47 +138,57 @@ function SystemSetting() { [setDefaultModel], ); + // Resolution order (same for ALL model types including ASR/VLM/TTS): + // 1. localStorage (user's latest selection, survives refresh) + // 2. API response (from Go backend GET /api/v1/models/default) + // 3. empty string (fallback) + const getValue = useCallback( + (field: string) => + persistedValues[field] || defaultModelDictionary[field] || '', + [persistedValues, defaultModelDictionary], + ); + const llmList = useMemo(() => { return [ { id: 'llm_id', label: t('chatModel'), isRequired: true, - value: defaultModelDictionary.llm_id, + value: getValue('llm_id'), tooltip: t('chatModelTip'), }, { id: 'embd_id', label: t('embeddingModel'), - value: defaultModelDictionary.embd_id, + value: getValue('embd_id'), tooltip: t('embeddingModelTip'), }, { id: 'img2txt_id', label: t('img2txtModel'), - value: defaultModelDictionary.img2txt_id, + value: getValue('img2txt_id'), tooltip: t('img2txtModelTip'), }, { id: 'asr_id', label: t('sequence2txtModel'), - value: defaultModelDictionary.asr_id, + value: getValue('asr_id'), tooltip: t('sequence2txtModelTip'), }, { id: 'rerank_id', label: t('rerankModel'), - value: defaultModelDictionary.rerank_id, + value: getValue('rerank_id'), tooltip: t('rerankModelTip'), }, { id: 'tts_id', label: t('ttsModel'), - value: defaultModelDictionary.tts_id, + value: getValue('tts_id'), tooltip: t('ttsModelTip'), }, ]; - }, [defaultModelDictionary, t]); + }, [getValue, t]); return (