From c44dc85143fded1b543d006b2ae1887466924bdd Mon Sep 17 00:00:00 2001 From: VincentLambert Date: Sat, 9 May 2026 04:40:58 +0200 Subject: [PATCH] =?UTF-8?q?Fix:=20IMAGE2TEXT=E2=86=92CHAT=20fallback=20wit?= =?UTF-8?q?h=20model=5Ftype=20normalization=20in=20tenant=5Fmodel=5Fservic?= =?UTF-8?q?e=20(#14704)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - When a model is registered as `chat` in `tenant_llm` but has the `IMAGE2TEXT` tag in `llm_factories.json`, requesting it as `image2text` (e.g. PDF parser) fails with `Tenant Model with name and type image2text not found`. - After resolution via the new fallback, the returned `config_dict["model_type"]` was still `"chat"`, causing `tenant_llm_service.model_instance()` to instantiate `ChatModel` instead of `CvModel` — breaking `describe_with_prompt` at ingestion time. ## What problem does this PR solve? RAGFlow already has a `CHAT→IMAGE2TEXT` fallback: when a chat model is not found, it retries with `image2text`. The symmetric fallback (`IMAGE2TEXT→CHAT`) was missing. This matters for multimodal models declared as `model_type: "chat"` with an `IMAGE2TEXT` tag in `llm_factories.json` (e.g. models added after tenant creation, or providers where a single model serves both purposes). The frontend PDF parser selector correctly surfaces these models via the `IMAGE2TEXT` tag, but the backend fails to resolve them at runtime. ## Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ## Changes **`api/db/joint_services/tenant_model_service.py`** 1. Add `IMAGE2TEXT→CHAT` fallback in `get_model_config_by_type_and_name`: when an `image2text` model is not found in `tenant_llm`, retry with `chat` — but only if the `llm` table confirms `IMAGE2TEXT` capability via the `tags` field. This mirrors the philosophy of the existing `CHAT→IMAGE2TEXT` fallback: substitution is only allowed when the model has declared the required capability. 2. Normalize `config_dict["model_type"]` to `image2text` after the fallback, so the caller (`model_instance`) correctly routes to `CvModel` instead of `ChatModel`. 3. Extend the type validation guard to allow `(requested=image2text, found=chat)` alongside the existing `(requested=chat, found=image2text)` exception. ## Test plan - [ ] Add a model with `model_type=chat` and `tags` containing `IMAGE2TEXT` to a tenant - [ ] Select it as PDF parser in a knowledge base - [ ] Verify ingestion succeeds without `image2text not found` or `describe_with_prompt` errors - [ ] Verify the same model still works correctly in chat context 🤖 Generated with [Claude Code](https://claude.ai/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 --- api/db/joint_services/tenant_model_service.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/api/db/joint_services/tenant_model_service.py b/api/db/joint_services/tenant_model_service.py index 9f9487286c..645d756381 100644 --- a/api/db/joint_services/tenant_model_service.py +++ b/api/db/joint_services/tenant_model_service.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import logging import os import enum from common import settings @@ -20,6 +21,8 @@ from common.constants import LLMType from api.db.services.llm_service import LLMService from api.db.services.tenant_llm_service import TenantLLMService, TenantService +logger = logging.getLogger(__name__) + def get_model_config_by_id(tenant_model_id: int) -> dict: found, model_config = TenantLLMService.get_by_id(tenant_model_id) @@ -71,6 +74,23 @@ def get_model_config_by_type_and_name(tenant_id: str, model_type: str, model_nam if not model_config: raise LookupError(f"Tenant Model with name {model_name} and type {model_type_val} not found") config_dict = model_config.to_dict() + elif model_type_val == LLMType.IMAGE2TEXT.value: + model_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, LLMType.IMAGE2TEXT.value) + if not model_config: + # Fall back to a chat model only if it has declared IMAGE2TEXT capability (tag check via llm table) + chat_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, LLMType.CHAT.value) + logger.debug("IMAGE2TEXT config not found for %s; chat_config found: %s", pure_model_name, chat_config is not None) + if chat_config: + llm_entry = LLMService.query(fid=chat_config.llm_factory, llm_name=chat_config.llm_name) + tags = [t.strip() for t in (llm_entry[0].tags or "").split(",")] if llm_entry else [] + logger.debug("LLM tags for %s/%s: %s", chat_config.llm_factory, chat_config.llm_name, tags) + if "IMAGE2TEXT" in tags: + logger.debug("Promoting chat config to IMAGE2TEXT for %s", pure_model_name) + model_config = chat_config + if not model_config: + raise LookupError(f"Tenant Model with name {model_name} and type {model_type_val} not found") + config_dict = model_config.to_dict() + config_dict["model_type"] = LLMType.IMAGE2TEXT.value else: model_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, model_type_val) if not model_config: @@ -90,6 +110,9 @@ def get_model_config_by_type_and_name(tenant_id: str, model_type: str, model_nam if config_model_type != model_type_val and not ( model_type_val == LLMType.CHAT.value and config_model_type == LLMType.IMAGE2TEXT.value + ) and not ( + model_type_val == LLMType.IMAGE2TEXT.value + and config_model_type == LLMType.CHAT.value ): raise LookupError( f"Tenant Model with name {model_name} has type {config_model_type}, expected {model_type_val}"