mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-06 03:18:36 +08:00
Fix: IMAGE2TEXT→CHAT fallback with model_type normalization in tenant_model_service (#14704)
## Summary - When a model is registered as `chat` in `tenant_llm` but has the `IMAGE2TEXT` tag in `llm_factories.json`, requesting it as `image2text` (e.g. PDF parser) fails with `Tenant Model with name <model> and type image2text not found`. - After resolution via the new fallback, the returned `config_dict["model_type"]` was still `"chat"`, causing `tenant_llm_service.model_instance()` to instantiate `ChatModel` instead of `CvModel` — breaking `describe_with_prompt` at ingestion time. ## What problem does this PR solve? RAGFlow already has a `CHAT→IMAGE2TEXT` fallback: when a chat model is not found, it retries with `image2text`. The symmetric fallback (`IMAGE2TEXT→CHAT`) was missing. This matters for multimodal models declared as `model_type: "chat"` with an `IMAGE2TEXT` tag in `llm_factories.json` (e.g. models added after tenant creation, or providers where a single model serves both purposes). The frontend PDF parser selector correctly surfaces these models via the `IMAGE2TEXT` tag, but the backend fails to resolve them at runtime. ## Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ## Changes **`api/db/joint_services/tenant_model_service.py`** 1. Add `IMAGE2TEXT→CHAT` fallback in `get_model_config_by_type_and_name`: when an `image2text` model is not found in `tenant_llm`, retry with `chat` — but only if the `llm` table confirms `IMAGE2TEXT` capability via the `tags` field. This mirrors the philosophy of the existing `CHAT→IMAGE2TEXT` fallback: substitution is only allowed when the model has declared the required capability. 2. Normalize `config_dict["model_type"]` to `image2text` after the fallback, so the caller (`model_instance`) correctly routes to `CvModel` instead of `ChatModel`. 3. Extend the type validation guard to allow `(requested=image2text, found=chat)` alongside the existing `(requested=chat, found=image2text)` exception. ## Test plan - [ ] Add a model with `model_type=chat` and `tags` containing `IMAGE2TEXT` to a tenant - [ ] Select it as PDF parser in a knowledge base - [ ] Verify ingestion succeeds without `image2text not found` or `describe_with_prompt` errors - [ ] Verify the same model still works correctly in chat context 🤖 Generated with [Claude Code](https://claude.ai/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import logging
|
||||
import os
|
||||
import enum
|
||||
from common import settings
|
||||
@@ -20,6 +21,8 @@ from common.constants import LLMType
|
||||
from api.db.services.llm_service import LLMService
|
||||
from api.db.services.tenant_llm_service import TenantLLMService, TenantService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_model_config_by_id(tenant_model_id: int) -> dict:
|
||||
found, model_config = TenantLLMService.get_by_id(tenant_model_id)
|
||||
@@ -71,6 +74,23 @@ def get_model_config_by_type_and_name(tenant_id: str, model_type: str, model_nam
|
||||
if not model_config:
|
||||
raise LookupError(f"Tenant Model with name {model_name} and type {model_type_val} not found")
|
||||
config_dict = model_config.to_dict()
|
||||
elif model_type_val == LLMType.IMAGE2TEXT.value:
|
||||
model_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, LLMType.IMAGE2TEXT.value)
|
||||
if not model_config:
|
||||
# Fall back to a chat model only if it has declared IMAGE2TEXT capability (tag check via llm table)
|
||||
chat_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, LLMType.CHAT.value)
|
||||
logger.debug("IMAGE2TEXT config not found for %s; chat_config found: %s", pure_model_name, chat_config is not None)
|
||||
if chat_config:
|
||||
llm_entry = LLMService.query(fid=chat_config.llm_factory, llm_name=chat_config.llm_name)
|
||||
tags = [t.strip() for t in (llm_entry[0].tags or "").split(",")] if llm_entry else []
|
||||
logger.debug("LLM tags for %s/%s: %s", chat_config.llm_factory, chat_config.llm_name, tags)
|
||||
if "IMAGE2TEXT" in tags:
|
||||
logger.debug("Promoting chat config to IMAGE2TEXT for %s", pure_model_name)
|
||||
model_config = chat_config
|
||||
if not model_config:
|
||||
raise LookupError(f"Tenant Model with name {model_name} and type {model_type_val} not found")
|
||||
config_dict = model_config.to_dict()
|
||||
config_dict["model_type"] = LLMType.IMAGE2TEXT.value
|
||||
else:
|
||||
model_config = TenantLLMService.get_api_key(tenant_id, pure_model_name, model_type_val)
|
||||
if not model_config:
|
||||
@@ -90,6 +110,9 @@ def get_model_config_by_type_and_name(tenant_id: str, model_type: str, model_nam
|
||||
if config_model_type != model_type_val and not (
|
||||
model_type_val == LLMType.CHAT.value
|
||||
and config_model_type == LLMType.IMAGE2TEXT.value
|
||||
) and not (
|
||||
model_type_val == LLMType.IMAGE2TEXT.value
|
||||
and config_model_type == LLMType.CHAT.value
|
||||
):
|
||||
raise LookupError(
|
||||
f"Tenant Model with name {model_name} has type {config_model_type}, expected {model_type_val}"
|
||||
|
||||
Reference in New Issue
Block a user