mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-06-29 15:31:05 +08:00
Fix memory resolution regression for multimodal Gemini models (#14209)
### What problem does this PR solve? Fixes #14206. This issue is a regression. PR #9520 previously changed Gemini models from `image2text` to `chat` to fix chat-side resolution, but PR #13073 later restored those Gemini entries to `image2text` during model-list updates, which reintroduced the bug. The underlying problem is that Gemini models are multimodal and advertise both `CHAT` and `IMAGE2TEXT`, while tenant model resolution still depends on a single stored `model_type`. That makes chat-only flows such as memory extraction fragile when a compatible model is stored as `image2text`. This PR fixes the issue at the model resolution layer instead of changing `llm_factories.json` again: - keep the stored tenant model type unchanged - try exact `model_type` lookup first - if no exact match is found, fall back only when the model metadata shows the requested capability is supported - coerce the runtime config to the requested type for chat callers - fail fast in memory creation instead of silently persisting `tenant_llm_id=0` This preserves existing multimodal and `image2text` behavior while restoring chat compatibility for memory-related flows. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ### Testing - Re-checked the current memory creation and memory message extraction paths against the updated resolution logic - Verified locally that a Gemini-style tenant model stored as `image2text` but tagged with `CHAT` can still be resolved for `chat` - Verified `get_model_config_by_type_and_name(..., CHAT, ...)` returns a chat-compatible runtime config - Verified `get_model_config_by_id(..., CHAT)` also returns a chat-compatible runtime config - Verified strict resolution still fails when the model metadata does not advertise chat capability
This commit is contained in:
@@ -14,6 +14,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
from common.constants import LLMType
|
||||
from common.exceptions import ArgumentException
|
||||
from api.db.services.tenant_llm_service import TenantLLMService
|
||||
|
||||
_KEY_TO_MODEL_TYPE = {
|
||||
@@ -25,13 +26,20 @@ _KEY_TO_MODEL_TYPE = {
|
||||
"tts_id": LLMType.TTS,
|
||||
}
|
||||
|
||||
def ensure_tenant_model_id_for_params(tenant_id: str, param_dict: dict) -> dict:
|
||||
def ensure_tenant_model_id_for_params(tenant_id: str, param_dict: dict, *, strict: bool = False) -> dict:
|
||||
for key in ["llm_id", "embd_id", "asr_id", "img2txt_id", "rerank_id", "tts_id"]:
|
||||
if param_dict.get(key) and not param_dict.get(f"tenant_{key}"):
|
||||
model_type = _KEY_TO_MODEL_TYPE.get(key)
|
||||
tenant_model = TenantLLMService.get_api_key(tenant_id, param_dict[key], model_type)
|
||||
if not tenant_model and model_type == LLMType.CHAT:
|
||||
tenant_model = TenantLLMService.get_api_key(tenant_id, param_dict[key])
|
||||
if tenant_model:
|
||||
param_dict.update({f"tenant_{key}": tenant_model.id})
|
||||
else:
|
||||
if strict:
|
||||
model_type_val = model_type.value if hasattr(model_type, "value") else model_type
|
||||
raise ArgumentException(
|
||||
f"Tenant Model with name {param_dict[key]} and type {model_type_val} not found"
|
||||
)
|
||||
param_dict.update({f"tenant_{key}": 0})
|
||||
return param_dict
|
||||
|
||||
Reference in New Issue
Block a user