Fix: model list (#15860)

### What problem does this PR solve?

Remove tenant_llm call in rag.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

api/apps/restful_apis/chunk_api.py

@@ -25,6 +25,7 @@ from quart import request
 
 from api.apps import login_required
 from api.db.joint_services.tenant_model_service import (
+    split_model_name,
     get_model_config_from_provider_instance,
     get_tenant_default_model_by_type,
 )
@@ -266,7 +267,7 @@ async def retrieval_test(tenant_id):
         if not KnowledgebaseService.accessible(kb_id=id, user_id=tenant_id):
             return get_error_data_result(f"You don't own the dataset {id}.")
     kbs = KnowledgebaseService.get_by_ids(kb_ids)
-    embd_nms = list(set([TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]))
+    embd_nms = list(set([split_model_name(kb.embd_id)[0] for kb in kbs]))
     if len(embd_nms) != 1:
         return get_result(message="Datasets use different embedding models.", code=RetCode.DATA_ERROR)
     if "question" not in req:

api/apps/restful_apis/user_api.py

@@ -27,10 +27,7 @@ from werkzeug.security import check_password_hash, generate_password_hash
 
 from api.apps.auth import get_auth_client
 from api.db import FileType, UserTenantRole
-from api.db.db_models import TenantLLM
 from api.db.services.file_service import FileService
-from api.db.services.llm_service import get_init_tenant_llm
-from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
 from common.time_utils import current_timestamp, datetime_format, get_format_time
 from common.misc_utils import download_img, get_uuid
@@ -412,10 +409,6 @@ def rollback_user_registration(user_id):
             UserTenantService.delete_by_id(u[0].id)
     except Exception:
         pass
-    try:
-        TenantLLM.delete().where(TenantLLM.tenant_id == user_id).execute()
-    except Exception:
-        pass
 
 
 def user_register(user_id, user):
@@ -448,13 +441,13 @@ def user_register(user_id, user):
         "location": "",
     }
 
-    tenant_llm = get_init_tenant_llm(user_id)
+    # tenant_llm = get_init_tenant_llm(user_id)
 
     if not UserService.save(**user):
         return None
     TenantService.insert(**tenant)
     UserTenantService.insert(**usr_tenant)
-    TenantLLMService.insert_many(tenant_llm)
+    # TenantLLMService.insert_many(tenant_llm)
     FileService.insert(file)
     return UserService.query(email=user["email"])
 

api/apps/services/dataset_api_service.py

@@ -29,7 +29,6 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.connector_service import Connector2KbService
 from api.db.services.task_service import GRAPH_RAPTOR_FAKE_DOC_ID, TaskService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
-from api.db.services.tenant_llm_service import TenantLLMService
 from common.constants import FileSource, StatusEnum
 from api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_keys, verify_embedding_availability
 
@@ -1276,7 +1275,7 @@ async def search_datasets(tenant_id: str, req: dict):
     :param req: search request containing dataset_ids and other params
     :return: (success, result) or (success, error_message)
     """
-    from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type
+    from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, split_model_name
     from api.db.services.doc_metadata_service import DocMetadataService
     from api.db.services.llm_service import LLMBundle
     from api.db.services.search_service import SearchService
@@ -1315,7 +1314,7 @@ async def search_datasets(tenant_id: str, req: dict):
         return False, "Datasets not found!"
 
     # All datasets must use the same embedding model
-    embd_nms = list(set([TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]))
+    embd_nms = list(set([split_model_name(kb.embd_id)[0] for kb in kbs]))
     if len(embd_nms) != 1:
         return False, "Datasets use different embedding models."
 

api/apps/services/models_api_service.py

@@ -16,7 +16,7 @@
 import os
 import logging
 
-from api.db.joint_services.tenant_model_service import ensure_mineru_from_env, ensure_paddleocr_from_env
+from api.db.joint_services.tenant_model_service import ensure_mineru_from_env, ensure_paddleocr_from_env, ensure_opendataloader_from_env
 from common.constants import ActiveStatusEnum, LLMType
 from common.settings import FACTORY_LLM_INFOS
 from api.db.services.tenant_model_provider_service import TenantModelProviderService
@@ -304,6 +304,7 @@ def list_tenant_added_models(tenant_id: str, model_type_filter: str=None):
 
     ensure_mineru_from_env(tenant_id)
     ensure_paddleocr_from_env(tenant_id)
+    ensure_opendataloader_from_env(tenant_id)
 
     if model_type_filter:
         model_type_filter = model_type_filter.lower()

api/db/init_data.py

@@ -107,6 +107,7 @@ def init_superuser(nickname=DEFAULT_SUPERUSER_NICKNAME, email=DEFAULT_SUPERUSER_
 
 
 def init_llm_factory():
+    # todo deprecated
     LLMFactoriesService.filter_delete([1 == 1])
     factory_llm_infos = settings.FACTORY_LLM_INFOS
     for factory_llm_info in factory_llm_infos:

api/db/joint_services/tenant_model_service.py

@@ -18,7 +18,7 @@ import os
 import enum
 import json
 from common import settings
-from common.constants import ActiveStatusEnum, LLMType, MINERU_DEFAULT_CONFIG, MINERU_ENV_KEYS, PADDLEOCR_DEFAULT_CONFIG, PADDLEOCR_ENV_KEYS
+from common.constants import ActiveStatusEnum, LLMType, MINERU_DEFAULT_CONFIG, MINERU_ENV_KEYS, OPENDATALOADER_DEFAULT_CONFIG, OPENDATALOADER_ENV_KEYS, PADDLEOCR_DEFAULT_CONFIG, PADDLEOCR_ENV_KEYS
 from api.db.services.tenant_llm_service import TenantService
 from api.db.services.tenant_model_provider_service import TenantModelProviderService
 from api.db.services.tenant_model_instance_service import TenantModelInstanceService
@@ -294,3 +294,31 @@ def delete_models_by_instance_ids(instance_ids: list[str]):
 
 def delete_instances_by_provider_ids(provider_ids: list[str]):
     return TenantModelInstanceService.delete_by_provider_ids(provider_ids)
+
+
+def ensure_opendataloader_from_env(tenant_id: str) -> str | None:
+    return _ensure_ocr_provider_from_env(
+        tenant_id,
+        "OpenDataLoader",
+        "opendataloader-from-env",
+        _collect_env_config(OPENDATALOADER_ENV_KEYS, OPENDATALOADER_DEFAULT_CONFIG),
+    )
+
+
+def get_models_by_tenant_and_provider_and_model_type(tenant_id: str, provider_name: str, model_type: str):
+    """
+    Query TenantModel records by tenant_id, provider_name and model_name.
+    Returns all matching model records under all instances of the specified provider.
+    """
+    provider_obj = TenantModelProviderService.get_by_tenant_id_and_provider_name(tenant_id, provider_name)
+    if not provider_obj:
+        return []
+    instances = TenantModelInstanceService.get_all_by_provider_id(provider_obj.id)
+    if not instances:
+        return []
+    results = []
+    for inst in instances:
+        models = TenantModelService.get_by_provider_id_and_instance_id_and_model_type(provider_obj.id, inst.id, model_type)
+        if models:
+            results.extend(models)
+    return results

api/db/joint_services/user_account_service.py

@@ -27,12 +27,10 @@ from api.db.services.doc_metadata_service import DocMetadataService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.langfuse_service import TenantLangfuseService
-from api.db.services.llm_service import get_init_tenant_llm
 from api.db.services.file_service import FileService
 from api.db.services.mcp_server_service import MCPServerService
 from api.db.services.search_service import SearchService
 from api.db.services.task_service import TaskService
-from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
 from api.db.services.memory_service import MemoryService
@@ -90,14 +88,14 @@ def create_new_user(user_info: dict) -> dict:
         "location": "",
     }
     try:
-        tenant_llm = get_init_tenant_llm(user_id)
+        # tenant_llm = get_init_tenant_llm(user_id)
 
         if not UserService.save(**user_info):
             return {"success": False}
 
         TenantService.insert(**tenant)
         UserTenantService.insert(**usr_tenant)
-        TenantLLMService.insert_many(tenant_llm)
+        # TenantLLMService.insert_many(tenant_llm)
         FileService.insert(file)
 
         return {
@@ -123,10 +121,6 @@ def create_new_user(user_info: dict) -> dict:
                 UserTenantService.delete_by_id(u[0].id)
         except Exception as e:
             logging.exception(e)
-        try:
-            TenantLLMService.delete_by_tenant_id(user_id)
-        except Exception as e:
-            logging.exception(e)
         try:
             FileService.delete_by_id(file["id"])
         except Exception as e:
@@ -209,9 +203,9 @@ def delete_user_data(user_id: str) -> dict:
                 # step1.1.7 delete search
                 search_delete_res = SearchService.delete_by_tenant_id(usr.id)
                 done_msg += f"- Deleted {search_delete_res} search records.\n"
-            # step1.2 delete tenant_llm and tenant_langfuse
-            llm_delete_res = TenantLLMService.delete_by_tenant_id(tenant_id)
-            done_msg += f"- Deleted {llm_delete_res} tenant-LLM records.\n"
+            # step1.2 delete tenant_langfuse
+            # llm_delete_res = TenantLLMService.delete_by_tenant_id(tenant_id)
+            # done_msg += f"- Deleted {llm_delete_res} tenant-LLM records.\n"
             langfuse_delete_res = TenantLangfuseService.delete_ty_tenant_id(tenant_id)
             done_msg += f"- Deleted {langfuse_delete_res} langfuse records.\n"
             try:

api/db/services/tenant_model_service.py

@@ -35,6 +35,15 @@ class TenantModelService(CommonService):
             cls.model.model_name == model_name
         )
 
+    @classmethod
+    @DB.connection_context()
+    def get_by_provider_id_and_instance_id_and_model_type(cls, provider_id, instance_id, model_type):
+        return cls.model.get_or_none(
+            cls.model.provider_id == provider_id,
+            cls.model.instance_id == instance_id,
+            cls.model.model_type == model_type
+        )
+
     @classmethod
     @DB.connection_context()
     def get_models_by_instance_id(cls, instance_id):