Fix: handle llm_setting switch (#17745)

api/apps/restful_apis/_generation_params.py

@@ -18,6 +18,48 @@ from copy import deepcopy
 
 GENERATION_CONFIG_KEYS = ("temperature", "top_p", "frequency_penalty", "presence_penalty", "max_tokens")
 
+# Default values for the four LLM generation parameters stored in
+# search_config.llm_setting.  When the corresponding ``_enabled`` flag is
+# ``False`` (or the key is absent), the default is used instead of whatever
+# the user may have stored in ``llm_setting``.
+LLM_SETTING_DEFAULTS = {
+    "temperature": 0.1,
+    "top_p": 0.3,
+    "frequency_penalty": 0.7,
+    "presence_penalty": 0.4,
+}
+
+
+def resolve_llm_setting(llm_setting):
+    """Resolve *llm_setting* values according to their enable flags.
+
+    For each of the four generation parameters the dictionary may carry a
+    ``{key}_enabled`` boolean.  When the flag is ``True`` and the value
+    exists in *llm_setting*, the user-configured value is kept; otherwise
+    the default from :data:`LLM_SETTING_DEFAULTS` is substituted.
+
+    Keys whose name ends with ``_enabled`` are stripped from the result so
+    they are never forwarded to the downstream LLM call.
+    """
+    if not llm_setting:
+        return dict(LLM_SETTING_DEFAULTS)
+
+    resolved = {}
+    for key, default_val in LLM_SETTING_DEFAULTS.items():
+        enabled_key = f"{key}_enabled"
+        if llm_setting.get(enabled_key, True) and key in llm_setting:
+            resolved[key] = llm_setting[key]
+        else:
+            resolved[key] = default_val
+
+    # Carry over any extra keys that are not generation parameters and not
+    # enable flags (e.g. ``llm_id``, ``model_type``).
+    for key, val in llm_setting.items():
+        if key not in resolved and not key.endswith("_enabled"):
+            resolved[key] = val
+
+    return resolved
+
 
 def extract_generation_config(req):
     return {key: req[key] for key in GENERATION_CONFIG_KEYS if key in req and req[key] is not None}

api/apps/restful_apis/bot_api.py

@@ -36,6 +36,7 @@ from common.metadata_utils import apply_meta_data_filter
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
 from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, resolve_model_config
+from api.apps.restful_apis._generation_params import resolve_llm_setting
 from common.misc_utils import thread_pool_exec
 from api.utils.api_utils import get_error_data_result, get_json_result, add_tenant_id_to_kwargs, get_result, get_request_json, server_error_response, validate_request
 from rag.app.tag import label_question
@@ -493,7 +494,7 @@ async def related_questions_embedded(tenant_id=None):
         chat_model_config = await thread_pool_exec(get_tenant_default_model_by_type, tenant_id, LLMType.CHAT)
     chat_mdl = LLMBundle(tenant_id, chat_model_config)
 
-    gen_conf = search_config.get("llm_setting", {"temperature": 0.9})
+    gen_conf = resolve_llm_setting(search_config.get("llm_setting"))
     prompt = load_prompt("related_question")
     ans = await chat_mdl.async_chat(
         prompt,

api/apps/restful_apis/chat_api.py

@@ -27,7 +27,7 @@ from quart import Response, request
 from werkzeug.exceptions import BadRequest
 
 from api.apps import current_user, login_required
-from api.apps.restful_apis._generation_params import merge_generation_config, pop_generation_config
+from api.apps.restful_apis._generation_params import merge_generation_config, pop_generation_config, resolve_llm_setting
 from api.db.joint_services.tenant_model_service import (
     get_api_key,
     get_composite_model_name_by_id,
@@ -1176,7 +1176,7 @@ async def recommendation():
         chat_model_config = get_tenant_default_model_by_type(current_user.id, LLMType.CHAT)
     chat_mdl = LLMBundle(current_user.id, chat_model_config)
 
-    gen_conf = search_config.get("llm_setting", {"temperature": 0.9})
+    gen_conf = resolve_llm_setting(search_config.get("llm_setting"))
     if "parameter" in gen_conf:
         del gen_conf["parameter"]
     prompt = load_prompt("related_question")

api/db/db_models.py

@@ -1206,13 +1206,16 @@ class Search(DataBaseModel):
             "top_k": 1024,
             # chat settings
             "summary": False,
-            "chat_id": "",
-            # Leave it here for reference, don't need to set default values
+            "chat_id": "",  # id of chat model in tenant_model table
             "llm_setting": {
-                # "temperature": 0.1,
-                # "top_p": 0.3,
-                # "frequency_penalty": 0.7,
-                # "presence_penalty": 0.4,
+                "temperature": 0.1,
+                "top_p": 0.3,
+                "frequency_penalty": 0.7,
+                "presence_penalty": 0.4,
+                "temperature_enabled": True,
+                "top_p_enabled": True,
+                "frequency_penalty_enabled": True,
+                "presence_penalty_enabled": True,
             },
             "chat_settingcross_languages": [],
             "highlight": False,