fix: normalize legacy parser configuration (#17761)

api/apps/restful_apis/document_api.py

@@ -79,6 +79,34 @@ from common.ssrf_guard import assert_url_is_safe
 from rag.nlp import search
 
 
+def _normalize_legacy_raptor_config(req: dict) -> None:
+    """Drop RAPTOR fields removed from the current parser-config schema."""
+    parser_config = req.get("parser_config")
+    if not isinstance(parser_config, dict):
+        return
+    raptor = parser_config.get("raptor")
+    if not isinstance(raptor, dict):
+        return
+
+    normalized_fields = []
+    legacy_ext = raptor.pop("ext", None)
+    if legacy_ext is not None:
+        normalized_fields.append("ext")
+        if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor:
+            raptor["clustering_threshold"] = legacy_ext["clustering_threshold"]
+            normalized_fields.append("ext.clustering_threshold")
+    for field in ("threshold", "clustering_method", "tree_builder"):
+        if field in raptor:
+            raptor.pop(field)
+            normalized_fields.append(field)
+    max_token = raptor.get("max_token")
+    if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:
+        raptor["max_token"] = 512
+        normalized_fields.append("max_token")
+    if normalized_fields:
+        logging.debug("Document RAPTOR config normalized legacy fields: %s", sorted(normalized_fields))
+
+
 def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool:
     from rag.svr.task_executor_refactor.chunk_post_processor import (
         _parser_config_compilation_template_group_ids,
@@ -212,6 +240,7 @@ async def update_document(tenant_id, dataset_id, document_id):
           type: object
     """
     req = await get_request_json()
+    _normalize_legacy_raptor_config(req)
 
     # An explicit null name is a type error, not an unset field.
     if "name" in req and req["name"] is None:

api/db/services/llm_service.py

@@ -27,7 +27,7 @@ from langfuse import propagate_attributes
 from api.db.db_models import LLM
 from api.db.services.common_service import CommonService
 from api.db.services.tenant_llm_service import LLM4Tenant
-from common.token_utils import num_tokens_from_string, record_run_token_usage, langfuse_run_attrs
+from common.token_utils import langfuse_run_attrs, num_tokens_from_string, record_run_token_usage, truncate
 
 
 class LLMService(CommonService):
@@ -144,7 +144,14 @@ class LLMBundle(LLM4Tenant):
             token_size = num_tokens_from_string(text)
             if token_size > self.max_length * 0.95:
                 target_len = int(self.max_length * 0.95)
-                safe_texts.append(text[:target_len])
+                logging.debug(
+                    "LLMBundle.encode truncating input: index=%d model=%s original_tokens=%d target_tokens=%d",
+                    idx,
+                    self.model_config["llm_name"],
+                    token_size,
+                    target_len,
+                )
+                safe_texts.append(truncate(text, target_len))
             else:
                 safe_texts.append(text)
 

api/utils/validation_utils.py

@@ -371,6 +371,35 @@ class RaptorConfig(Base):
     scope: Annotated[Literal["file", "dataset"], Field(default="file")]
     auto_disable_for_structured_data: Annotated[bool, Field(default=True)]
 
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_legacy_fields(cls, value: Any) -> Any:
+        """Accept old RAPTOR fields but do not retain them in the config."""
+        if not isinstance(value, dict):
+            return value
+
+        normalized = dict(value)
+        changed_fields = []
+        legacy_ext = normalized.pop("ext", None)
+        if legacy_ext is not None:
+            changed_fields.append("ext")
+        if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None:
+            if "clustering_threshold" in legacy_ext:
+                normalized["clustering_threshold"] = legacy_ext["clustering_threshold"]
+                changed_fields.append("ext.clustering_threshold")
+
+        for field in ("threshold", "clustering_method", "tree_builder"):
+            if field in normalized:
+                normalized.pop(field)
+                changed_fields.append(field)
+        max_token = normalized.get("max_token")
+        if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:
+            normalized["max_token"] = 512
+            changed_fields.append("max_token")
+        if changed_fields:
+            logging.debug("RaptorConfig normalized legacy fields: %s", sorted(changed_fields))
+        return normalized
+
 
 class GraphragConfig(Base):
     """Dataset parser configuration for GraphRAG generation."""