From 3e11914144861262723db47c1048847d56bdc9ef Mon Sep 17 00:00:00 2001 From: buua436 Date: Tue, 4 Aug 2026 13:49:44 +0800 Subject: [PATCH] fix: normalize legacy parser configuration (#17761) --- api/apps/restful_apis/document_api.py | 29 +++++++++++++++ api/db/services/llm_service.py | 11 ++++-- api/utils/validation_utils.py | 29 +++++++++++++++ rag/llm/embedding_model.py | 36 ++++++++++++++++++- test/testcases/restful_api/test_datasets.py | 4 --- .../test_create_dataset.py | 4 --- .../test_update_dataset.py | 4 --- .../test_create_dataset.py | 4 --- .../test_update_dataset.py | 4 --- 9 files changed, 102 insertions(+), 23 deletions(-) diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py index cb1a575de7..195fcd72d1 100644 --- a/api/apps/restful_apis/document_api.py +++ b/api/apps/restful_apis/document_api.py @@ -79,6 +79,34 @@ from common.ssrf_guard import assert_url_is_safe from rag.nlp import search +def _normalize_legacy_raptor_config(req: dict) -> None: + """Drop RAPTOR fields removed from the current parser-config schema.""" + parser_config = req.get("parser_config") + if not isinstance(parser_config, dict): + return + raptor = parser_config.get("raptor") + if not isinstance(raptor, dict): + return + + normalized_fields = [] + legacy_ext = raptor.pop("ext", None) + if legacy_ext is not None: + normalized_fields.append("ext") + if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor: + raptor["clustering_threshold"] = legacy_ext["clustering_threshold"] + normalized_fields.append("ext.clustering_threshold") + for field in ("threshold", "clustering_method", "tree_builder"): + if field in raptor: + raptor.pop(field) + normalized_fields.append(field) + max_token = raptor.get("max_token") + if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512: + raptor["max_token"] = 512 + normalized_fields.append("max_token") + if normalized_fields: + logging.debug("Document RAPTOR config normalized legacy fields: %s", sorted(normalized_fields)) + + def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool: from rag.svr.task_executor_refactor.chunk_post_processor import ( _parser_config_compilation_template_group_ids, @@ -212,6 +240,7 @@ async def update_document(tenant_id, dataset_id, document_id): type: object """ req = await get_request_json() + _normalize_legacy_raptor_config(req) # An explicit null name is a type error, not an unset field. if "name" in req and req["name"] is None: diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.py index 148cba7373..9537e260da 100644 --- a/api/db/services/llm_service.py +++ b/api/db/services/llm_service.py @@ -27,7 +27,7 @@ from langfuse import propagate_attributes from api.db.db_models import LLM from api.db.services.common_service import CommonService from api.db.services.tenant_llm_service import LLM4Tenant -from common.token_utils import num_tokens_from_string, record_run_token_usage, langfuse_run_attrs +from common.token_utils import langfuse_run_attrs, num_tokens_from_string, record_run_token_usage, truncate class LLMService(CommonService): @@ -144,7 +144,14 @@ class LLMBundle(LLM4Tenant): token_size = num_tokens_from_string(text) if token_size > self.max_length * 0.95: target_len = int(self.max_length * 0.95) - safe_texts.append(text[:target_len]) + logging.debug( + "LLMBundle.encode truncating input: index=%d model=%s original_tokens=%d target_tokens=%d", + idx, + self.model_config["llm_name"], + token_size, + target_len, + ) + safe_texts.append(truncate(text, target_len)) else: safe_texts.append(text) diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py index cc3e7cdcc9..4a54756e73 100644 --- a/api/utils/validation_utils.py +++ b/api/utils/validation_utils.py @@ -371,6 +371,35 @@ class RaptorConfig(Base): scope: Annotated[Literal["file", "dataset"], Field(default="file")] auto_disable_for_structured_data: Annotated[bool, Field(default=True)] + @model_validator(mode="before") + @classmethod + def normalize_legacy_fields(cls, value: Any) -> Any: + """Accept old RAPTOR fields but do not retain them in the config.""" + if not isinstance(value, dict): + return value + + normalized = dict(value) + changed_fields = [] + legacy_ext = normalized.pop("ext", None) + if legacy_ext is not None: + changed_fields.append("ext") + if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None: + if "clustering_threshold" in legacy_ext: + normalized["clustering_threshold"] = legacy_ext["clustering_threshold"] + changed_fields.append("ext.clustering_threshold") + + for field in ("threshold", "clustering_method", "tree_builder"): + if field in normalized: + normalized.pop(field) + changed_fields.append(field) + max_token = normalized.get("max_token") + if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512: + normalized["max_token"] = 512 + changed_fields.append("max_token") + if changed_fields: + logging.debug("RaptorConfig normalized legacy fields: %s", sorted(changed_fields)) + return normalized + class GraphragConfig(Base): """Dataset parser configuration for GraphRAG generation.""" diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py index 560b324d6a..a1c137dfe8 100644 --- a/rag/llm/embedding_model.py +++ b/rag/llm/embedding_model.py @@ -973,7 +973,41 @@ class SILICONFLOWEmbed(Base): def _clean_batch(self, batch): if self.model_name in ["BAAI/bge-large-zh-v1.5", "BAAI/bge-large-en-v1.5"]: # limit 512, 340 is almost safe - return [" " if not text.strip() else truncate(text, 256) for text in batch] + limit = 256 + cleaned = [] + for index, text in enumerate(batch): + if not text.strip(): + cleaned.append(" ") + continue + original_tokens = num_tokens_from_string(text) + if original_tokens > limit: + logger.debug( + "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d", + self.model_name, + index, + original_tokens, + limit, + ) + cleaned.append(truncate(text, limit)) + return cleaned + if self.model_name in ["BAAI/bge-m3", "Pro/BAAI/bge-m3"]: + limit = 4096 + cleaned = [] + for index, text in enumerate(batch): + if not text.strip(): + cleaned.append(" ") + continue + original_tokens = num_tokens_from_string(text) + if original_tokens > limit: + logger.debug( + "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d", + self.model_name, + index, + original_tokens, + limit, + ) + cleaned.append(truncate(text, limit)) + return cleaned return [" " if not text.strip() else text for text in batch] def _call(self, batch): diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py index f4f8d22145..b1f8d70cad 100644 --- a/test/testcases/restful_api/test_datasets.py +++ b/test/testcases/restful_api/test_datasets.py @@ -1020,9 +1020,7 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase ({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ({"raptor": {"prompt": ""}}, "String should have at least 1 character"), ({"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"), ({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ({"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), @@ -1745,9 +1743,7 @@ def test_dataset_create_parser_config_invalid_contract(rest_client, clear_datase ("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"), ("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"), ("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), diff --git a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py index ed2c4319cf..68787962d2 100644 --- a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py +++ b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py @@ -563,9 +563,7 @@ class TestDatasetCreate: ("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"), ("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"), ("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), @@ -621,9 +619,7 @@ class TestDatasetCreate: "raptor_type_invalid", "raptor_prompt_empty", "raptor_prompt_space", - "raptor_max_token_min_limit", "raptor_max_token_max_limit", - "raptor_max_token_float_not_allowed", "raptor_max_token_type_invalid", "raptor_clustering_threshold_min_limit", "raptor_clustering_threshold_max_limit", diff --git a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py index d216b35793..65fe4b2441 100644 --- a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py +++ b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py @@ -693,9 +693,7 @@ class TestDatasetUpdate: ({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ({"raptor": {"prompt": ""}}, "String should have at least 1 character"), ({"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"), ({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ({"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), @@ -749,9 +747,7 @@ class TestDatasetUpdate: "raptor_type_invalid", "raptor_prompt_empty", "raptor_prompt_space", - "raptor_max_token_min_limit", "raptor_max_token_max_limit", - "raptor_max_token_float_not_allowed", "raptor_max_token_type_invalid", "raptor_clustering_threshold_min_limit", "raptor_clustering_threshold_max_limit", diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py index ab626f0b48..8a60697e14 100644 --- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py +++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py @@ -512,9 +512,7 @@ class TestDatasetCreate: ("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"), ("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"), ("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), @@ -568,9 +566,7 @@ class TestDatasetCreate: "raptor_type_invalid", "raptor_prompt_empty", "raptor_prompt_space", - "raptor_max_token_min_limit", "raptor_max_token_max_limit", - "raptor_max_token_float_not_allowed", "raptor_max_token_type_invalid", "raptor_clustering_threshold_min_limit", "raptor_clustering_threshold_max_limit", diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py index daba719c6c..4047ec9b26 100644 --- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py +++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py @@ -581,9 +581,7 @@ class TestDatasetUpdate: ({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"), ({"raptor": {"prompt": ""}}, "String should have at least 1 character"), ({"raptor": {"prompt": " "}}, "String should have at least 1 character"), - ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"), ({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"), - ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"), ({"raptor": {"max_token": "string"}}, "Input should be a valid integer"), ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"), ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"), @@ -637,9 +635,7 @@ class TestDatasetUpdate: "raptor_type_invalid", "raptor_prompt_empty", "raptor_prompt_space", - "raptor_max_token_min_limit", "raptor_max_token_max_limit", - "raptor_max_token_float_not_allowed", "raptor_max_token_type_invalid", "raptor_clustering_threshold_min_limit", "raptor_clustering_threshold_max_limit",