zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-05 07:10:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: normalize legacy parser configuration (#17761)

Browse Source
This commit is contained in:
buua436
2026-08-04 13:49:44 +08:00
committed by GitHub
parent 539eb470e3
commit 3e11914144
9 changed files with 102 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
api/db/services/llm_service.py
View File

@@ -27,7 +27,7 @@ from langfuse import propagate_attributes
from api.db.db_models import LLM
from api.db.services.common_service import CommonService
from api.db.services.tenant_llm_service import LLM4Tenant
from common.token_utils import num_tokens_from_string, record_run_token_usage, langfuse_run_attrs
from common.token_utils import langfuse_run_attrs, num_tokens_from_string, record_run_token_usage, truncate
class LLMService(CommonService):
@@ -144,7 +144,14 @@ class LLMBundle(LLM4Tenant):
token_size = num_tokens_from_string(text)
if token_size > self.max_length * 0.95:
target_len = int(self.max_length * 0.95)
safe_texts.append(text[:target_len])
logging.debug(
"LLMBundle.encode truncating input: index=%d model=%s original_tokens=%d target_tokens=%d",
idx,
self.model_config["llm_name"],
token_size,
target_len,
)
safe_texts.append(truncate(text, target_len))
else:
safe_texts.append(text)