mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-06-29 15:31:05 +08:00
fix: tighten embedding truncation threshold (#16123)
### What problem does this PR solve? Use a 95% max_length threshold before truncating embedding inputs, which reduces the chance of provider-side invalid-parameter errors on near-limit chunks. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
@@ -137,7 +137,7 @@ class LLMBundle(LLM4Tenant):
|
||||
safe_texts.append("None")
|
||||
continue
|
||||
token_size = num_tokens_from_string(text)
|
||||
if token_size > self.max_length:
|
||||
if token_size > self.max_length * 0.95:
|
||||
target_len = int(self.max_length * 0.95)
|
||||
safe_texts.append(text[:target_len])
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user