mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-10 13:45:44 +08:00
Closes #14428 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
@@ -97,7 +97,24 @@ class LLMBundle(LLM4Tenant):
|
||||
generation = self.langfuse.start_observation(trace_context=self.trace_context, as_type="generation", name="encode", model=self.model_config["llm_name"], input={"texts": texts})
|
||||
|
||||
safe_texts = []
|
||||
for text in texts:
|
||||
for idx, text in enumerate(texts):
|
||||
# Embedding APIs (OpenAI-compatible, Zhipu, etc.) reject empty or
|
||||
# whitespace-only inputs with errors like "Input at index N cannot
|
||||
# be empty or whitespace only". Upstream parsers can produce such
|
||||
# chunks — e.g. when OCR/vision on an embedded DOCX image returns
|
||||
# nothing, or a table has only empty cells — so coerce to a safe
|
||||
# placeholder here, at the single boundary every embedding path
|
||||
# funnels through.
|
||||
if text is None or not str(text).strip():
|
||||
marker = "None" if text is None else "whitespace-only"
|
||||
logging.warning(
|
||||
"LLMBundle.encode: empty input at index %d (%s) coerced to placeholder 'None' for model %s",
|
||||
idx,
|
||||
marker,
|
||||
self.model_config["llm_name"],
|
||||
)
|
||||
safe_texts.append("None")
|
||||
continue
|
||||
token_size = num_tokens_from_string(text)
|
||||
if token_size > self.max_length:
|
||||
target_len = int(self.max_length * 0.95)
|
||||
@@ -121,6 +138,14 @@ class LLMBundle(LLM4Tenant):
|
||||
if self.langfuse:
|
||||
generation = self.langfuse.start_observation(trace_context=self.trace_context, as_type="generation", name="encode_queries", model=self.model_config["llm_name"], input={"query": query})
|
||||
|
||||
if query is None or not str(query).strip():
|
||||
marker = "None" if query is None else "whitespace-only"
|
||||
logging.warning(
|
||||
"LLMBundle.encode_queries: empty query (%s) coerced to placeholder 'None' for model %s",
|
||||
marker,
|
||||
self.model_config["llm_name"],
|
||||
)
|
||||
query = "None"
|
||||
emd, used_tokens = self.mdl.encode_queries(query)
|
||||
if self.model_config["llm_factory"] == "Builtin":
|
||||
logging.info("LLMBundle.encode_queries query: {}, emd len: {}, used_tokens: {}. Builtin model don't need to update token usage".format(query, len(emd), used_tokens))
|
||||
|
||||
Reference in New Issue
Block a user