zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-01 13:33:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: accept empty value as 0 chunk (#14220)

Browse Source 
### What problem does this PR solve?

Fix: accept empty value as 0 chunk
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Magicbook1108
2026-04-20 12:53:47 +08:00
committed by GitHub
parent f554f6ae85
commit 19eedeec61
3 changed files with 33 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
rag/flow/tokenizer/schema.py
View File

@@ -37,17 +37,19 @@ class TokenizerFromUpstream(BaseModel):
@model_validator(mode="after")
def _check_payloads(self) -> "TokenizerFromUpstream":
if self.chunks:
# Empty chunk arrays are valid upstream results for nearly empty files.
if self.output_format == "chunks" and self.chunks is not None:
return self
if self.output_format in {"markdown", "text", "html"}:
if self.output_format == "markdown" and not self.markdown_result:
if self.output_format == "markdown" and self.markdown_result is None:
raise ValueError("output_format=markdown requires a markdown payload (field: 'markdown' or 'markdown_result').")
if self.output_format == "text" and not self.text_result:
if self.output_format == "text" and self.text_result is None:
raise ValueError("output_format=text requires a text payload (field: 'text' or 'text_result').")
if self.output_format == "html" and not self.html_result:
if self.output_format == "html" and self.html_result is None:
raise ValueError("output_format=text requires a html payload (field: 'html' or 'html_result').")
else:
if not self.json_result and not self.chunks:
# Distinguish a missing JSON payload from a present-but-empty one.
if self.json_result is None and self.chunks is None:
raise ValueError("When no chunks are provided and output_format is not markdown/text, a JSON list payload is required (field: 'json' or 'json_result').")
return self

12
rag/flow/tokenizer/tokenizer.py
View File

@@ -52,6 +52,10 @@ class Tokenizer(ProcessBase):
component_name = "Tokenizer"
async def _embedding(self, name, chunks):
# Tokenization may legitimately produce zero chunks; embedding should be a no-op.
if not chunks:
return [], 0
parts = sum(["full_text" in self._param.search_method, "embedding" in self._param.search_method])
token_count = 0
if self._canvas._kb_id:
@@ -121,8 +125,9 @@ class Tokenizer(ProcessBase):
parts = sum(["full_text" in self._param.search_method, "embedding" in self._param.search_method])
if "full_text" in self._param.search_method:
self.callback(random.randint(1, 5) / 100.0, "Start to tokenize.")
if from_upstream.chunks:
chunks = from_upstream.chunks
# Branch on the declared upstream format so an empty chunk list stays on the chunk path.
if from_upstream.output_format == "chunks":
chunks = from_upstream.chunks or []
for i, ck in enumerate(chunks):
ck["chunk_order_int"] = i
ck["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", from_upstream.name))
@@ -161,7 +166,8 @@ class Tokenizer(ProcessBase):
ck["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(ck["content_ltks"])
chunks = [ck]
else:
chunks = from_upstream.json_result
# Empty JSON payloads are valid and should remain empty downstream.
chunks = from_upstream.json_result or []
for i, ck in enumerate(chunks):
ck["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", from_upstream.name))
ck["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(ck["title_tks"])