From 19eedeec61da33b4f5d4f2e33b63feef9eaab6a9 Mon Sep 17 00:00:00 2001 From: Magicbook1108 Date: Mon, 20 Apr 2026 12:53:47 +0800 Subject: [PATCH] Fix: accept empty value as 0 chunk (#14220) ### What problem does this PR solve? Fix: accept empty value as 0 chunk ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- rag/flow/tokenizer/schema.py | 12 +++++++----- rag/flow/tokenizer/tokenizer.py | 12 +++++++++--- rag/svr/task_executor.py | 25 +++++++++++++++++-------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/rag/flow/tokenizer/schema.py b/rag/flow/tokenizer/schema.py index e74a5825fe..40e676aa09 100644 --- a/rag/flow/tokenizer/schema.py +++ b/rag/flow/tokenizer/schema.py @@ -37,17 +37,19 @@ class TokenizerFromUpstream(BaseModel): @model_validator(mode="after") def _check_payloads(self) -> "TokenizerFromUpstream": - if self.chunks: + # Empty chunk arrays are valid upstream results for nearly empty files. + if self.output_format == "chunks" and self.chunks is not None: return self if self.output_format in {"markdown", "text", "html"}: - if self.output_format == "markdown" and not self.markdown_result: + if self.output_format == "markdown" and self.markdown_result is None: raise ValueError("output_format=markdown requires a markdown payload (field: 'markdown' or 'markdown_result').") - if self.output_format == "text" and not self.text_result: + if self.output_format == "text" and self.text_result is None: raise ValueError("output_format=text requires a text payload (field: 'text' or 'text_result').") - if self.output_format == "html" and not self.html_result: + if self.output_format == "html" and self.html_result is None: raise ValueError("output_format=text requires a html payload (field: 'html' or 'html_result').") else: - if not self.json_result and not self.chunks: + # Distinguish a missing JSON payload from a present-but-empty one. + if self.json_result is None and self.chunks is None: raise ValueError("When no chunks are provided and output_format is not markdown/text, a JSON list payload is required (field: 'json' or 'json_result').") return self diff --git a/rag/flow/tokenizer/tokenizer.py b/rag/flow/tokenizer/tokenizer.py index ea2e59aec4..9992ca722b 100644 --- a/rag/flow/tokenizer/tokenizer.py +++ b/rag/flow/tokenizer/tokenizer.py @@ -52,6 +52,10 @@ class Tokenizer(ProcessBase): component_name = "Tokenizer" async def _embedding(self, name, chunks): + # Tokenization may legitimately produce zero chunks; embedding should be a no-op. + if not chunks: + return [], 0 + parts = sum(["full_text" in self._param.search_method, "embedding" in self._param.search_method]) token_count = 0 if self._canvas._kb_id: @@ -121,8 +125,9 @@ class Tokenizer(ProcessBase): parts = sum(["full_text" in self._param.search_method, "embedding" in self._param.search_method]) if "full_text" in self._param.search_method: self.callback(random.randint(1, 5) / 100.0, "Start to tokenize.") - if from_upstream.chunks: - chunks = from_upstream.chunks + # Branch on the declared upstream format so an empty chunk list stays on the chunk path. + if from_upstream.output_format == "chunks": + chunks = from_upstream.chunks or [] for i, ck in enumerate(chunks): ck["chunk_order_int"] = i ck["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", from_upstream.name)) @@ -161,7 +166,8 @@ class Tokenizer(ProcessBase): ck["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(ck["content_ltks"]) chunks = [ck] else: - chunks = from_upstream.json_result + # Empty JSON payloads are valid and should remain empty downstream. + chunks = from_upstream.json_result or [] for i, ck in enumerate(chunks): ck["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", from_upstream.name)) ck["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(ck["title_tks"]) diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index 775cd52656..99dfe6dfa7 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -656,16 +656,25 @@ async def run_dataflow(task: dict): return embedding_token_consumption = chunks.get("embedding_token_consumption", 0) - if chunks.get("chunks"): + # The output key may exist with an empty payload; check presence, not truthiness. + if "chunks" in chunks: chunks = copy.deepcopy(chunks["chunks"]) - elif chunks.get("json"): + elif "json" in chunks: chunks = copy.deepcopy(chunks["json"]) - elif chunks.get("markdown"): - chunks = [{"text": [chunks["markdown"]]}] - elif chunks.get("text"): - chunks = [{"text": [chunks["text"]]}] - elif chunks.get("html"): - chunks = [{"text": [chunks["html"]]}] + elif "markdown" in chunks: + chunks = [{"text": [chunks["markdown"]]}] if chunks["markdown"] else [] + elif "text" in chunks: + chunks = [{"text": [chunks["text"]]}] if chunks["text"] else [] + elif "html" in chunks: + chunks = [{"text": [chunks["html"]]}] if chunks["html"] else [] + else: + chunks = [] + + # An empty normalized payload means "nothing parsed", so stop before embedding/indexing. + if not chunks: + PipelineOperationLogService.create(document_id=doc_id, pipeline_id=dataflow_id, + task_type=PipelineTaskType.PARSE, dsl=str(pipeline)) + return keys = [k for o in chunks for k in list(o.keys())] if not any([re.match(r"q_[0-9]+_vec", k) for k in keys]):