diff --git a/deepdoc/parser/json_parser.py b/deepdoc/parser/json_parser.py index 249ce1ccb9..36045cbd4c 100644 --- a/deepdoc/parser/json_parser.py +++ b/deepdoc/parser/json_parser.py @@ -45,6 +45,15 @@ class RAGFlowJsonParser: """Calculate the size of the serialized JSON object.""" return len(json.dumps(data, ensure_ascii=False)) + @staticmethod + def _is_empty_chunk(chunk: Any) -> bool: + """Only null and empty containers carry no content; 0 and false do.""" + if chunk is None: + return True + if isinstance(chunk, (dict, list, str)): + return not chunk + return False + @staticmethod def _set_nested_dict(d: dict, path: list[str], value: Any) -> None: """Set a value in a nested dictionary based on the given path.""" @@ -93,7 +102,11 @@ class RAGFlowJsonParser: self._json_split(value, new_path, chunks) else: # handle single item - self._set_nested_dict(chunks[-1], current_path, data) + if not current_path: + # top-level scalar (number/string/bool/null) has no key to nest under + chunks[-1] = data + else: + self._set_nested_dict(chunks[-1], current_path, data) return chunks def split_json( @@ -110,7 +123,7 @@ class RAGFlowJsonParser: chunks = self._json_split(json_data, None, None) # Remove the last chunk if it's empty - if not chunks[-1]: + if chunks and self._is_empty_chunk(chunks[-1]): chunks.pop() return chunks @@ -132,7 +145,7 @@ class RAGFlowJsonParser: try: json_data = json.loads(content) chunks = self.split_json(json_data, True) - sections = [json.dumps(line, ensure_ascii=False) for line in chunks if line] + sections = [json.dumps(line, ensure_ascii=False) for line in chunks if not self._is_empty_chunk(line)] except json.JSONDecodeError: pass return sections @@ -146,7 +159,7 @@ class RAGFlowJsonParser: try: data = json.loads(line) chunks = self.split_json(data, convert_lists=True) - all_chunks.extend(json.dumps(chunk, ensure_ascii=False) for chunk in chunks if chunk) + all_chunks.extend(json.dumps(chunk, ensure_ascii=False) for chunk in chunks if not self._is_empty_chunk(chunk)) except json.JSONDecodeError: continue return all_chunks diff --git a/test/unit_test/deepdoc/parser/test_json_parser.py b/test/unit_test/deepdoc/parser/test_json_parser.py new file mode 100644 index 0000000000..a548bcf452 --- /dev/null +++ b/test/unit_test/deepdoc/parser/test_json_parser.py @@ -0,0 +1,86 @@ +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Unit tests for RAGFlowJsonParser. + +Regression for the case where a .json upload whose top-level value is a bare +JSON scalar (a number, string, boolean, or null - all valid JSON) reached +``_json_split`` with an empty ``current_path``. ``_set_nested_dict`` then indexed +``path[-1]`` on an empty list and raised ``IndexError``, which ``_parse_json`` +does not catch (it only guards ``json.JSONDecodeError``), so the whole upload +crashed. A top-level scalar has no key to nest under and must be stored as the +chunk directly. +""" + +import importlib.util +import os +import sys +from unittest import mock + +# Load json_parser by file path so we don't trigger deepdoc/parser/__init__.py +# (which pulls in heavy parsers). json_parser only imports ``find_codec`` from +# rag.nlp, and only inside ``__call__``; stub rag.nlp so the module imports. +if "rag" not in sys.modules: + sys.modules["rag"] = mock.MagicMock() +if "rag.nlp" not in sys.modules: + sys.modules["rag.nlp"] = mock.MagicMock() + + +def _find_project_root(marker="pyproject.toml"): + d = os.path.dirname(os.path.abspath(__file__)) + while d != os.path.dirname(d): + if os.path.exists(os.path.join(d, marker)): + return d + d = os.path.dirname(d) + return None + + +_PROJECT_ROOT = _find_project_root() + +_json_spec = importlib.util.spec_from_file_location( + "deepdoc.parser.json_parser", + os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "json_parser.py"), +) +_json_mod = importlib.util.module_from_spec(_json_spec) +sys.modules["deepdoc.parser.json_parser"] = _json_mod +_json_spec.loader.exec_module(_json_mod) + +RAGFlowJsonParser = _json_mod.RAGFlowJsonParser + + +def test_top_level_scalars_do_not_crash(): + # Previously raised IndexError instead of returning a chunk. + parser = RAGFlowJsonParser() + assert parser._parse_json("42") == ["42"] + assert parser._parse_json('"hello"') == ['"hello"'] + assert parser._parse_json("true") == ["true"] + assert parser._parse_json("0") == ["0"] + assert parser._parse_json("false") == ["false"] + + +def test_top_level_null_yields_no_chunk(): + # null carries no content; it should be dropped, not crash. + parser = RAGFlowJsonParser() + assert parser._parse_json("null") == [] + assert parser._parse_json('""') == [] + assert parser._parse_json("{}") == [] + assert parser._parse_json("[]") == [] + + +def test_objects_and_arrays_still_chunk(): + parser = RAGFlowJsonParser() + assert parser._parse_json('{"a": 1}') == ['{"a": 1}'] + assert parser._parse_json("[1, 2, 3]") != []