diff --git a/api/utils/api_utils.py b/api/utils/api_utils.py index e9fd8af972..d8f052577f 100644 --- a/api/utils/api_utils.py +++ b/api/utils/api_utils.py @@ -390,7 +390,7 @@ def get_parser_config(chunk_method, parser_config): "one": None, "knowledge_graph": { "chunk_token_num": 8192, - "delimiter": r"\n", + "delimiter": "\n", "entity_types": ["organization", "person", "location", "event", "time"], "raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}, diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py index 4a54756e73..0f7b22f970 100644 --- a/api/utils/validation_utils.py +++ b/api/utils/validation_utils.py @@ -425,7 +425,7 @@ class ParentChildConfig(Base): """Dataset parser configuration for parent-child chunking.""" use_parent_child: Annotated[bool, Field(default=False)] - children_delimiter: Annotated[str, Field(default=r"\n", min_length=1)] + children_delimiter: Annotated[str, Field(default="\n", min_length=1)] class AutoMetadataField(Base): @@ -453,7 +453,7 @@ class ParserConfig(Base): auto_keywords: Annotated[int, Field(default=0, ge=0, le=32)] auto_questions: Annotated[int, Field(default=0, ge=0, le=10)] chunk_token_num: Annotated[int, Field(default=512, ge=1, le=2048)] - delimiter: Annotated[str, Field(default=r"\n", min_length=1)] + delimiter: Annotated[str, Field(default="\n", min_length=1)] graphrag: Annotated[GraphragConfig, Field(default_factory=lambda: GraphragConfig(use_graphrag=False))] html4excel: Annotated[bool, Field(default=False)] layout_recognize: Annotated[str, Field(default="DeepDOC")] diff --git a/test/unit_test/api/utils/test_default_delimiter.py b/test/unit_test/api/utils/test_default_delimiter.py new file mode 100644 index 0000000000..b0d0ff958f --- /dev/null +++ b/test/unit_test/api/utils/test_default_delimiter.py @@ -0,0 +1,50 @@ +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""The API default delimiters must be an actual newline, not the raw 2-char +escape ``r"\\n"``. + +A default stored as ``r"\\n"`` reaches ``rag.nlp.delim.parse_delimiter_field`` as +a backslash followed by the letter ``n``. The canonical parser honours bare +characters, so it then splits documents on every ``n`` (``"information"`` -> +``"i"``, ``"formatio"``) and on the stray backslash. An actual newline parses to +a single newline delimiter. The frontend already sends a real newline and the +``naive`` default already uses one; these guard the ``knowledge_graph`` and +``ParserConfig`` / ``ParentChildConfig`` defaults that did not. +""" + +from api.utils.api_utils import get_parser_config +from api.utils.validation_utils import ParentChildConfig, ParserConfig +from rag.nlp.delim import parse_delimiter_field + + +def _assert_real_newline(delimiter): + assert delimiter == "\n", f"default delimiter should be a real newline, got {delimiter!r}" + # A real newline parses to a single newline delimiter. The raw escape would + # instead parse to ["\\", "n"] and shred the document on the letter n. + assert parse_delimiter_field(delimiter) == ["\n"] + + +def test_parser_config_delimiter_default_is_real_newline(): + _assert_real_newline(ParserConfig().delimiter) + + +def test_parent_child_children_delimiter_default_is_real_newline(): + _assert_real_newline(ParentChildConfig().children_delimiter) + + +def test_knowledge_graph_delimiter_default_is_real_newline(): + _assert_real_newline(get_parser_config("knowledge_graph", None)["delimiter"])