mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 23:24:05 +08:00
51 lines
2.1 KiB
Python
51 lines
2.1 KiB
Python
|
#
|
||
|
|
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
#
|
||
|
|
|
||
|
|
"""The API default delimiters must be an actual newline, not the raw 2-char
|
||
|
|
escape ``r"\\n"``.
|
||
|
|
|
||
|
|
A default stored as ``r"\\n"`` reaches ``rag.nlp.delim.parse_delimiter_field`` as
|
||
|
|
a backslash followed by the letter ``n``. The canonical parser honours bare
|
||
|
|
characters, so it then splits documents on every ``n`` (``"information"`` ->
|
||
|
|
``"i"``, ``"formatio"``) and on the stray backslash. An actual newline parses to
|
||
|
|
a single newline delimiter. The frontend already sends a real newline and the
|
||
|
|
``naive`` default already uses one; these guard the ``knowledge_graph`` and
|
||
|
|
``ParserConfig`` / ``ParentChildConfig`` defaults that did not.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from api.utils.api_utils import get_parser_config
|
||
|
|
from api.utils.validation_utils import ParentChildConfig, ParserConfig
|
||
|
|
from rag.nlp.delim import parse_delimiter_field
|
||
|
|
|
||
|
|
|
||
|
|
def _assert_real_newline(delimiter):
|
||
|
|
assert delimiter == "\n", f"default delimiter should be a real newline, got {delimiter!r}"
|
||
|
|
# A real newline parses to a single newline delimiter. The raw escape would
|
||
|
|
# instead parse to ["\\", "n"] and shred the document on the letter n.
|
||
|
|
assert parse_delimiter_field(delimiter) == ["\n"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_parser_config_delimiter_default_is_real_newline():
|
||
|
|
_assert_real_newline(ParserConfig().delimiter)
|
||
|
|
|
||
|
|
|
||
|
|
def test_parent_child_children_delimiter_default_is_real_newline():
|
||
|
|
_assert_real_newline(ParentChildConfig().children_delimiter)
|
||
|
|
|
||
|
|
|
||
|
|
def test_knowledge_graph_delimiter_default_is_real_newline():
|
||
|
|
_assert_real_newline(get_parser_config("knowledge_graph", None)["delimiter"])
|