From 1e39d24540450f9aa26d5c43418d16532ac22091 Mon Sep 17 00:00:00 2001 From: xugangqiang Date: Tue, 11 Aug 2026 16:43:07 +0800 Subject: [PATCH] refactor(parser): move default delimiter consts to non-test delimiter.go DefaultTextCodeDelimiter and DefaultMarkdownDelimiter are shared by both the production parsers and the alignment tests. They previously lived in align_test.go (a _test.go file), so production text_parser.go had to keep a private duplicate that could drift silently. Move them to a non-test delimiter.go as the single source of truth. --- internal/parser/parser/align_test.go | 8 ------- internal/parser/parser/delimiter.go | 34 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 internal/parser/parser/delimiter.go diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.go index 51911f2279..b138e764f9 100644 --- a/internal/parser/parser/align_test.go +++ b/internal/parser/parser/align_test.go @@ -317,14 +317,6 @@ func MarkdownAlignOptions(delimiter string) AlignOptions { } } -// DefaultMarkdownDelimiter is the flow parser's default Markdown delimiter -// set, used when generating/loading the golden baseline. -const DefaultMarkdownDelimiter = "\n!?;。;!?" - -// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter -// set, used when generating/loading the golden baseline. -const DefaultTextCodeDelimiter = "\n!?;。;!?" - // TextCodeAlignOptions returns the normalizer preset for the text&code family. // Unlike markdown it has no syntax or HTML markup to strip, so only the // delimiter-set replacement and whitespace collapse run: diff --git a/internal/parser/parser/delimiter.go b/internal/parser/parser/delimiter.go new file mode 100644 index 0000000000..9ab0e62e1d --- /dev/null +++ b/internal/parser/parser/delimiter.go @@ -0,0 +1,34 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// Warranties, INCLUDING THE WARRANTIES OF MERCHANTABILITY AND +// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +package parser + +// DefaultMarkdownDelimiter is the flow parser's default markdown delimiter +// set, used when generating/loading the golden baseline and when the markdown +// alignment test normalizes delimiters. It is a non-test symbol so both the +// production parsers and the alignment tests share one source of truth. +const DefaultMarkdownDelimiter = "\n!?;。;!?" + +// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter +// set, used by TextParser and when the text&code alignment test normalizes +// delimiters. It is a non-test symbol so both the production parser and the +// alignment tests share one source of truth (no duplicate hard-coded copy in +// text_parser.go, which would otherwise drift silently). +const DefaultTextCodeDelimiter = "\n!?;。;!?"