zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-15 05:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(parser): move default delimiter consts to non-test delimiter.go

Browse Source 
DefaultTextCodeDelimiter and DefaultMarkdownDelimiter are shared by both
the production parsers and the alignment tests. They previously lived in
align_test.go (a _test.go file), so production text_parser.go had to keep
a private duplicate that could drift silently. Move them to a non-test
delimiter.go as the single source of truth.
This commit is contained in:
xugangqiang
2026-08-11 16:43:07 +08:00
parent 5cfadd16b1
commit 1e39d24540
2 changed files with 34 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/parser/parser/align_test.go
View File

@@ -317,14 +317,6 @@ func MarkdownAlignOptions(delimiter string) AlignOptions {
}
}
// DefaultMarkdownDelimiter is the flow parser's default Markdown delimiter
// set, used when generating/loading the golden baseline.
const DefaultMarkdownDelimiter = "\n!?;。；！？"
// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter
// set, used when generating/loading the golden baseline.
const DefaultTextCodeDelimiter = "\n!?;。；！？"
// TextCodeAlignOptions returns the normalizer preset for the text&code family.
// Unlike markdown it has no syntax or HTML markup to strip, so only the
// delimiter-set replacement and whitespace collapse run:

34
internal/parser/parser/delimiter.go Normal file
View File

@@ -0,0 +1,34 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// Warranties, INCLUDING THE WARRANTIES OF MERCHANTABILITY AND
// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
package parser
// DefaultMarkdownDelimiter is the flow parser's default markdown delimiter
// set, used when generating/loading the golden baseline and when the markdown
// alignment test normalizes delimiters. It is a non-test symbol so both the
// production parsers and the alignment tests share one source of truth.
const DefaultMarkdownDelimiter = "\n!?;。；！？"
// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter
// set, used by TextParser and when the text&code alignment test normalizes
// delimiters. It is a non-test symbol so both the production parser and the
// alignment tests share one source of truth (no duplicate hard-coded copy in
// text_parser.go, which would otherwise drift silently).
const DefaultTextCodeDelimiter = "\n!?;。；！？"