refactor(go): clean up TokenChunker delimiter routing and children appliers (#17954)

internal/ingestion/component/chunker/common.go

@@ -78,9 +78,10 @@ func stringListFromAny(in []any) []string {
 
 // compileDelimPattern compiles a TokenChunker-style []string delimiter list.
 // Only backtick-wrapped entries produce an active pattern (Python
-// token_chunker / rag/nlp/delim list helper). Plain entries are ignored here
-// and used by mergeByTokenSize for sentence-level splitting when no active
-// pattern exists.
+// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries
+// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,
+// not this list. Canonical single-string parser_config.delimiter parsing
+// lives in ragflow/internal/parser/chunk (ParseDelimiterField).
 func compileDelimPattern(delims []string) *regexp.Regexp {
 	return chunk.CompileDelimiterListPattern(delims)
 }

internal/ingestion/component/chunker/token.go

@@ -336,19 +336,18 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
 	if len(cleaned) == 0 {
 		return emptyOutputs()
 	}
-	docs := applyChildrenDelim(cleaned, childrenPattern)
-
-	// Python's naive_merge: custom (backtick) delimiters produce one
-	// chunk per segment — no token-size merge (naive_merge:1194-1213).
-	if hasCustomDelim(c.param.Delimiters) {
-		return chunkOutputs(docs)
+	textDocs := make([]schema.ChunkDoc, 0, len(cleaned))
+	for _, s := range cleaned {
+		textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})
 	}
+	docs := applyChildrenDelimText(textDocs, childrenPattern)
 
-	// Split-then-merge: split on delimiters, then greedily merge to
-	// chunk_token_size with optional overlap.
-	perItem := [][]schema.ChunkDoc{docs}
-	merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent, c.param.MergeStrategy())
-	return chunkOutputs(flatten(merged))
+	// Python's naive_merge: a custom (backtick) delimiter yields one chunk
+	// per segment and no token-size merge (naive_merge:1194-1213). A
+	// non-custom active delimiter cannot reach here — delimPattern is
+	// non-nil only when a backtick delimiter exists, so the split-then-
+	// merge branch was unreachable and has been removed.
+	return chunkOutputs(docs)
 }
 
 // sentenceDelimiter is the sentence/clause-boundary regex used to split
@@ -1108,12 +1107,11 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.
 // shared text-payload helpers (used by TitleChunker et al.)
 // ---------------------------------------------------------------------------
 
-// hasActiveDelimiter reports whether a regex compiled by
-// compileDelimPattern contains any non-placeholder pattern. The "match
-// nothing" sentinel regexp makes a quick `pattern.MatchString("")`
-// viable as a check without re-walking the source slice.
+// hasActiveDelimiter reports whether a compiled delimiter pattern is
+// present (non-nil). compileDelimPattern returns nil when no active
+// pattern exists, so a nil check is sufficient.
 func hasActiveDelimiter(p *regexp.Regexp) bool {
-	return p != nil && p.String() != `\A(?!)`
+	return p != nil
 }
 
 // hasCustomDelim reports whether any delimiter uses backtick syntax
@@ -1123,34 +1121,6 @@ func hasCustomDelim(delims []string) bool {
 	return chunk.HasCustomDelimiterList(delims)
 }
 
-// applyChildrenDelim mirrors token_chunker.py:325-334.
-func applyChildrenDelim(segs []string, pattern *regexp.Regexp) []schema.ChunkDoc {
-	if pattern == nil {
-		out := make([]schema.ChunkDoc, 0, len(segs))
-		for _, s := range segs {
-			out = append(out, schema.ChunkDoc{
-				Text:    s,
-				DocType: "text",
-				CKType:  "text",
-			})
-		}
-		return out
-	}
-	var docs []schema.ChunkDoc
-	for _, seg := range segs {
-		if strings.TrimSpace(seg) == "" {
-			continue
-		}
-		for _, child := range splitDroppingDelim(seg, pattern) {
-			if strings.TrimSpace(child) == "" {
-				continue
-			}
-			docs = append(docs, schema.ChunkDoc{Text: child, Mom: seg})
-		}
-	}
-	return docs
-}
-
 func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
 	if pattern == nil {
 		return docs

internal/ingestion/component/chunker/token_children_delim_test.go

@@ -45,9 +45,8 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {
 }
 
 // TestTokenChunker_ChildrenDelimiterDroppedText asserts the text/markdown/html
-// children_delimiters split also DROPS the delimiter (applyChildrenDelim /
-// applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent
-// segment in "mom".
+// children_delimiters split also DROPS the delimiter (applyChildrenDelimText
+// mirrors _split_text_by_pattern), keeping the parent segment in "mom".
 // TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a
 // backtick-wrapped children_delimiter contributes its INNER content as the
 // split pattern (not the literal wrapped token), and the matched delimiter is