zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-22 16:23:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix ingestion pipeline child delimiter extra newline (#18490)

Browse Source
This commit is contained in:
Wang Qi
2026-08-19 13:47:09 +08:00
committed by GitHub
parent 140029e2c7
commit 5f92323ced
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/ingestion/component/chunker/token.go
View File

@@ -1113,8 +1113,8 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.
out = append(out, ck)
continue
}
mom := ck.Text
parts := splitDroppingDelim(mom, pattern)
mom := strings.TrimPrefix(ck.Text, "\n")
parts := splitDroppingDelim(ck.Text, pattern)
for _, p := range parts {
if strings.TrimSpace(p) == "" {
continue
@@ -1160,7 +1160,7 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
if strings.TrimSpace(child) == "" {
continue
}
out = append(out, schema.ChunkDoc{Text: child, Mom: t})
out = append(out, schema.ChunkDoc{Text: child, Mom: strings.TrimPrefix(t, "\n")})
}
}
return out

4
rag/flow/chunker/token_chunker.py
View File

@@ -373,7 +373,7 @@ def _split_chunk_docs_by_children(chunks, pattern):
split_texts = _split_text_by_pattern(chunk.get("text", ""), pattern)
mom = chunk.get("text", "")
mom = chunk.get("text", "").removeprefix("\n")
for text in split_texts:
if not text.strip():
continue
@@ -428,7 +428,7 @@ class TokenChunker(ProcessBase):
for text in _split_text_by_pattern(c, custom_pattern):
if not text.strip():
continue
docs.append({"text": text, "mom": c})
docs.append({"text": text, "mom": c.removeprefix("\n")})
self.set_output("chunks", docs)
else:
self.set_output("chunks", [{"text": c.strip()} for c in cks if c.strip()])