zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-20 22:51:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement builtin chunk method as ingestion pipeline in GO (#16822)

Browse Source 
### Summary

Implement builtin chunk mehtod as ingestion pipeline in GO
This commit is contained in:
qinling0210
2026-07-13 13:51:40 +08:00
committed by GitHub
parent ed27b5f9f8
commit d549194562
97 changed files with 11427 additions and 3285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/handler/components_testpkg/components_handler_test.go
View File

@@ -133,8 +133,8 @@ func TestComponentsHandler_NoFilter(t *testing.T) {
}
// TestComponentsHandler_FilterIngestion verifies the
// ?category=ingestion filter returns the 8 ingestion components
// (Extractor, File, Parser, Tokenizer + 4 chunker variants). Names
// ?category=ingestion filter returns the ingestion components
// (Extractor, File, Parser, Tokenizer + 9 chunker variants). Names
// must be sorted ascending (plan §4 task 1 stable output).
func TestComponentsHandler_FilterIngestion(t *testing.T) {
eng := newComponentsTestRig(t)
@@ -147,7 +147,8 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
"parser", "titlechunker", "tokenchunker", "tokenizer",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "ingestion", data, wantNames)
@@ -172,7 +173,8 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
"parser", "titlechunker", "tokenchunker", "tokenizer",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "ingestion,shared", data, wantNames)
}
@@ -272,7 +274,8 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
"parser", "titlechunker", "tokenchunker", "tokenizer",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "INGESTION (case-folded)", data, wantNames)
}