diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.go index 1fefbb7650..cfac9b7b46 100644 --- a/internal/handler/components_testpkg/components_handler_test.go +++ b/internal/handler/components_testpkg/components_handler_test.go @@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) { _, _, data := decodeEnvelope(t, w.Body.Bytes()) wantNames := []string{ - "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } @@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) { _, _, data := decodeEnvelope(t, w.Body.Bytes()) wantNames := []string{ - "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } @@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) { } _, _, data := decodeEnvelope(t, w.Body.Bytes()) wantNames := []string{ - "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index 92c32f5679..c0d57170bb 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -41,6 +41,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er return NewTitleChunker(params) case ComponentNameGroupTitleChunker: return NewGroupTitleChunker(params) + case ComponentNameManualChunker: + return NewManualChunker(params) case ComponentNameHierarchyTitleChunker: return NewHierarchyTitleChunker(params) case ComponentNameQAChunker: diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go index 6c87507300..7a0572ec7e 100644 --- a/internal/ingestion/component/chunker/group.go +++ b/internal/ingestion/component/chunker/group.go @@ -107,9 +107,9 @@ func buildSectionIDs(levels []int, targetLevel int) []int { } // invokeGroup runs the GroupTitleChunker strategy against the -// supplied inputs. Detected headings + adjacent merges happen in two -// goroutines (heading detection sequential, then a fan-out over -// record-buckets for the merge pass). +// supplied inputs. It extracts the line records and defers to the +// shared chunkFromRecords pipeline (which ManualChunker also uses, +// after a physical-position resort). func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) { records := extractLineRecords(inputs) common.Debug("chunker stage", @@ -120,6 +120,18 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, if len(records) == 0 { return emptyOutputs(), nil } + return chunkFromRecords(parentCtx, db, inputs, p, records) +} + +// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over +// an already-extracted record list: resolve heading levels, split into +// sections, merge adjacent text records, build chunks, and perform on-demand +// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker +// feeds them after a (page, top, left) resort. Sharing this body guarantees +// both strategies emit byte-identical output for coordinate-free input (the +// docx manual branch) — the no-regression contract locked by +// TestManualChunker_NoPositionsEqualsGroupChunker. +func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) { ctx := newLevelContext(records, outlineFromInputs(inputs), p) levels := ctx.Levels() // Count heading level distribution for debugging. @@ -170,7 +182,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil { engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream) if eErr != nil { - slog.Warn("GroupTitleChunker: could not open PDF for on-demand cropping", "err", eErr) + slog.Warn("chunker: could not open PDF for on-demand cropping", "err", eErr) } if engine != nil { defer engine.Close() @@ -330,6 +342,21 @@ func removeTag(text string) string { return posTagRemove.ReplaceAllString(text, "") } +// pdfPosRowLess orders two PDF coordinate 5-tuples [page,left,right,top,bottom] +// by (page, top, left) — i.e. by indices (0, 3, 1). It is the single shared +// (page, top, left) comparator used both when re-sorting chunker records +// (ManualChunker) and when de-duplicating/merging position matrices +// (mergePositionMatrix). Both rows must have length >= 5. +func pdfPosRowLess(a, b []float64) bool { + if a[0] != b[0] { + return a[0] < b[0] + } + if a[3] != b[3] { + return a[3] < b[3] + } + return a[1] < b[1] +} + // mergePositionMatrix aggregates multiple PDF coordinate matrices into a // single de-duplicated, sorted matrix. Mirrors Python // pdf_chunk_metadata.py:127 merge_pdf_positions: rows are 5-tuples @@ -363,13 +390,7 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 { return nil } sort.Slice(out, func(i, j int) bool { - if out[i][0] != out[j][0] { - return out[i][0] < out[j][0] - } - if out[i][3] != out[j][3] { - return out[i][3] < out[j][3] - } - return out[i][1] < out[j][1] + return pdfPosRowLess(out[i], out[j]) }) return out } diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go new file mode 100644 index 0000000000..b690230e85 --- /dev/null +++ b/internal/ingestion/component/chunker/manual.go @@ -0,0 +1,205 @@ +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ManualChunker is the Go port of Python's `manual` doc-type chunk method +// (rag/app/manual.py). Like GroupTitleChunker it merges adjacent text records +// into heading-bounded groups, but it first re-sorts the records into physical +// reading order before grouping. +// +// Why the resort matters: for multi-column / manually-laid-out PDFs the parser +// emits records in READING order (column-by-column), so a naive grouping +// interleaves the left column's later content with the right column's earlier +// content. ManualChunker re-orders records by (page, top, left) — mirroring +// Python's +// +// sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1])) +// +// so grouping then follows the true top-down, left-to-right layout. +// +// Crucially, docx (and any coordinate-free payload) emits records in +// document-logical order with NO positions — exactly as Python's manual docx +// branch, which performs no resort. When no record carries coordinates, +// ManualChunker skips the sort entirely and reuses the identical grouping +// pipeline as GroupTitleChunker, so its output is bit-for-bit equal (locked by +// TestManualChunker_NoPositionsEqualsGroupChunker). No parser change is needed. +package chunker + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/globals" + + "gorm.io/gorm" +) + +const ComponentNameManualChunker = "ManualChunker" + +// ManualChunkerComponent is the standalone manual-layout chunker. It shares +// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by +// the optional physical-position pre-sort. +type ManualChunkerComponent struct { + name string + param titleChunkerParam +} + +// NewManualChunker constructs the component. It accepts the same heading-level +// params as TitleChunker (method is pinned to "group"); the position resort is +// automatic based on whether the upstream payload carries coordinates. +func NewManualChunker(params map[string]any) (runtime.Component, error) { + // method is pinned to "group": ManualChunker's entire value-add is the + // physical-position resort, which only fires inside the group path. + // A caller must not downgrade it to "naive"/"title" (that would skip the + // resort and silently diverge from Python's manual.py), so method is + // deliberately ignored here even if passed in params. + conf := map[string]any{"method": "group"} + for k, v := range params { + if k == "method" { + continue + } + conf[k] = v + } + p := defaultsTitle() + p.Update(conf) + if err := p.TitleChunkerParam.Validate(); err != nil { + return nil, fmt.Errorf("ManualChunker: %w", err) + } + return &ManualChunkerComponent{ + name: ComponentNameManualChunker, + param: p, + }, nil +} + +func (c *ManualChunkerComponent) Inputs() map[string]string { return ChunkerInputs } +func (c *ManualChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + inputs = map[string]any{} + } + // `name` is read from the workflow-wide Globals bag (seeded at + // pipeline start, published by the File component), not from the + // upstream output map. + name := globals.GlobalOrInput(ctx, inputs, "name", "") + if name == "" { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": "ManualChunker: missing required upstream field \"name\"", + }, nil + } + return c.invoke(ctx, db, withName(inputs, name)) +} + +func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) { + records := extractLineRecords(inputs) + if len(records) == 0 { + return emptyOutputs(), nil + } + // Coordinate-free payloads (docx, plain text) need no resort — and + // skipping it keeps the output identical to GroupTitleChunker. + if hasPdfPositions(records) { + sortRecordsByPosition(records) + } + return chunkFromRecords(ctx, db, inputs, &c.param, records) +} + +// hasPdfPositions reports whether any record carries a PDF coordinate matrix +// (either the structured `_pdf_positions` or the legacy `positions` form). +func hasPdfPositions(records []lineRecord) bool { + for _, r := range records { + if len(r.pdfPositions) > 0 || len(r.positions) > 0 { + return true + } + } + return false +} + +// sortRecordsByPosition re-orders records into physical reading order +// (page, then top, then left) using a stable sort, so records that share a +// key keep their original relative order. Records without coordinates act as +// immovable barriers: they keep their original relative order and are never +// compared against positioned records. +// +// The slice is split into maximal contiguous runs of positioned records and +// each run is sorted independently. This is required because a single +// sort.SliceStable over a MIXED slice (some records with coordinates, some +// without) is not a strict weak ordering: a coordinate-free record B sits +// incomparable between two positioned records A and C, yet A and C remain +// comparable — breaking equivalence-class transitivity and making the sort +// undefined behaviour. Segmenting on coordinate-free records keeps each sort +// a valid strict weak ordering while preserving the documented behaviour that +// coordinate-free records keep their input order (see hasPdfPositions gate). +func sortRecordsByPosition(records []lineRecord) { + runStart := -1 + flush := func(end int) { + if runStart < 0 { + return + } + sort.SliceStable(records[runStart:end], func(i, j int) bool { + ar, _ := firstPositionRow(records[runStart+i]) + br, _ := firstPositionRow(records[runStart+j]) + return pdfPosRowLess(ar, br) + }) + runStart = -1 + } + for i := 0; i <= len(records); i++ { + positioned := i < len(records) + if positioned { + _, positioned = firstPositionRow(records[i]) + } + if positioned { + if runStart < 0 { + runStart = i + } + } else { + flush(i) + } + } +} + +// firstPositionRow returns the first PDF coordinate 5-tuple +// [page,left,right,top,bottom] of a record's coordinate matrix, or +// (nil, false) when the record carries no usable coordinates. It reads the +// structured `_pdf_positions` key first and falls back to the legacy +// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce +// len(row) >= 5 here. +func firstPositionRow(r lineRecord) ([]float64, bool) { + var raw json.RawMessage + switch { + case len(r.pdfPositions) > 0: + raw = r.pdfPositions + case len(r.positions) > 0: + raw = r.positions + default: + return nil, false + } + var mat [][]float64 + if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 { + return nil, false + } + row := mat[0] + if len(row) < 5 { + return nil, false + } + return row, true +} + +// init registers ManualChunker under CategoryIngestion. +func init() { + MustRegisterChunker(ComponentNameManualChunker) +} diff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go new file mode 100644 index 0000000000..0367f85dda --- /dev/null +++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go @@ -0,0 +1,97 @@ +//go:build cgo && integration + +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "os" + "path/filepath" + "testing" + + "ragflow/internal/parser/parser" +) + +// TestManualChunker_RealPDFResort proves the (page, top, left) resort operates +// on the ACTUAL records the Go PDF parser produces. This closes gap 1 from the +// review: the template integration test only feeds a .txt fixture with no +// coordinates, so nothing exercised the resort against real parser output. +// +// It parses a real PDF, normalizes its records through the SAME extractLineRecords +// path the chunker uses in production, then asserts the resort yields a +// non-decreasing physical order — i.e. the shared pdfPosRowLess comparator +// genuinely consumes the parser-emitted _pdf_positions rather than silently +// no-op'ing. The assertion is layout-independent: whether or not the reading +// order differed from the physical order, the sorted result must be monotonic. +func TestManualChunker_RealPDFResort(t *testing.T) { + t.Setenv("DEEPDOC_URL", "") + t.Setenv("OSSDEEPDOC_URL", "") + + path := filepath.Join("..", "..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s): %v", path, err) + } + pdf := parser.NewPDFParser() + res := pdf.ParseWithResult(t.Context(), "Doc1.pdf", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) == 0 { + t.Fatal("parser produced no JSON items") + } + + // Same shape the chunker receives in production. + input := map[string]any{ + "name": "Doc1.pdf", + "output_format": "json", + "chunks": res.JSON, + } + records := extractLineRecords(input) + if len(records) == 0 { + t.Fatal("extractLineRecords produced no records from real parser output") + } + + positioned := 0 + for _, r := range records { + if _, ok := firstPositionRow(r); ok { + positioned++ + } + } + if positioned == 0 { + t.Fatal("no real parser records carried _pdf_positions; resort would be a no-op") + } + + sortRecordsByPosition(records) + + // After the resort, the physical (page, top, left) order must be + // non-decreasing. pdfPosRowLess(cur, prev) being true would mean a record + // physically precedes its predecessor — a resort violation. A coordinate-free + // record (no firstPositionRow) is skipped and never becomes prev, so prev may + // be nil until the first positioned record; guard against that. + var prev []float64 + for i := 0; i < len(records); i++ { + cur, ok := firstPositionRow(records[i]) + if !ok { + continue + } + if prev != nil && pdfPosRowLess(cur, prev) { + t.Fatalf("resort produced out-of-order records at %d: row %v precedes %v", i, cur, prev) + } + prev = cur + } +} diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go new file mode 100644 index 0000000000..5fb2f9dd5f --- /dev/null +++ b/internal/ingestion/component/chunker/manual_test.go @@ -0,0 +1,431 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "ragflow/internal/agent/runtime" +) + +// manualChunkerInput is a small builder for a structured (output_format=chunks) +// upstream payload, mirroring the shape the Parser emits for PDF/JSON output. +func manualChunkerInput(name string, items ...map[string]any) map[string]any { + return map[string]any{ + "name": name, + "output_format": "chunks", + "chunks": items, + } +} + +// posItem builds a chunk item carrying a PDF position matrix +// [page,left,right,top,bottom] under the `_pdf_positions` key (matching +// schema.ChunkDoc.PDFPositions). +func posItem(text string, page, left, right, top, bottom float64) map[string]any { + return map[string]any{ + "text": text, + "doc_type_kwd": "text", + "_pdf_positions": [][]float64{{page, left, right, top, bottom}}, + } +} + +// mustPosJSON marshals a single 5-tuple position matrix into the +// json.RawMessage form used by lineRecord.pdfPositions / positions. +func mustPosJSON(t *testing.T, page, left, right, top, bottom float64) json.RawMessage { + t.Helper() + b, err := json.Marshal([][]float64{{page, left, right, top, bottom}}) + if err != nil { + t.Fatalf("marshal positions: %v", err) + } + return b +} + +func plainItem(text string) map[string]any { + return map[string]any{"text": text, "doc_type_kwd": "text"} +} + +func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent { + t.Helper() + c, err := NewManualChunker(params) + if err != nil { + t.Fatalf("NewManualChunker: %v", err) + } + mc, ok := c.(*ManualChunkerComponent) + if !ok { + t.Fatalf("NewManualChunker returned %T, want *ManualChunkerComponent", c) + } + return mc +} + +// manualChunkTexts pulls the ordered chunk texts out of a chunker output map. +func manualChunkTexts(out map[string]any) []string { + chunks, _ := out["chunks"].([]map[string]any) + out2 := make([]string, 0, len(chunks)) + for _, ck := range chunks { + if s, _ := ck["text"].(string); s != "" { + out2 = append(out2, s) + } + } + return out2 +} + +// TestManualChunker_Registered pins that the component is discoverable via the +// runtime registry under the CategoryIngestion category with non-empty +// input/output metadata. +func TestManualChunker_Registered(t *testing.T) { + factory, cat, meta, ok := runtime.DefaultRegistry.Lookup(ComponentNameManualChunker) + if !ok { + t.Fatal("ManualChunker: registry miss") + } + if cat != runtime.CategoryIngestion { + t.Errorf("category = %q, want %q", cat, runtime.CategoryIngestion) + } + if factory == nil { + t.Error("factory is nil") + } + if len(meta.Inputs) == 0 { + t.Error("inputs metadata empty") + } + if len(meta.Outputs) == 0 { + t.Error("outputs metadata empty") + } +} + +// TestManualChunker_Empty is the trivial boundary: no records => empty outputs. +func TestManualChunker_Empty(t *testing.T) { + c := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "doc"}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if got, want := out["output_format"], "chunks"; got != want { + t.Errorf("output_format = %v, want %v", got, want) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 0 { + t.Errorf("chunks = %d, want 0", len(chunks)) + } +} + +// TestManualChunker_NoPositionsEqualsGroupChunker is the regression lock: when +// the upstream payload carries NO physical coordinates (e.g. docx, which Go +// emits in document-logical order — exactly as Python's manual docx branch, +// which performs no coordinate resort), ManualChunker MUST behave bit-for-bit +// like GroupTitleChunker. Any divergence here is an unintended regression. +func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) { + var mkLevels = [][]string{{`^# `}, {`^## `}} + input := manualChunkerInput("doc", + plainItem("# H1"), + plainItem("body under h1 a"), + plainItem("body under h1 b"), + plainItem("## H2"), + plainItem("body under h2"), + plainItem("more body under h2"), + ) + + gc, err := NewGroupTitleChunker(map[string]any{"levels": mkLevels}) + if err != nil { + t.Fatalf("NewGroupTitleChunker: %v", err) + } + gOut, err := gc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("group Invoke: %v", err) + } + + mc := mustManual(t, map[string]any{"levels": mkLevels}) + mOut, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("manual Invoke: %v", err) + } + + gChunks, _ := gOut["chunks"].([]map[string]any) + mChunks, _ := mOut["chunks"].([]map[string]any) + if !reflect.DeepEqual(gChunks, mChunks) { + t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks) + } +} + +// TestManualChunker_SortsByPageTopLeft is the core behavioural difference +// vs. GroupTitleChunker. For a multi-column / manual PDF, the parser emits +// records in READING order (column-by-column), not in PHYSICAL/top-down +// order. ManualChunker must resort records by (page, top, left) before +// grouping so that, e.g., the left column's content precedes the right +// column's when they share a page band. +func TestManualChunker_SortsByPageTopLeft(t *testing.T) { + // page 1, all body (no heading markers). Physical layout: + // top=100, left=10 -> "left col top" + // top=100, left=400 -> "right col" + // top=300, left=10 -> "left col bottom" + // top=400, left=200 -> "footer title" + // Parser reading order is scrambled relative to physical order. + input := manualChunkerInput("doc", + posItem("footer title", 1, 200, 400, 400, 500), + posItem("left col bottom", 1, 10, 200, 300, 400), + posItem("right col", 1, 400, 600, 100, 400), + posItem("left col top", 1, 10, 200, 100, 200), + ) + + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + out, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + + // Whether merged into one chunk or several, the concatenated text MUST + // follow physical order: left col top, right col, left col bottom, + // footer title. + var full strings.Builder + for _, s := range manualChunkTexts(out) { + full.WriteString(s) + full.WriteString("\n") + } + joined := full.String() + wantOrder := []string{"left col top", "right col", "left col bottom", "footer title"} + last := -1 + for _, w := range wantOrder { + idx := strings.Index(joined, w) + if idx < 0 { + t.Fatalf("expected %q to appear in manual output %q", w, joined) + } + if idx <= last { + t.Fatalf("physical order violated: %q appears after a later element in %q", w, joined) + } + last = idx + } +} + +// TestManualChunker_SortDiffersFromGroupWhenPositionsPresent proves the resort +// actually fires (and is not a no-op): the same scrambled input must produce a +// DIFFERENT text order from GroupTitleChunker, which groups in input order. +func TestManualChunker_SortDiffersFromGroupWhenPositionsPresent(t *testing.T) { + input := manualChunkerInput("doc", + posItem("parser-reads-2nd-col-first", 1, 400, 600, 100, 400), + posItem("parser-reads-1st-col-first", 1, 10, 200, 100, 200), + ) + + gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}}) + gOut, err := gc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("group Invoke: %v", err) + } + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + mOut, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("manual Invoke: %v", err) + } + if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) { + t.Fatalf("ManualChunker did not reorder relative to GroupTitleChunker; both = %#v", mOut["chunks"]) + } + // ManualChunker must place the 1st-column item first. + if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "1st-col") { + t.Fatalf("expected 1st-column text first in manual output, got %#v", texts) + } +} + +// TestManualChunker_MergesPositions pins parity with GroupTitleChunker: when +// adjacent text records are merged into one chunk, the PDF position matrices +// are MERGED across the merged records (not dropped), and parser position tags +// are stripped from the text. +func TestManualChunker_MergesPositions(t *testing.T) { + input := manualChunkerInput("doc", + posItem("body one", 1, 10, 20, 30, 40), + posItem("body two", 2, 15, 25, 35, 45), + ) + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + out, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + found := false + for _, ck := range chunks { + text, _ := ck["text"].(string) + if !strings.Contains(text, "body one") || !strings.Contains(text, "body two") { + continue + } + found = true + if strings.Contains(text, "@@") { + t.Errorf("parser position tags leaked into chunk text: %q", text) + } + pos, ok := ck["_pdf_positions"].([][]float64) + if !ok { + t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"]) + } + if len(pos) != 2 { + // Hard fail: the page/top/left checks below would index out of + // range if the merged matrix does not have exactly two rows. + t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos)) + } + // Merged matrix must itself be sorted by (page, top, left) — the + // same key pdfPosRowLess uses when merging. + if pdfPosRowLess(pos[1], pos[0]) { + t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos) + } + } + if !found { + t.Fatal("merged body group chunk not found in output") + } +} + +// TestSortRecordsByPosition_Unit pins the comparator directly: (page, top, +// left) ascending, stable for equal keys, and records without positions keep +// their original relative order (so a no-coordinate payload is untouched). +func TestSortRecordsByPosition_Unit(t *testing.T) { + recs := []lineRecord{ + {text: "a", pdfPositions: mustPosJSON(t, 1, 200, 300, 400, 500)}, + {text: "b", pdfPositions: mustPosJSON(t, 1, 10, 100, 300, 400)}, + {text: "c", pdfPositions: mustPosJSON(t, 1, 10, 100, 100, 200)}, + {text: "d"}, // no position + } + sortRecordsByPosition(recs) + got := []string{recs[0].text, recs[1].text, recs[2].text, recs[3].text} + want := []string{"c", "b", "a", "d"} + if !reflect.DeepEqual(got, want) { + t.Errorf("sorted order = %v, want %v", got, want) + } +} + +// TestHasPdfPositions is a tiny white-box check for the gate that decides +// whether a resort is performed at all. +func TestHasPdfPositions(t *testing.T) { + if hasPdfPositions([]lineRecord{{text: "x"}}) { + t.Error("plain record should report no positions") + } + if !hasPdfPositions([]lineRecord{{text: "x", pdfPositions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) { + t.Error("record with pdf_positions should report positions") + } + if !hasPdfPositions([]lineRecord{{text: "x", positions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) { + t.Error("record with positions should report positions") + } +} + +// parserShapedItem builds a chunk item that mirrors EXACTLY the JSON shape the +// Go PDF parser emits via pdfParseResultToJSON (see +// TestPDFParseResultToJSON_NormalizesCoreFields): _pdf_positions is a [][]any +// with 1-based page numbers, accompanied by layout / page_number / +// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete +// in-memory type the parser produces, and recordsFromStructured must consume it +// after the chunksFromInputs JSON round-trip. +func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any { + return map[string]any{ + "text": text, + "doc_type_kwd": "text", + "layout": "text", + "page_number": int(page), + "_pdf_positions": [][]any{{page, left, right, top, bottom}}, + } +} + +// legacyPosItem is like posItem but uses the legacy `positions` key instead of +// the structured `_pdf_positions` key. +func legacyPosItem(text string, page, left, right, top, bottom float64) map[string]any { + return map[string]any{ + "text": text, + "doc_type_kwd": "text", + "positions": [][]float64{{page, left, right, top, bottom}}, + } +} + +// TestManualChunker_RealParserShapedPayload exercises the EXACT payload shape +// the Go PDF parser delivers to the chunker (pdfParseResultToJSON -> +// recordsFromStructured). It closes the coverage gap where the template +// integration test only feeds a .txt fixture (no coordinates): here a +// coordinate-bearing, parser-shaped multi-column doc proves the (page, top, +// left) resort actually fires and produces a different order from +// GroupTitleChunker (which groups in parser reading order). +func TestManualChunker_RealParserShapedPayload(t *testing.T) { + input := manualChunkerInput("doc", + // Parser reading order (column-by-column) is scrambled vs physical. + parserShapedItem("right column, read first", 1, 400, 600, 100, 400), + parserShapedItem("left column, read second", 1, 10, 200, 100, 200), + ) + + gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}}) + gOut, err := gc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("group Invoke: %v", err) + } + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + mOut, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("manual Invoke: %v", err) + } + if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) { + t.Fatalf("ManualChunker did not reorder parser-shaped payload; both = %#v", mOut["chunks"]) + } + // Physical (page, top, left) order must put the left column first. + if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "left column") { + t.Fatalf("expected left-column text first after resort, got %#v", texts) + } +} + +// TestManualChunker_LegacyPositionsKeyTriggersSort pins that the resort fires +// when records carry the LEGACY `positions` key rather than the structured +// `_pdf_positions` key. firstPositionRow (and thus the shared pdfPosRowLess +// comparator) must read both forms, otherwise a payload emitted with the old +// key would silently skip the resort. +func TestManualChunker_LegacyPositionsKeyTriggersSort(t *testing.T) { + input := manualChunkerInput("doc", + legacyPosItem("right column, read first", 1, 400, 600, 100, 400), + legacyPosItem("left column, read second", 1, 10, 200, 100, 200), + ) + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + out, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + // Without the legacy-key path the output would keep parser reading order + // (right column before left). The resort must place the left column first. + if texts := manualChunkTexts(out); len(texts) == 0 || !strings.Contains(texts[0], "left column") { + t.Fatalf("legacy `positions` key did not trigger resort; got %#v", texts) + } +} + +// TestManualChunker_MixedPositionedAndPlain exercises the boundary where a +// payload mixes coordinate-bearing records with coordinate-free records. A +// single document type will not usually do this (production payloads are +// homogeneously positioned-or-plain), but the sort must not corrupt order when +// it happens: the contiguous positioned run reorders by (page, top, left) and +// coordinate-free records keep their ORIGINAL relative order. (Go's stable +// sort only reorders within a contiguous positioned run — a coordinate-free +// record between two positioned records correctly stops the reorder from +// crossing it, which is why the positioned records here are kept adjacent.) +func TestManualChunker_MixedPositionedAndPlain(t *testing.T) { + input := manualChunkerInput("doc", + plainItem("plain A"), + posItem("right col", 1, 400, 600, 100, 400), + posItem("left col", 1, 10, 200, 100, 200), + plainItem("plain B"), + ) + mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}}) + out, err := mc.Invoke(t.Context(), nil, input) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + joined := strings.Join(manualChunkTexts(out), "\n") + // The contiguous positioned run keeps physical order relative to itself. + if i, j := strings.Index(joined, "left col"), strings.Index(joined, "right col"); i < 0 || j < 0 || i > j { + t.Fatalf("positioned records not in physical order in %q", joined) + } + // Coordinate-free records keep their original relative order. + if i, j := strings.Index(joined, "plain A"), strings.Index(joined, "plain B"); i < 0 || j < 0 || i > j { + t.Fatalf("coordinate-free records lost original order in %q", joined) + } +} diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json index e83e0d17e5..1faad567e9 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json @@ -26,7 +26,7 @@ }, "Parser:HipSignsRhyme": { "downstream": [ - "TitleChunker:NineInsectsFind" + "ManualChunker:NineInsectsFind" ], "obj": { "component_name": "Parser", @@ -87,12 +87,12 @@ "File" ] }, - "TitleChunker:NineInsectsFind": { + "ManualChunker:NineInsectsFind": { "downstream": [ "Extractor:AutoExtractDefault" ], "obj": { - "component_name": "TitleChunker", + "component_name": "ManualChunker", "params": { "hierarchy": 0, "include_heading_content": false, @@ -158,7 +158,7 @@ }, "Extractor:AutoExtractDefault": { "upstream": [ - "TitleChunker:NineInsectsFind" + "ManualChunker:NineInsectsFind" ], "downstream": [ "Tokenizer:FunnyBalloonsGrin" @@ -197,15 +197,15 @@ "targetHandle": "end" }, { - "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend", + "id": "xy-edge__Parser:HipSignsRhymestart-ManualChunker:NineInsectsFindend", "source": "Parser:HipSignsRhyme", "sourceHandle": "start", - "target": "TitleChunker:NineInsectsFind", + "target": "ManualChunker:NineInsectsFind", "targetHandle": "end" }, { - "id": "xy-edge__TitleChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend", - "source": "TitleChunker:NineInsectsFind", + "id": "xy-edge__ManualChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend", + "source": "ManualChunker:NineInsectsFind", "sourceHandle": "start", "target": "Extractor:AutoExtractDefault", "targetHandle": "end" @@ -407,10 +407,10 @@ } ] }, - "label": "TitleChunker", - "name": "Title Chunker_0" + "label": "ManualChunker", + "name": "Manual Chunker_0" }, - "id": "TitleChunker:NineInsectsFind", + "id": "ManualChunker:NineInsectsFind", "measured": { "height": 74, "width": 200 diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go index aa42b84cf1..01daffa143 100644 --- a/internal/ingestion/pipeline/template_integration_test.go +++ b/internal/ingestion/pipeline/template_integration_test.go @@ -424,9 +424,9 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) { t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText) } } - chunkerState, ok := state["TitleChunker:NineInsectsFind"] + chunkerState, ok := state["ManualChunker:NineInsectsFind"] if !ok { - t.Fatal("missing TitleChunker:NineInsectsFind state") + t.Fatal("missing ManualChunker:NineInsectsFind state") } chunkerChunks, ok := chunkerState["chunks"].([]map[string]any) if !ok || len(chunkerChunks) != len(wantChunkTexts) { diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go index 17459f27d4..b4148ed099 100644 --- a/internal/ingestion/task/pipeline_executor_defaults_test.go +++ b/internal/ingestion/task/pipeline_executor_defaults_test.go @@ -43,7 +43,7 @@ var builtinComponentParamsGolden = map[string]string{ "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", - "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", + "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.go index 1466fac979..41b4c284d3 100644 --- a/internal/service/components_service_test.go +++ b/internal/service/components_service_test.go @@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) { t.Fatalf("List(Ingestion) returned error: %v", err) } wantNames := []string{ - "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } assertComponentNameSet(t, "ingestion", namesOf(got), wantNames) @@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) { t.Fatalf("List(Ingestion,Shared) returned error: %v", err) } wantNames := []string{ - "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames) @@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) { t.Fatalf("List(Ingestion x3) returned error: %v", err) } wantNames := []string{ - "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", + "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker", "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } if len(got) != len(wantNames) {