diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index 7a07252b97..69fbfa5b97 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -178,8 +178,6 @@ func emptyOutputs() map[string]any { } } -func emptyChunkDocs() []schema.ChunkDoc { return []schema.ChunkDoc{} } - // chunkOutputs builds the canonical chunker output (output_format="chunks" + // chunks). The Go runtime passes only this explicit output to the next node, // so the run-level metadata that downstream components still need (e.g. diff --git a/internal/ingestion/component/chunker/image.go b/internal/ingestion/component/chunker/image.go deleted file mode 100644 index e00b628697..0000000000 --- a/internal/ingestion/component/chunker/image.go +++ /dev/null @@ -1,109 +0,0 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package chunker - -import ( - "bytes" - "image" - "image/draw" - _ "image/gif" - _ "image/jpeg" - "image/png" -) - -// concatImg vertically concatenates two images represented as raw bytes. -// Mirrors Python's rag/nlp.concat_img: -// -// - img1 is img2 (same slice pointer) → returns img1 unchanged. -// - one side is nil → returns the non-nil side. -// - both nil → returns nil. -// - both non-nil → decodes both, creates a new RGB image of -// max(width1, width2) × (height1 + height2), pastes img1 at the top -// and img2 below it, then re-encodes as PNG. -// -// Unlike Python's version this function works with raw image bytes rather -// than PIL Image / LazyImage objects. The LazyImage blob-list merge -// (LazyImage.merge) is not needed here because Go chunker items carry -// decoded image bytes rather than deferred-load blobs. -func concatImg(img1, img2 []byte) []byte { - // Same-reference guard (Python: img1 is img2). - if len(img1) > 0 && len(img2) > 0 && &img1[0] == &img2[0] { - return img1 - } - // Nil / empty guard (Python: img1 and not img2 → img1, etc.). - if len(img1) == 0 && len(img2) == 0 { - return nil - } - if len(img1) == 0 { - return img2 - } - if len(img2) == 0 { - return img1 - } - - // Decode both images. - dec1, _, err1 := image.Decode(bytes.NewReader(img1)) - dec2, _, err2 := image.Decode(bytes.NewReader(img2)) - if err1 != nil { - if err2 != nil { - return nil - } - return img2 - } - if err2 != nil { - return img1 - } - - // Pixel-data equality guard (Python: img1.tobytes() == img2.tobytes()). - if img1data, img2data := imgBytes(dec1), imgBytes(dec2); img1data != nil && img2data != nil && bytes.Equal(img1data, img2data) { - return img1 - } - - // Compute dimensions. - bounds1 := dec1.Bounds() - bounds2 := dec2.Bounds() - w1, h1 := bounds1.Dx(), bounds1.Dy() - w2, h2 := bounds2.Dx(), bounds2.Dy() - - newW := w1 - if w2 > newW { - newW = w2 - } - newH := h1 + h2 - - // Create new RGBA image and paste img1 at top, img2 below. - dst := image.NewRGBA(image.Rect(0, 0, newW, newH)) - draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) - draw.Draw(dst, image.Rect(0, 0, w1, h1), dec1, bounds1.Min, draw.Over) - draw.Draw(dst, image.Rect(0, h1, w2, h1+h2), dec2, bounds2.Min, draw.Over) - - var buf bytes.Buffer - if err := png.Encode(&buf, dst); err != nil { - return nil - } - return buf.Bytes() -} - -// imgBytes returns the raw RGBA pixel data of img as a []byte. Returns nil -// when img is not convertible (should not happen for formats decoded by -// the Go image library, which always produce RGBA or NRGBA). -func imgBytes(img image.Image) []byte { - b := img.Bounds() - rgba := image.NewRGBA(b) - draw.Draw(rgba, b, img, b.Min, draw.Src) - return rgba.Pix -} diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go index 3c28a24223..3c26c1f258 100644 --- a/internal/ingestion/component/chunker/register.go +++ b/internal/ingestion/component/chunker/register.go @@ -14,10 +14,10 @@ // limitations under the License. // -// Package chunker holds the ingestion chunker components: TokenChunker, -// TitleChunker, GroupTitleChunker, HierarchyTitleChunker. The four -// components share the same upstream payload (schema.ChunkerFromUpstream) -// and the same output shape (schema.ChunkerOutputs). +// Package chunker holds the ingestion chunker components. The variants share +// the same upstream payload (schema.ChunkerFromUpstream) and the same output +// shape (schema.ChunkerOutputs); each registers via MustRegisterChunker from +// its own file. // // The package is intentionally separate from internal/agent/component/ // (the agent canvas) and from internal/ingestion/component/schema/ @@ -35,7 +35,7 @@ import ( ) // MustRegisterChunker registers a single chunker component under -// CategoryIngestion. The four chunker files each carry exactly one +// CategoryIngestion. Each chunker file carries exactly one // init() that calls this with the registered component's name; the // factory body resolves the typed constructor via newChunkerByName // (in common.go). @@ -100,23 +100,26 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m } // ChunkerInputs is the static, registered input descriptor shared -// by all four chunker variants. +// by all chunker variants. var ChunkerInputs = map[string]string{ "text": "Plain-text input. The chunker slices this into downstream chunks.", "content": "Alias for \"text\".", "chunks": "Optional upstream chunk list (structured JSON form).", - "name": "Source document name. Required by the upstream payload convention.", + "name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.", "_created_time": "Optional upstream timestamp (RFC3339Nano, s).", "_elapsed_time": "Optional upstream elapsed time (s).", } // ChunkerOutputs is the static, registered output descriptor shared -// by all four chunker variants. +// by all chunker variants. +// +// Note: this map is the component's emitted output only. Run-level metadata +// that downstream components still need — name, tenant_id, kb_id — is NOT +// re-emitted here; it lives in the workflow-wide CanvasState.Globals bag and +// is read directly from ctx (see runtime.CanvasState.Globals and +// globals.GlobalOrInput). Do not add those keys here. var ChunkerOutputs = map[string]string{ "output_format": "Always \"chunks\" on success.", "chunks": "list[object]: per-chunk map (text + optional meta keys).", - "name": "Source document name, carried forward from upstream (pass-through) when present — Tokenizer consumes it for title embedding.", - "tenant_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.", - "kb_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.", "_ERROR": "Set only on validation failure.", } diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.go new file mode 100644 index 0000000000..5b55ddf326 --- /dev/null +++ b/internal/ingestion/component/chunker/token_delim_contract_test.go @@ -0,0 +1,102 @@ +package chunker + +import ( + "context" + "strings" + "testing" +) + +// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick) +// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting +// delimiter_mode + a bare delimiter is effectively a no-op — the text is +// merged by token_size and the bare token survives inside a chunk rather +// than acting as a split point. Regression guard for the "bare entries are +// ignored" contract. +func TestTokenChunker_BareDelimiterIgnored(t *testing.T) { + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{"::"}, + "chunk_token_size": float64(8), + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + const text = "alpha::beta::gamma::delta" + out, err := c.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": text, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) == 0 { + t.Fatalf("no chunks produced") + } + var joined strings.Builder + for _, ck := range chunks { + joined.WriteString(ck["text"].(string)) + } + // No content dropped and the bare "::" is preserved (just chunked by + // token size, not split on "::"). + if joined.String() != text { + t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks)) + } +} + +// TestTokenChunker_MultiByteBacktickDelimiter locks T2: a backtick-wrapped +// multi-byte (CJK) delimiter contributes its INNER content as the split +// pattern, and the longest delimiter wins over a shorter prefix of it +// (rune-descending sort). Both ensure the chunker-level delimiter handling +// is correct for non-ASCII delimiters. +func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) { + // Single multi-byte delimiter splits on its inner content. + c, err := NewTokenChunker(map[string]any{ + "delimiters": []string{"`段落`"}, + "chunk_token_size": float64(1024), + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + out, err := c.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "第一部分段落第二部分", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + want := []string{"第一部分", "第二部分"} + if len(chunks) != len(want) { + t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks)) + } + for i, w := range want { + if got := chunks[i]["text"].(string); got != w { + t.Errorf("chunk[%d] text: want %q got %q", i, w, got) + } + } + + // Longest delimiter must win over its shorter prefix: `段落` (2 runes) + // beats `段` (1 rune), so "A段落B" splits on "段落", not "段". + c2, err := NewTokenChunker(map[string]any{ + "delimiters": []string{"`段落`", "`段`"}, + "chunk_token_size": float64(1024), + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + out2, err := c2.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "A段落B", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks2, _ := out2["chunks"].([]map[string]any) + if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" { + t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2)) + } +}