refactor[Go]: remove chunker dead code & fix stale/false descriptors (L1/L2/L3) (#17960)

internal/ingestion/component/chunker/common.go

@@ -178,8 +178,6 @@ func emptyOutputs() map[string]any {
 	}
 }
 
-func emptyChunkDocs() []schema.ChunkDoc { return []schema.ChunkDoc{} }
-
 // chunkOutputs builds the canonical chunker output (output_format="chunks" +
 // chunks). The Go runtime passes only this explicit output to the next node,
 // so the run-level metadata that downstream components still need (e.g.

internal/ingestion/component/chunker/image.go

@@ -1,109 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-//
-
-package chunker
-
-import (
-	"bytes"
-	"image"
-	"image/draw"
-	_ "image/gif"
-	_ "image/jpeg"
-	"image/png"
-)
-
-// concatImg vertically concatenates two images represented as raw bytes.
-// Mirrors Python's rag/nlp.concat_img:
-//
-//   - img1 is img2 (same slice pointer) → returns img1 unchanged.
-//   - one side is nil → returns the non-nil side.
-//   - both nil → returns nil.
-//   - both non-nil → decodes both, creates a new RGB image of
-//     max(width1, width2) × (height1 + height2), pastes img1 at the top
-//     and img2 below it, then re-encodes as PNG.
-//
-// Unlike Python's version this function works with raw image bytes rather
-// than PIL Image / LazyImage objects. The LazyImage blob-list merge
-// (LazyImage.merge) is not needed here because Go chunker items carry
-// decoded image bytes rather than deferred-load blobs.
-func concatImg(img1, img2 []byte) []byte {
-	// Same-reference guard (Python: img1 is img2).
-	if len(img1) > 0 && len(img2) > 0 && &img1[0] == &img2[0] {
-		return img1
-	}
-	// Nil / empty guard (Python: img1 and not img2 → img1, etc.).
-	if len(img1) == 0 && len(img2) == 0 {
-		return nil
-	}
-	if len(img1) == 0 {
-		return img2
-	}
-	if len(img2) == 0 {
-		return img1
-	}
-
-	// Decode both images.
-	dec1, _, err1 := image.Decode(bytes.NewReader(img1))
-	dec2, _, err2 := image.Decode(bytes.NewReader(img2))
-	if err1 != nil {
-		if err2 != nil {
-			return nil
-		}
-		return img2
-	}
-	if err2 != nil {
-		return img1
-	}
-
-	// Pixel-data equality guard (Python: img1.tobytes() == img2.tobytes()).
-	if img1data, img2data := imgBytes(dec1), imgBytes(dec2); img1data != nil && img2data != nil && bytes.Equal(img1data, img2data) {
-		return img1
-	}
-
-	// Compute dimensions.
-	bounds1 := dec1.Bounds()
-	bounds2 := dec2.Bounds()
-	w1, h1 := bounds1.Dx(), bounds1.Dy()
-	w2, h2 := bounds2.Dx(), bounds2.Dy()
-
-	newW := w1
-	if w2 > newW {
-		newW = w2
-	}
-	newH := h1 + h2
-
-	// Create new RGBA image and paste img1 at top, img2 below.
-	dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
-	draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)
-	draw.Draw(dst, image.Rect(0, 0, w1, h1), dec1, bounds1.Min, draw.Over)
-	draw.Draw(dst, image.Rect(0, h1, w2, h1+h2), dec2, bounds2.Min, draw.Over)
-
-	var buf bytes.Buffer
-	if err := png.Encode(&buf, dst); err != nil {
-		return nil
-	}
-	return buf.Bytes()
-}
-
-// imgBytes returns the raw RGBA pixel data of img as a []byte. Returns nil
-// when img is not convertible (should not happen for formats decoded by
-// the Go image library, which always produce RGBA or NRGBA).
-func imgBytes(img image.Image) []byte {
-	b := img.Bounds()
-	rgba := image.NewRGBA(b)
-	draw.Draw(rgba, b, img, b.Min, draw.Src)
-	return rgba.Pix
-}

internal/ingestion/component/chunker/register.go

@@ -14,10 +14,10 @@
 //  limitations under the License.
 //
 
-// Package chunker holds the ingestion chunker components: TokenChunker,
-// TitleChunker, GroupTitleChunker, HierarchyTitleChunker. The four
-// components share the same upstream payload (schema.ChunkerFromUpstream)
-// and the same output shape (schema.ChunkerOutputs).
+// Package chunker holds the ingestion chunker components. The variants share
+// the same upstream payload (schema.ChunkerFromUpstream) and the same output
+// shape (schema.ChunkerOutputs); each registers via MustRegisterChunker from
+// its own file.
 //
 // The package is intentionally separate from internal/agent/component/
 // (the agent canvas) and from internal/ingestion/component/schema/
@@ -35,7 +35,7 @@ import (
 )
 
 // MustRegisterChunker registers a single chunker component under
-// CategoryIngestion. The four chunker files each carry exactly one
+// CategoryIngestion. Each chunker file carries exactly one
 // init() that calls this with the registered component's name; the
 // factory body resolves the typed constructor via newChunkerByName
 // (in common.go).
@@ -100,23 +100,26 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
 }
 
 // ChunkerInputs is the static, registered input descriptor shared
-// by all four chunker variants.
+// by all chunker variants.
 var ChunkerInputs = map[string]string{
 	"text":          "Plain-text input. The chunker slices this into downstream chunks.",
 	"content":       "Alias for \"text\".",
 	"chunks":        "Optional upstream chunk list (structured JSON form).",
-	"name":          "Source document name. Required by the upstream payload convention.",
+	"name":          "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.",
 	"_created_time": "Optional upstream timestamp (RFC3339Nano, s).",
 	"_elapsed_time": "Optional upstream elapsed time (s).",
 }
 
 // ChunkerOutputs is the static, registered output descriptor shared
-// by all four chunker variants.
+// by all chunker variants.
+//
+// Note: this map is the component's emitted output only. Run-level metadata
+// that downstream components still need — name, tenant_id, kb_id — is NOT
+// re-emitted here; it lives in the workflow-wide CanvasState.Globals bag and
+// is read directly from ctx (see runtime.CanvasState.Globals and
+// globals.GlobalOrInput). Do not add those keys here.
 var ChunkerOutputs = map[string]string{
 	"output_format": "Always \"chunks\" on success.",
 	"chunks":        "list[object]: per-chunk map (text + optional meta keys).",
-	"name":          "Source document name, carried forward from upstream (pass-through) when present — Tokenizer consumes it for title embedding.",
-	"tenant_id":     "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.",
-	"kb_id":         "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.",
 	"_ERROR":        "Set only on validation failure.",
 }

internal/ingestion/component/chunker/token_delim_contract_test.go

@@ -0,0 +1,102 @@
+package chunker
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)
+// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting
+// delimiter_mode + a bare delimiter is effectively a no-op — the text is
+// merged by token_size and the bare token survives inside a chunk rather
+// than acting as a split point. Regression guard for the "bare entries are
+// ignored" contract.
+func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode":   "delimiter",
+		"delimiters":       []string{"::"},
+		"chunk_token_size": float64(8),
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	const text = "alpha::beta::gamma::delta"
+	out, err := c.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          text,
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	if len(chunks) == 0 {
+		t.Fatalf("no chunks produced")
+	}
+	var joined strings.Builder
+	for _, ck := range chunks {
+		joined.WriteString(ck["text"].(string))
+	}
+	// No content dropped and the bare "::" is preserved (just chunked by
+	// token size, not split on "::").
+	if joined.String() != text {
+		t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))
+	}
+}
+
+// TestTokenChunker_MultiByteBacktickDelimiter locks T2: a backtick-wrapped
+// multi-byte (CJK) delimiter contributes its INNER content as the split
+// pattern, and the longest delimiter wins over a shorter prefix of it
+// (rune-descending sort). Both ensure the chunker-level delimiter handling
+// is correct for non-ASCII delimiters.
+func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {
+	// Single multi-byte delimiter splits on its inner content.
+	c, err := NewTokenChunker(map[string]any{
+		"delimiters":       []string{"`段落`"},
+		"chunk_token_size": float64(1024),
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	out, err := c.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          "第一部分段落第二部分",
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	want := []string{"第一部分", "第二部分"}
+	if len(chunks) != len(want) {
+		t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+	}
+	for i, w := range want {
+		if got := chunks[i]["text"].(string); got != w {
+			t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+		}
+	}
+
+	// Longest delimiter must win over its shorter prefix: `段落` (2 runes)
+	// beats `段` (1 rune), so "A段落B" splits on "段落", not "段".
+	c2, err := NewTokenChunker(map[string]any{
+		"delimiters":       []string{"`段落`", "`段`"},
+		"chunk_token_size": float64(1024),
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	out2, err := c2.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          "A段落B",
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks2, _ := out2["chunks"].([]map[string]any)
+	if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" {
+		t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2))
+	}
+}