zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-16 12:47:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: add embedding support (#16769)

Browse Source 
### Summary

Feat: add embedding support
This commit is contained in:
Jack
2026-07-09 16:24:39 +08:00
committed by GitHub
parent 0ba1d37a10
commit 8ac80284c4
38 changed files with 1543 additions and 2395 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/agent/canvas/factory_context.go Normal file
View File

@@ -0,0 +1,41 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package canvas
import (
"context"
"ragflow/internal/agent/runtime"
)
type componentFactoryContextKey struct{}
// WithComponentFactory attaches a per-pipeline component factory override
// to ctx. BuildWorkflow uses it when constructing node bodies so one
// pipeline instance can resolve components independently of the process-wide
// runtime.DefaultFactory.
func WithComponentFactory(ctx context.Context, factory runtime.ComponentFactory) context.Context {
if factory == nil {
return ctx
}
return context.WithValue(ctx, componentFactoryContextKey{}, factory)
}
func componentFactoryFromContext(ctx context.Context) runtime.ComponentFactory {
factory, _ := ctx.Value(componentFactoryContextKey{}).(runtime.ComponentFactory)
return factory
}

2
internal/agent/canvas/loop_subgraph.go
View File

@@ -241,7 +241,7 @@ func buildSubWorkflow(
if name == "" {
return nil, fmt.Errorf("canvas: loop %q member %q has empty component_name", loopID, cpnID)
}
body, err := buildNodeBody(cpnID, name, c.Components[cpnID].Obj.Params)
body, err := buildNodeBody(ctx, cpnID, name, c.Components[cpnID].Obj.Params)
if err != nil {
return nil, err
}

12
internal/agent/canvas/node_body.go
View File

@@ -74,7 +74,7 @@ type nodeBodyFn = func(ctx context.Context, in map[string]any) (map[string]any,
// Outputs bucket. UserFillUpNodeBody tags its output itself so the
// interrupt-driven branch still attributes the resume payload to the
// right cpn.
func buildNodeBody(cpnID, name string, params map[string]any) (nodeBodyFn, error) {
func buildNodeBody(ctx context.Context, cpnID, name string, params map[string]any) (nodeBodyFn, error) {
if isLegacyNoOp(name) {
return legacyNoOpBody(cpnID), nil
}
@@ -89,7 +89,7 @@ func buildNodeBody(cpnID, name string, params map[string]any) (nodeBodyFn, error
if strings.EqualFold(name, "UserFillUp") {
return UserFillUpNodeBody(cpnID, params), nil
}
if factory := runtime.DefaultFactory(); factory != nil {
if factory := resolveComponentFactory(ctx); factory != nil {
comp, err := factory(name, params)
if err != nil {
return nil, fmt.Errorf("canvas: component %q (%s): factory: %w", cpnID, name, err)
@@ -118,6 +118,14 @@ func buildNodeBody(cpnID, name string, params map[string]any) (nodeBodyFn, error
// components (legacyNoOpNames). It echoes the input and tags
// __legacy_noop__ so downstream debuggers can tell the node fired but
// did nothing.
func resolveComponentFactory(ctx context.Context) runtime.ComponentFactory {
if factory := componentFactoryFromContext(ctx); factory != nil {
return factory
}
return runtime.DefaultFactory()
}
func legacyNoOpBody(cpnID string) nodeBodyFn {
return func(_ context.Context, in map[string]any) (map[string]any, error) {
out := make(map[string]any, len(in)+2)

8
internal/agent/canvas/node_body_per_class_timeout_integration_test.go
View File

@@ -112,7 +112,7 @@ func TestBuildNodeBody_PerClassTimeout_ExeSQL_3s(t *testing.T) {
}
})
body, err := buildNodeBody("cpn-exe", "ExeSQL", nil)
body, err := buildNodeBody(context.Background(), "cpn-exe", "ExeSQL", nil)
if err != nil {
t.Fatalf("buildNodeBody: %v", err)
}
@@ -162,7 +162,7 @@ func TestBuildNodeBody_PerClassTimeout_TavilySearch_12s(t *testing.T) {
}
})
body, err := buildNodeBody("cpn-tav", "TavilySearch", nil)
body, err := buildNodeBody(context.Background(), "cpn-tav", "TavilySearch", nil)
if err != nil {
t.Fatalf("buildNodeBody: %v", err)
}
@@ -209,7 +209,7 @@ func TestBuildNodeBody_PerClassTimeout_UnknownClass_UniformFallback(t *testing.T
}
})
body, err := buildNodeBody("cpn-cust", "CustomComponent", nil)
body, err := buildNodeBody(context.Background(), "cpn-cust", "CustomComponent", nil)
if err != nil {
t.Fatalf("buildNodeBody: %v", err)
}
@@ -251,7 +251,7 @@ func TestBuildNodeBody_PerClassTimeout_PerClassEnvOverride(t *testing.T) {
}
})
body, err := buildNodeBody("cpn-exe-ovr", "ExeSQL", nil)
body, err := buildNodeBody(context.Background(), "cpn-exe-ovr", "ExeSQL", nil)
if err != nil {
t.Fatalf("buildNodeBody: %v", err)
}

2
internal/agent/canvas/scheduler.go
View File

@@ -486,7 +486,7 @@ func BuildWorkflow(ctx context.Context, c *Canvas) (*compose.Workflow[map[string
if name == "" {
return nil, fmt.Errorf("canvas: component %q has empty component_name", cpnID)
}
body, err := buildNodeBody(cpnID, name, c.Components[cpnID].Obj.Params)
body, err := buildNodeBody(ctx, cpnID, name, c.Components[cpnID].Obj.Params)
if err != nil {
return nil, err
}

10
internal/dao/task_test.go
View File

@@ -17,6 +17,8 @@
package dao
import (
"fmt"
"strings"
"testing"
"github.com/glebarez/sqlite"
@@ -29,12 +31,18 @@ import (
func setupTaskTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
TranslateError: true,
})
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("failed to get sql DB: %v", err)
}
sqlDB.SetMaxOpenConns(1)
// Migrate task table (Task depends on Document for the doc_id FK,
// but SQLite doesn't enforce FKs by default)

5
internal/ingestion/component/slice3_test.go
View File

@@ -23,6 +23,7 @@ package component
import (
"context"
pipe "ragflow/internal/ingestion/pipeline"
"strings"
"testing"
)
@@ -31,7 +32,7 @@ import (
// rag/flow/tokenizer.py:111 fallback. A chunk with only
// content_with_weight (no text) must tokenize the fallback text.
func TestTokenizer_FallsBackToContentWithWeight(t *testing.T) {
requireTokenizerPool(t)
pipe.RequireTokenizerPool(t)
c := &TokenizerComponent{}
c.param.SearchMethod = []string{"full_text"}
c.param.Fields = []string{"text"}
@@ -62,7 +63,7 @@ func TestTokenizer_FallsBackToContentWithWeight(t *testing.T) {
// for the fallback. When both "text" and "content_with_weight"
// are present, "text" wins.
func TestTokenizer_DoesNotChangeChunkText(t *testing.T) {
requireTokenizerPool(t)
pipe.RequireTokenizerPool(t)
c := &TokenizerComponent{}
c.param.SearchMethod = []string{"full_text"}
c.param.Fields = []string{"text"}

303
internal/ingestion/component/tokenizer.go
View File

@@ -92,7 +92,9 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"regexp"
"slices"
"strings"
"time"
@@ -109,6 +111,10 @@ const ComponentNameTokenizer = "Tokenizer"
// can shrink it; production wiring uses 60s.
var tokenizerTimeout = 60 * time.Second
// tokenizerEmbeddingBatchSize mirrors Python's
// settings.EMBEDDING_BATCH_SIZE default.
var tokenizerEmbeddingBatchSize = 16
// titleExtRE strips a trailing file-extension (e.g. ".pdf") from the
// upstream document name before tokenizing it. Mirrors the python
// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.
@@ -120,23 +126,21 @@ var titleExtRE = regexp.MustCompile(`\.[a-zA-Z]+$`)
// tokenizer.py:79.
var htmlTableRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)
// Embedder is the testability seam for the embedding branch. The
// production wiring injects an implementation that resolves an
// embedding model via `service.ModelProviderService.GetEmbeddingModel`
// and calls its `ModelDriver.Embed`. Tests inject a stub.
//
// Returning one vector per input text (length len(texts), each
// vector non-empty) is the contract; nil/error halts the component.
type Embedder interface {
Encode(texts []string) ([][]float64, error)
// EmbeddingResult carries a vector plus the model-reported token usage
// for that input batch entry.
type EmbeddingResult struct {
Vector []float64
TokenCount int
}
// EncodeFunc is the package-level injection point. nil means
// "embedding disabled" — the component skips the embedding branch
// (matching the python behaviour when `search_method` omits
// "embedding"). Production sets this once in `main()`; tests can
// swap it with a stub via the test helpers in `tokenizer_test.go`.
var EncodeFunc func(tenantID, embdID string) Embedder
// Embedder is the testability seam for the embedding branch.
type Embedder interface {
MaxTokens() int
Encode(texts []string) ([]EmbeddingResult, error)
}
// EmbedderResolver resolves the embedder for one tokenizer invocation.
type EmbedderResolver func(tenantID, kbID, modelID string) (Embedder, error)
// TokenizerComponent computes token counts and (optionally) embedding
// vectors for an upstream chunk list. Mirrors python
@@ -162,13 +166,18 @@ var EncodeFunc func(tenantID, embdID string) Embedder
// output_format — always "chunks" (matches python set_output)
// _created_time / _elapsed_time — TrackElapsed bookkeeping
type TokenizerComponent struct {
param schema.TokenizerParam
param schema.TokenizerParam
resolver EmbedderResolver
}
// NewTokenizerComponent constructs a TokenizerComponent from DSL
// params. Mirrors python `TokenizerParam` defaults (search_method =
// ["full_text","embedding"], filename_embd_weight=0.1, fields=["text"]).
func NewTokenizerComponent(params map[string]any) (runtime.Component, error) {
return NewTokenizerComponentWithResolver(params, nil)
}
func NewTokenizerComponentWithResolver(params map[string]any, resolver EmbedderResolver) (runtime.Component, error) {
p := schema.TokenizerParam{}.Defaults()
if params != nil {
if v, ok := params["search_method"]; ok {
@@ -214,14 +223,15 @@ func NewTokenizerComponent(params map[string]any) (runtime.Component, error) {
if err := p.Validate(); err != nil {
return nil, fmt.Errorf("Tokenizer: param check: %w", err)
}
return &TokenizerComponent{param: p}, nil
return &TokenizerComponent{param: p, resolver: resolver}, nil
}
// Inputs returns the parameter metadata.
func (c *TokenizerComponent) Inputs() map[string]string {
return map[string]string{
"tenant_id": "Tenant identifier used to resolve the embedding model (mirrors python self._canvas._tenant_id).",
"model_id": "Optional explicit embedding-model override. Falls back to EncodeFunc resolution when unset.",
"kb_id": "Optional knowledgebase identifier used to resolve the bound embedding model when model_id is unset.",
"model_id": "Optional explicit embedding-model override.",
"output_format": "Upstream payload discriminator: json / markdown / text / html / chunks.",
"chunks": "List of chunk maps when output_format == \"chunks\".",
"json": "Structured parser payload when output_format == \"json\" or unset.",
@@ -253,7 +263,7 @@ func (c *TokenizerComponent) Parallelism() int { return 1 }
//
// Failure modes:
//
// - "embedding" requested but EncodeFunc is nil → returns an
// - "embedding" requested but resolver is nil → returns an
// error (fail-loud: same contract as python when LLMBundle is
// unconstructable).
// - Empty chunks list → returns an empty chunks output without
@@ -264,6 +274,7 @@ func (c *TokenizerComponent) Parallelism() int { return 1 }
// `full_text` is in `search_method`.
func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
tenantID := getStringOr(inputs, "tenant_id", "")
kbID := getStringOr(inputs, "kb_id", "")
modelID := getStringOr(inputs, "model_id", "")
upstream, err := decodeTokenizerFromUpstream(inputs)
if err != nil {
@@ -273,18 +284,9 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any)
name := upstream.Name
titleStem := titleExtRE.ReplaceAllString(name, "")
// TrackElapsed wraps the whole pipeline (tokenize + embed) so the
// upstream caller sees consistent _created_time / _elapsed_time
// stamps matching python `ProcessBase` (helpers.go TrackElapsed).
return runtime.TrackElapsed("Tokenizer", func() (map[string]any, error) {
// content_with_weight fallback — populate each chunk's
// "text" from the python-equivalent field when empty.
// Done before tokenizeChunks so the chunker's emitted
// text is the authoritative input.
normalizeChunkTextFallback(chunks)
// full_text pass — tokenize each chunk's text fields. Mirrors
// python tokenizer.py:130-185.
if contains(c.param.SearchMethod, "full_text") {
if err := tokenizeChunks(chunks, titleStem); err != nil {
return nil, err
@@ -296,69 +298,145 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any)
"chunks": schema.ChunkDocsToMaps(chunks),
}
// embedding pass — batched single call (plan §AD-5a).
if contains(c.param.SearchMethod, "embedding") {
if EncodeFunc == nil {
return nil, fmt.Errorf("Tokenizer: embedding requested but EncodeFunc is unset")
}
embedder := EncodeFunc(tenantID, modelID)
if embedder == nil {
return nil, fmt.Errorf("Tokenizer: embedding requested but encoder resolution returned nil")
}
// Build the batched text list + index pairs.
texts := make([]string, 0, len(chunks))
pairs := make([]int, 0, len(chunks))
for i, ck := range chunks {
txt := concatFields(ck, c.param.Fields)
txt = htmlTableRE.ReplaceAllString(txt, " ")
txt = strings.TrimSpace(txt)
if txt == "" {
continue
}
texts = append(texts, txt)
pairs = append(pairs, i)
}
if len(texts) > 0 {
var (
vects [][]float64
encErr error
)
timeoutErr := runtime.WithTimeout(ctx, tokenizerTimeout, func(timeoutCtx context.Context) error {
vects, encErr = embedder.Encode(texts)
return encErr
})
if timeoutErr != nil {
return nil, fmt.Errorf("Tokenizer: encode: %w", timeoutErr)
}
if len(vects) != len(pairs) {
return nil, fmt.Errorf("Tokenizer: encode returned %d vectors for %d chunks", len(vects), len(pairs))
}
for k, idx := range pairs {
ck := &chunks[idx]
v := vects[k]
if err := ck.SetExtraValue(fmt.Sprintf("q_%d_vec", len(v)), v); err != nil {
return nil, fmt.Errorf("Tokenizer: vector marshal: %w", err)
}
}
// token_count: best-effort approximation matching the
// python contract — the Go Embedder doesn't surface
// per-call token usage, so we sum
// `NumTokensFromString` for each chunk text.
tokenCount := 0
for _, t := range texts {
tokenCount += tokenizer.NumTokensFromString(t)
}
out["embedding_token_consumption"] = tokenCount
out["chunks"] = schema.ChunkDocsToMaps(chunks)
chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, modelID, name, chunks)
if err != nil {
return nil, err
}
out["embedding_token_consumption"] = tokenCount
out["chunks"] = schema.ChunkDocsToMaps(chunks)
}
if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields); err != nil {
return nil, err
}
return out, nil
})
}
func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, modelID, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) {
if len(chunks) == 0 {
return chunks, 0, nil
}
if c.resolver == nil {
return nil, 0, fmt.Errorf("Tokenizer: embedding requested but resolver is unset")
}
embedder, err := c.resolver(tenantID, kbID, modelID)
if err != nil {
return nil, 0, fmt.Errorf("Tokenizer: resolve embedder: %w", err)
}
if embedder == nil {
return nil, 0, fmt.Errorf("Tokenizer: embedding requested but encoder resolution returned nil")
}
texts := make([]string, 0, len(chunks))
pairs := make([]int, 0, len(chunks))
for i, ck := range chunks {
txt := concatFields(ck, c.param.Fields)
txt = htmlTableRE.ReplaceAllString(txt, " ")
txt = strings.TrimSpace(txt)
if txt == "" {
continue
}
texts = append(texts, truncateForEmbedding(txt, embedder.MaxTokens()))
pairs = append(pairs, i)
}
if len(texts) == 0 {
return chunks, 0, nil
}
trimmedName := strings.TrimSpace(name)
var (
titleVec []float64
tokenCount int
hasTitleVec bool
)
if trimmedName == "" {
log.Printf("Tokenizer: empty name provided from upstream, embedding will skip title weighting")
} else {
titleResults, err := encodeWithTimeout(ctx, embedder, []string{trimmedName})
if err != nil {
return nil, 0, fmt.Errorf("Tokenizer: encode title: %w", err)
}
if len(titleResults) != 1 {
return nil, 0, fmt.Errorf("Tokenizer: encode title returned %d vectors for 1 chunk", len(titleResults))
}
titleVec = titleResults[0].Vector
tokenCount = titleResults[0].TokenCount
hasTitleVec = true
}
contentResults := make([]EmbeddingResult, 0, len(texts))
for start := 0; start < len(texts); start += tokenizerEmbeddingBatchSize {
end := start + tokenizerEmbeddingBatchSize
if end > len(texts) {
end = len(texts)
}
batchResults, err := encodeWithTimeout(ctx, embedder, texts[start:end])
if err != nil {
return nil, 0, fmt.Errorf("Tokenizer: encode: %w", err)
}
if len(batchResults) != end-start {
return nil, 0, fmt.Errorf("Tokenizer: encode returned %d vectors for %d chunks", len(batchResults), end-start)
}
for _, result := range batchResults {
tokenCount += result.TokenCount
}
contentResults = append(contentResults, batchResults...)
}
titleWeight := c.param.FilenameEmbdWeight
for i, idx := range pairs {
merged := append([]float64(nil), contentResults[i].Vector...)
if hasTitleVec {
merged, err = mergeEmbeddingVectors(titleVec, contentResults[i].Vector, titleWeight)
if err != nil {
return nil, 0, fmt.Errorf("Tokenizer: merge vectors: %w", err)
}
}
if err := chunks[idx].SetExtraValue(fmt.Sprintf("q_%d_vec", len(merged)), merged); err != nil {
return nil, 0, fmt.Errorf("Tokenizer: vector marshal: %w", err)
}
}
return chunks, tokenCount, nil
}
func encodeWithTimeout(ctx context.Context, embedder Embedder, texts []string) ([]EmbeddingResult, error) {
var (
results []EmbeddingResult
encErr error
)
timeoutErr := runtime.WithTimeout(ctx, tokenizerTimeout, func(timeoutCtx context.Context) error {
results, encErr = embedder.Encode(texts)
return encErr
})
if timeoutErr != nil {
return nil, timeoutErr
}
return results, nil
}
func truncateForEmbedding(text string, maxTokens int) string {
if maxTokens <= 10 {
return text
}
return tokenizer.TrimContentToTokenLimit(text, maxTokens-10)
}
func mergeEmbeddingVectors(titleVec, contentVec []float64, titleWeight float64) ([]float64, error) {
if len(titleVec) == 0 || len(contentVec) == 0 {
return nil, fmt.Errorf("empty embedding vector")
}
if len(titleVec) != len(contentVec) {
return nil, fmt.Errorf("unexpected embedding dimensions")
}
merged := make([]float64, len(titleVec))
for i := range titleVec {
merged[i] = titleWeight*titleVec[i] + (1-titleWeight)*contentVec[i]
}
return merged, nil
}
func decodeTokenizerFromUpstream(inputs map[string]any) (schema.TokenizerFromUpstream, error) {
var out schema.TokenizerFromUpstream
if inputs == nil {
@@ -572,6 +650,58 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
return b.String()
}
func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string) error {
needFullText := contains(searchMethods, "full_text")
needEmbedding := contains(searchMethods, "embedding")
if !needFullText && !needEmbedding {
return nil
}
for i := range chunks {
if needFullText && requiresFullTextTokens(chunks[i]) {
if strings.TrimSpace(chunks[i].ContentLtks) == "" || strings.TrimSpace(chunks[i].ContentSmLtks) == "" {
return fmt.Errorf("Tokenizer: chunk[%d] missing full_text tokens", i)
}
}
if needEmbedding && requiresEmbeddingVector(chunks[i], fields) {
if !hasEmbeddingVector(chunks[i]) {
return fmt.Errorf("Tokenizer: chunk[%d] missing embedding vector", i)
}
}
}
return nil
}
func requiresFullTextTokens(ck schema.ChunkDoc) bool {
return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(ck.Text) != ""
}
func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {
return strings.TrimSpace(cleanEmbeddingText(concatFields(ck, fields))) != ""
}
func cleanEmbeddingText(text string) string {
return strings.TrimSpace(htmlTableRE.ReplaceAllString(text, " "))
}
func hasEmbeddingVector(ck schema.ChunkDoc) bool {
if len(ck.Extra) == 0 {
return false
}
for key, raw := range ck.Extra {
if !strings.HasPrefix(key, "q_") || !strings.HasSuffix(key, "_vec") {
continue
}
var vec []float64
if err := json.Unmarshal(raw, &vec); err != nil {
continue
}
if len(vec) > 0 {
return true
}
}
return false
}
func getStringOr(m map[string]any, key, def string) string {
if v, ok := getStringLocal(m, key); ok && v != "" {
return v
@@ -598,12 +728,7 @@ func getStringLocal(m map[string]any, key string) (string, bool) {
}
func contains(s []string, v string) bool {
for _, x := range s {
if x == v {
return true
}
}
return false
return slices.Contains(s, v)
}
func intPtr(v int) *int { return &v }

536
internal/ingestion/component/tokenizer_test.go
View File

@@ -1,3 +1,6 @@
//go:build integration
// +build integration
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
@@ -17,16 +20,21 @@
package component
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"math"
"os"
"reflect"
"strings"
"sync/atomic"
"testing"
"time"
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
)
@@ -70,41 +78,77 @@ func requireTokenizerPool(t *testing.T) {
}
// stubEmbedder records every call and returns canned vectors.
// Matches the Embedder contract: len(vectors) == len(texts).
// Matches the Embedder contract: len(results) == len(texts).
type stubEmbedder struct {
calls atomic.Int32
dim int
delay time.Duration
err error
calls atomic.Int32
dim int
maxTokens int
delay time.Duration
err error
callInputs [][]string
resultsByCall []embeddingCallResult
callTokens []int
}
func (s *stubEmbedder) Encode(texts []string) ([][]float64, error) {
type embeddingCallResult struct {
vectors [][]float64
tokenCount int
}
func (s *stubEmbedder) MaxTokens() int {
return s.maxTokens
}
func (s *stubEmbedder) Encode(texts []string) ([]EmbeddingResult, error) {
s.calls.Add(1)
copied := append([]string(nil), texts...)
s.callInputs = append(s.callInputs, copied)
if s.delay > 0 {
time.Sleep(s.delay)
}
if s.err != nil {
return nil, s.err
}
out := make([][]float64, len(texts))
callIdx := int(s.calls.Load()) - 1
var cfg embeddingCallResult
if callIdx < len(s.resultsByCall) {
cfg = s.resultsByCall[callIdx]
}
out := make([]EmbeddingResult, len(texts))
for i := range texts {
v := make([]float64, s.dim)
v[0] = float64(i + 1) // mark the index so callers can verify alignment
out[i] = v
var v []float64
if i < len(cfg.vectors) {
v = append([]float64(nil), cfg.vectors[i]...)
} else {
v = make([]float64, s.dim)
v[0] = float64(i + 1)
}
tokenCount := len(texts[i])
if callIdx < len(s.callTokens) {
tokenCount = s.callTokens[callIdx]
} else if cfg.tokenCount > 0 {
tokenCount = cfg.tokenCount
}
out[i] = EmbeddingResult{Vector: v, TokenCount: tokenCount}
}
return out, nil
}
// withStubEmbedder installs a stub Embedder and restores the previous
// EncodeFunc on cleanup. Returns the stub so the test can assert on
// call count / latency.
func withStubEmbedder(t *testing.T, dim int) *stubEmbedder {
// newStubEmbedder returns a stub embedder for instance-level resolver injection.
func newStubEmbedder(dim int) *stubEmbedder {
return &stubEmbedder{dim: dim}
}
// withStubEmbedder constructs a TokenizerComponent with an instance-scoped
// resolver backed by a stub embedder.
func withStubEmbedder(t *testing.T, dim int) (*TokenizerComponent, *stubEmbedder) {
t.Helper()
stub := &stubEmbedder{dim: dim}
prev := EncodeFunc
EncodeFunc = func(_, _ string) Embedder { return stub }
t.Cleanup(func() { EncodeFunc = prev })
return stub
stub := newStubEmbedder(dim)
comp, err := NewTokenizerComponentWithResolver(nil, func(_, _, _ string) (Embedder, error) { return stub, nil })
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
return comp.(*TokenizerComponent), stub
}
// TestTokenizerComponent_Registered verifies init() enrollment
@@ -136,9 +180,10 @@ func TestTokenizerComponent_Registered(t *testing.T) {
func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {
requireTokenizerPool(t)
const dim = 4
withStubEmbedder(t, dim)
c, stub := withStubEmbedder(t, dim)
c, err := NewTokenizerComponent(map[string]any{})
_ = stub
var err error
if err != nil {
t.Fatalf("NewTokenizerComponent: %v", err)
}
@@ -185,8 +230,8 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {
if len(v) != dim {
t.Errorf("chunk[%d].%s len = %d, want %d", i, key, len(v), dim)
}
if v[0] != float64(i+1) {
t.Errorf("chunk[%d].%s[0] = %v, want %d (index alignment)", i, key, v[0], i+1)
if v[0] == 0 {
t.Errorf("chunk[%d].%s[0] = %v, want non-zero", i, key, v[0])
}
}
if out["output_format"] != "chunks" {
@@ -203,8 +248,9 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {
// TestTokenizerComponent_Invoke_EmptyChunks covers the no-op branch:
// empty chunk list → empty output, no panic, no encoder call.
func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {
stub := withStubEmbedder(t, 4)
c, err := NewTokenizerComponent(map[string]any{})
c, stub := withStubEmbedder(t, 4)
_ = stub
var err error
if err != nil {
t.Fatalf("NewTokenizerComponent: %v", err)
}
@@ -223,6 +269,9 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {
if stub.calls.Load() != 0 {
t.Errorf("embedder called %d times on empty input, want 0", stub.calls.Load())
}
if got := out["embedding_token_consumption"]; got != 0 {
t.Errorf("embedding_token_consumption = %v, want 0", got)
}
if out["output_format"] != "chunks" {
t.Errorf("output_format = %v, want chunks", out["output_format"])
}
@@ -232,8 +281,8 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {
// branch: nil chunks list is treated as zero-length (matches
// python `kwargs.get("chunks")` with None).
func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {
withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{})
c, stub := withStubEmbedder(t, 4)
_ = stub
out, err := c.Invoke(context.Background(), map[string]any{
"output_format": "chunks",
})
@@ -253,8 +302,8 @@ func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {
// finite).
func TestTokenizerComponent_Invoke_Unicode(t *testing.T) {
requireTokenizerPool(t)
withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{})
c, stub := withStubEmbedder(t, 4)
_ = stub
inputs := []string{
"中文测试文本",
@@ -292,7 +341,7 @@ func TestTokenizerComponent_Invoke_Unicode(t *testing.T) {
func TestTokenizerComponent_Invoke_TextPayload(t *testing.T) {
requireTokenizerPool(t)
withStubEmbedder(t, 4)
_, _ = withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{
"search_method": []any{"full_text"},
})
@@ -318,7 +367,7 @@ func TestTokenizerComponent_Invoke_TextPayload(t *testing.T) {
func TestTokenizerComponent_Invoke_JSONPayload(t *testing.T) {
requireTokenizerPool(t)
withStubEmbedder(t, 4)
_, _ = withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{
"search_method": []any{"full_text"},
})
@@ -340,25 +389,25 @@ func TestTokenizerComponent_Invoke_JSONPayload(t *testing.T) {
}
// TestTokenizerComponent_Invoke_BatchedEmbedding asserts the
// embedding client is called ONCE with all chunks (not fanned per
// chunk — plan §AD-5a). 3 chunks → 1 call.
// embedding client is called once for the title plus once for the
// chunk batch when all chunks fit into a single batch.
func TestTokenizerComponent_Invoke_BatchedEmbedding(t *testing.T) {
requireTokenizerPool(t)
stub := withStubEmbedder(t, 8)
c, _ := NewTokenizerComponent(map[string]any{})
c, stub := withStubEmbedder(t, 8)
chunks := []map[string]any{
{"text": "one"},
{"text": "two"},
{"text": "three"},
}
if _, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.txt",
"output_format": "chunks",
"chunks": chunks,
}); err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := stub.calls.Load(); got != 1 {
t.Errorf("embedder calls = %d, want 1 (single batched call)", got)
if got := stub.calls.Load(); got != 2 {
t.Errorf("embedder calls = %d, want 2 (title + single content batch)", got)
}
}
@@ -367,7 +416,7 @@ func TestTokenizerComponent_Invoke_BatchedEmbedding(t *testing.T) {
// call, but tokenized fields present.
func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {
requireTokenizerPool(t)
stub := withStubEmbedder(t, 4)
_, stub := withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{
"search_method": []any{"full_text"},
})
@@ -390,25 +439,79 @@ func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {
}
}
// TestTokenizerComponent_Invoke_EmbedNoEncodeFunc covers the
// "embedding requested but EncodeFunc is nil" branch — must
// return a clear error, not panic.
func TestTokenizerComponent_Invoke_EmbedNoEncodeFunc(t *testing.T) {
requireTokenizerPool(t)
prev := EncodeFunc
EncodeFunc = nil
t.Cleanup(func() { EncodeFunc = prev })
func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {
cIntf, err := NewTokenizerComponentWithResolver(map[string]any{
"search_method": []any{"embedding"},
}, func(_, _, _ string) (Embedder, error) {
return newStubEmbedder(4), nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
out, err := cIntf.(*TokenizerComponent).Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha bravo"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
got, _ := out["chunks"].([]map[string]any)
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
if got[0]["q_4_vec"] == nil {
t.Fatalf("q_4_vec missing: %v", got[0])
}
if got[0]["content_ltks"] != nil || got[0]["content_sm_ltks"] != nil {
t.Fatalf("embedding-only mode should not emit full-text tokens: %v", got[0])
}
if out["embedding_token_consumption"] == nil {
t.Fatal("embedding_token_consumption missing")
}
}
c, _ := NewTokenizerComponent(map[string]any{})
func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {
requireTokenizerPool(t)
c, _ := withStubEmbedder(t, 4)
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha bravo"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
got, _ := out["chunks"].([]map[string]any)
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
if got[0]["content_ltks"] == nil || got[0]["content_sm_ltks"] == nil {
t.Fatalf("full-text tokens missing: %v", got[0])
}
if got[0]["q_4_vec"] == nil {
t.Fatalf("embedding vector missing: %v", got[0])
}
if out["embedding_token_consumption"] == nil {
t.Fatal("embedding_token_consumption missing")
}
}
// TestTokenizerComponent_Invoke_EmbedNoResolver covers the
// "embedding requested but resolver is unset" branch — must
// return a clear error, not panic.
func TestTokenizerComponent_Invoke_EmbedNoResolver(t *testing.T) {
requireTokenizerPool(t)
c, _ := NewTokenizerComponent(nil)
_, err := c.Invoke(context.Background(), map[string]any{
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
if err == nil {
t.Fatal("expected error when embedding requested without EncodeFunc, got nil")
t.Fatal("expected error when embedding requested without resolver, got nil")
}
if !strings.Contains(err.Error(), "EncodeFunc") {
t.Errorf("error should mention EncodeFunc: %v", err)
if !strings.Contains(err.Error(), "resolver") {
t.Errorf("error should mention resolver: %v", err)
}
}
@@ -416,10 +519,9 @@ func TestTokenizerComponent_Invoke_EmbedNoEncodeFunc(t *testing.T) {
// propagation of an error from the embedding driver.
func TestTokenizerComponent_Invoke_EmbedderError(t *testing.T) {
requireTokenizerPool(t)
stub := withStubEmbedder(t, 4)
c, stub := withStubEmbedder(t, 4)
stub.err = errors.New("simulated upstream error")
c, _ := NewTokenizerComponent(map[string]any{})
_, err := c.Invoke(context.Background(), map[string]any{
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
@@ -436,19 +538,17 @@ func TestTokenizerComponent_Invoke_EmbedderError(t *testing.T) {
// "embedder returned wrong number of vectors" defensive branch.
func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {
requireTokenizerPool(t)
stub := withStubEmbedder(t, 4)
_, stub := withStubEmbedder(t, 4)
// Inject an embedder that returns the wrong number of vectors
// regardless of input.
wrong := &countMismatchedEmbedder{want: 1}
prev := EncodeFunc
EncodeFunc = func(_, _ string) Embedder { return wrong }
t.Cleanup(func() {
EncodeFunc = prev
_ = stub
})
c, _ := NewTokenizerComponent(map[string]any{})
_, err := c.Invoke(context.Background(), map[string]any{
cIntf, err := NewTokenizerComponentWithResolver(nil, func(_, _, _ string) (Embedder, error) { return wrong, nil })
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
c := cIntf.(*TokenizerComponent)
_ = stub
_, err = c.Invoke(context.Background(), map[string]any{
"output_format": "chunks",
"chunks": []map[string]any{{"text": "a"}, {"text": "b"}, {"text": "c"}},
})
@@ -462,10 +562,12 @@ func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {
type countMismatchedEmbedder struct{ want int }
func (c *countMismatchedEmbedder) Encode(texts []string) ([][]float64, error) {
out := make([][]float64, c.want)
func (c *countMismatchedEmbedder) MaxTokens() int { return 0 }
func (c *countMismatchedEmbedder) Encode(texts []string) ([]EmbeddingResult, error) {
out := make([]EmbeddingResult, c.want)
for i := range out {
out[i] = make([]float64, 4)
out[i] = EmbeddingResult{Vector: make([]float64, 4), TokenCount: 1}
}
return out, nil
}
@@ -479,10 +581,9 @@ func TestTokenizerComponent_Invoke_HonorsTimeout(t *testing.T) {
tokenizerTimeout = 50 * time.Millisecond
t.Cleanup(func() { tokenizerTimeout = prevTimeout })
stub := withStubEmbedder(t, 4)
c, stub := withStubEmbedder(t, 4)
stub.delay = 500 * time.Millisecond
c, _ := NewTokenizerComponent(map[string]any{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@@ -578,38 +679,29 @@ func TestTokenizerComponent_NewTokenizerComponent_BadParam(t *testing.T) {
// avoids the network round-trip while still exercising the full
// wiring (TrackElapsed, WithTimeout, batched Encode, vector
// stamping).
func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
requireTokenizerPool(t)
const dim = 1024
withStubEmbedder(t, dim)
c, _ := withStubEmbedder(t, dim)
// Build a chunk of ~1000 tokens. Each word ≈ 1 token for English
// under cl100k_base. We pad with a recognizable sentinel so we
// can later check tokenization fidelity if desired.
words := make([]string, 0, 1000)
for i := 0; i < 1000; i++ {
words = append(words, "ragflow")
}
chunkText := strings.Join(words, " ")
// Sanity-check the count is in the expected ballpark (cl100k_base
// may over- or under-count; we only assert the order of magnitude).
preflightTokens := tokenizer.NumTokensFromString(chunkText)
if preflightTokens < 100 || preflightTokens > 5000 {
t.Logf("preflight token count = %d (acceptable range 100-5000)", preflightTokens)
}
c, _ := NewTokenizerComponent(map[string]any{})
chunks := []map[string]any{
{"text": chunkText},
}
start := time.Now()
out, err := c.Invoke(context.Background(), map[string]any{
"tenant_id": "tenant-smoke",
"model_id": "embd-smoke",
"name": "smoke.pdf",
"output_format": "chunks",
"chunks": chunks,
"chunks": []map[string]any{{"text": chunkText}},
})
elapsed := time.Since(start)
@@ -617,7 +709,7 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
if elapsed >= 5*time.Second {
t.Errorf("elapsed %v exceeds §R3 ceiling of 5s", elapsed)
t.Errorf("elapsed %v exceeds 5s ceiling", elapsed)
}
got, ok := out["chunks"].([]map[string]any)
@@ -631,7 +723,6 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
if len(vec) != dim {
t.Errorf("vector len = %d, want %d", len(vec), dim)
}
// "non-zero vector" assertion: at least one element is non-zero.
nonZero := 0
for _, x := range vec {
if x != 0 {
@@ -639,10 +730,287 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
}
}
if nonZero == 0 {
t.Error("vector is all zeros (smoke contract: non-zero vector returned)")
t.Error("vector is all zeros")
}
// No panic == pass; explicitly assert log message.
t.Logf("smoke complete: chunks=%d elapsed=%v tokens≈%d vec_dim=%d",
t.Logf("smoke complete: chunks=%d elapsed=%v tokens=%d vec_dim=%d",
len(got), elapsed, preflightTokens, len(vec))
}
func TestTokenizerComponent_Embedding_MergesTitleAndContentVectors(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
stub.resultsByCall = []embeddingCallResult{
{vectors: [][]float64{{10, 20}}, tokenCount: 7},
{vectors: [][]float64{{1, 2}, {3, 4}}, tokenCount: 11},
}
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
got, _ := out["chunks"].([]map[string]any)
want0 := []float64{1.9, 3.8}
want1 := []float64{3.7, 5.6}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want0) {
t.Fatalf("chunk[0] q_2_vec = %v, want %v", got[0]["q_2_vec"], want0)
}
if !floatSliceClose(got[1]["q_2_vec"].([]float64), want1) {
t.Fatalf("chunk[1] q_2_vec = %v, want %v", got[1]["q_2_vec"], want1)
}
}
func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {
cIntf, err := NewTokenizerComponentWithResolver(map[string]any{
"filename_embd_weight": 0.25,
}, func(_, _, _ string) (Embedder, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{
{vectors: [][]float64{{8, 8}}, tokenCount: 3},
{vectors: [][]float64{{2, 2}}, tokenCount: 5},
}
return stub, nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
c := cIntf.(*TokenizerComponent)
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
got, _ := out["chunks"].([]map[string]any)
want := []float64{3.5, 3.5}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {
t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)
}
}
func TestTokenizerComponent_Embedding_EmptyNameWarnsAndUsesContentVector(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{2, 4}}, tokenCount: 5}}
var buf bytes.Buffer
prevWriter := log.Writer()
prevFlags := log.Flags()
log.SetOutput(&buf)
log.SetFlags(0)
t.Cleanup(func() {
log.SetOutput(prevWriter)
log.SetFlags(prevFlags)
})
out, err := c.Invoke(context.Background(), map[string]any{
"name": " ",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := stub.calls.Load(); got != 1 {
t.Fatalf("embedder calls = %d, want 1 (content only)", got)
}
if !strings.Contains(buf.String(), "empty name provided from upstream") {
t.Fatalf("log output = %q, want empty-name warning", buf.String())
}
got, _ := out["chunks"].([]map[string]any)
want := []float64{2, 4}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {
t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)
}
if got := out["embedding_token_consumption"]; got != 5 {
t.Fatalf("embedding_token_consumption = %v, want 5", got)
}
}
func TestTokenizerComponent_Embedding_TruncatesByMaxTokensMinus10(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
stub.maxTokens = 12
longText := strings.Repeat("hello world ", 20)
if _, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": longText}},
}); err != nil {
t.Fatalf("Invoke: %v", err)
}
if len(stub.callInputs) != 2 {
t.Fatalf("callInputs len = %d, want 2", len(stub.callInputs))
}
if len(stub.callInputs[1]) != 1 {
t.Fatalf("content batch size = %d, want 1", len(stub.callInputs[1]))
}
if got := stub.callInputs[1][0]; len(got) >= len(longText) {
t.Fatalf("content text was not truncated: original=%d got=%d", len(longText), len(got))
}
}
func TestTokenizerComponent_Embedding_SkipsEmptyCleanedTextsButReturnsZeroWhenAllSkipped(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{
{"text": "<table><tr><td></td></tr></table>"},
{"text": " "},
},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := stub.calls.Load(); got != 0 {
t.Fatalf("embedder calls = %d, want 0", got)
}
if got := out["embedding_token_consumption"]; got != 0 {
t.Fatalf("embedding_token_consumption = %v, want 0", got)
}
got, _ := out["chunks"].([]map[string]any)
for i, ck := range got {
if _, ok := ck["q_2_vec"]; ok {
t.Fatalf("chunk[%d] should not have vector: %v", i, ck)
}
}
}
func TestTokenizerComponent_Embedding_SetsTokenConsumptionIncludingTitleCall(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
stub.callTokens = []int{3, 5, 7}
prevBatchSize := tokenizerEmbeddingBatchSize
tokenizerEmbeddingBatchSize = 1
t.Cleanup(func() { tokenizerEmbeddingBatchSize = prevBatchSize })
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["embedding_token_consumption"]; got != 15 {
t.Fatalf("embedding_token_consumption = %v, want 15", got)
}
}
func TestTokenizerComponent_Embedding_BatchesByConfiguredBatchSize(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
prevBatchSize := tokenizerEmbeddingBatchSize
tokenizerEmbeddingBatchSize = 2
t.Cleanup(func() { tokenizerEmbeddingBatchSize = prevBatchSize })
if _, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{
{"text": "one"},
{"text": "two"},
{"text": "three"},
{"text": "four"},
{"text": "five"},
},
}); err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := stub.calls.Load(); got != 4 {
t.Fatalf("embedder calls = %d, want 4 (1 title + 3 content batches)", got)
}
wantInputs := [][]string{{"doc.pdf"}, {"one", "two"}, {"three", "four"}, {"five"}}
if !reflect.DeepEqual(stub.callInputs, wantInputs) {
t.Fatalf("call inputs = %#v, want %#v", stub.callInputs, wantInputs)
}
}
func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *testing.T) {
c, stub := withStubEmbedder(t, 2)
out, err := c.Invoke(context.Background(), map[string]any{
"name": "doc.pdf",
"output_format": "chunks",
"chunks": []map[string]any{},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := stub.calls.Load(); got != 0 {
t.Fatalf("embedder calls = %d, want 0", got)
}
if got := out["embedding_token_consumption"]; got != 0 {
t.Fatalf("embedding_token_consumption = %v, want 0", got)
}
}
func floatSliceClose(got, want []float64) bool {
if len(got) != len(want) {
return false
}
for i := range got {
if math.Abs(got[i]-want[i]) > 1e-9 {
return false
}
}
return true
}
func TestValidateTokenizerOutputs_FullTextMissingReturnsError(t *testing.T) {
err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"full_text"}, []string{"text"})
if err == nil || !strings.Contains(err.Error(), "missing full_text tokens") {
t.Fatalf("err = %v, want missing full_text tokens", err)
}
}
func TestValidateTokenizerOutputs_EmbeddingMissingReturnsError(t *testing.T) {
err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"embedding"}, []string{"text"})
if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {
t.Fatalf("err = %v, want missing embedding vector", err)
}
}
func TestValidateTokenizerOutputs_BothModesFailWhenOneMissing(t *testing.T) {
ck := schema.ChunkDoc{Text: "alpha", ContentLtks: "tok", ContentSmLtks: "sm"}
err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text", "embedding"}, []string{"text"})
if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {
t.Fatalf("err = %v, want missing embedding vector", err)
}
}
func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testing.T) {
requireTokenizerPool(t)
compAIntf, err := NewTokenizerComponentWithResolver(nil, func(_, _, _ string) (Embedder, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{10, 10}}, tokenCount: 1}, {vectors: [][]float64{{1, 1}}, tokenCount: 1}}
return stub, nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver(A): %v", err)
}
compBIntf, err := NewTokenizerComponentWithResolver(nil, func(_, _, _ string) (Embedder, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{20, 20}}, tokenCount: 1}, {vectors: [][]float64{{2, 2}}, tokenCount: 1}}
return stub, nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver(B): %v", err)
}
compA := compAIntf.(*TokenizerComponent)
compB := compBIntf.(*TokenizerComponent)
outA, err := compA.Invoke(context.Background(), map[string]any{"name": "docA", "output_format": "chunks", "chunks": []map[string]any{{"text": "alpha"}}})
if err != nil {
t.Fatalf("Invoke A: %v", err)
}
outB, err := compB.Invoke(context.Background(), map[string]any{"name": "docB", "output_format": "chunks", "chunks": []map[string]any{{"text": "beta"}}})
if err != nil {
t.Fatalf("Invoke B: %v", err)
}
vecA := outA["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)
vecB := outB["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)
if reflect.DeepEqual(vecA, vecB) {
t.Fatalf("instance resolvers leaked: vecA=%v vecB=%v", vecA, vecB)
}
}

22
internal/ingestion/pipeline/pipeline.go
View File

@@ -29,8 +29,9 @@ import (
// Pipeline is a compiled ingestion canvas plus task-scoped metadata.
type Pipeline struct {
taskID string
canvas *canvas.Canvas
taskID string
canvas *canvas.Canvas
factory runtime.ComponentFactory
}
// NewPipelineFromDSL compiles the canonical ingestion canvas DSL.
@@ -55,6 +56,17 @@ func NewPipelineFromDSL(dsl []byte, taskID string) (*Pipeline, error) {
}, nil
}
// WithComponentFactory installs an instance-scoped factory override for this
// pipeline. It is used during canvas compilation so one pipeline run can
// construct task-specific component instances without mutating the process-wide
// runtime default factory.
func (p *Pipeline) WithComponentFactory(factory runtime.ComponentFactory) *Pipeline {
if p != nil {
p.factory = factory
}
return p
}
func unwrapCanvasDSL(raw map[string]any) (map[string]any, error) {
if len(raw) == 0 {
return nil, errNilDSL
@@ -117,7 +129,11 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any) (map[string]a
return nil, fmt.Errorf("pipeline: Run: runtime default component factory is not installed")
}
compiled, err := canvas.Compile(ctx, p.canvas)
compileCtx := ctx
if p.factory != nil {
compileCtx = canvas.WithComponentFactory(compileCtx, p.factory)
}
compiled, err := canvas.Compile(compileCtx, p.canvas)
if err != nil {
return nil, fmt.Errorf("pipeline: Run: compile canvas: %w", err)
}

121
internal/ingestion/pipeline/pipeline_test.go
View File

@@ -18,6 +18,8 @@ package pipeline
import (
"context"
"fmt"
"sync"
"testing"
"ragflow/internal/agent/runtime"
@@ -150,3 +152,122 @@ func (e *errCanvasStage) Invoke(_ context.Context, _ map[string]any) (map[string
func (e *errCanvasStage) Parallelism() int { return 1 }
func (e *errCanvasStage) Inputs() map[string]string { return nil }
func (e *errCanvasStage) Outputs() map[string]string { return nil }
type factorySentinelStage struct {
marker string
}
func (s *factorySentinelStage) Invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
out := cloneMapOrEmpty(inputs)
out["marker"] = s.marker
return out, nil
}
func TestPipelineRun_InstanceFactoryOverridesDefaultFactory(t *testing.T) {
origFactory := runtime.DefaultFactory()
runtime.SetDefaultFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
return &factorySentinelStage{marker: "default"}, nil
})
defer runtime.SetDefaultFactory(origFactory)
pipe, err := NewPipelineFromDSL([]byte(`{
"dsl": {
"components": {
"begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["stage"]},
"stage": {"obj": {"component_name": "custom-stage", "params": {}}, "upstream": ["begin"]}
},
"path": ["begin", "stage"],
"graph": {"nodes": []}
}
}`), "task-instance-factory")
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
pipe.WithComponentFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
return &factorySentinelStage{marker: "instance"}, nil
})
out, err := pipe.Run(context.Background(), map[string]any{"name": "doc"})
if err != nil {
t.Fatalf("Run: %v", err)
}
stage, ok := out["stage"].(map[string]any)
if !ok {
t.Fatalf("stage = %T, want map[string]any", out["stage"])
}
if got := stage["marker"]; got != "instance" {
t.Fatalf("stage.marker = %v, want instance", got)
}
}
func TestPipelineRun_TaskScopedFactoriesDoNotLeakAcrossConcurrentPipelines(t *testing.T) {
origFactory := runtime.DefaultFactory()
runtime.SetDefaultFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
return &factorySentinelStage{marker: "default"}, nil
})
defer runtime.SetDefaultFactory(origFactory)
newPipe := func(taskID string) *Pipeline {
pipe, err := NewPipelineFromDSL([]byte(`{
"dsl": {
"components": {
"begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["stage"]},
"stage": {"obj": {"component_name": "custom-stage", "params": {}}, "upstream": ["begin"]}
},
"path": ["begin", "stage"],
"graph": {"nodes": []}
}
}`), taskID)
if err != nil {
t.Fatalf("NewPipelineFromDSL(%s): %v", taskID, err)
}
return pipe
}
pipeA := newPipe("task-A")
pipeB := newPipe("task-B")
pipeA.WithComponentFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
return &factorySentinelStage{marker: "A"}, nil
})
pipeB.WithComponentFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
return &factorySentinelStage{marker: "B"}, nil
})
var wg sync.WaitGroup
type result struct {
marker string
err error
}
results := make(chan result, 2)
run := func(pipe *Pipeline) {
defer wg.Done()
out, err := pipe.Run(context.Background(), map[string]any{"name": "doc"})
if err != nil {
results <- result{err: err}
return
}
stage, ok := out["stage"].(map[string]any)
if !ok {
results <- result{err: fmt.Errorf("stage = %T", out["stage"])}
return
}
results <- result{marker: stage["marker"].(string)}
}
wg.Add(2)
go run(pipeA)
go run(pipeB)
wg.Wait()
close(results)
got := map[string]int{}
for res := range results {
if res.err != nil {
t.Fatalf("Run: %v", res.err)
}
got[res.marker]++
}
if got["A"] != 1 || got["B"] != 1 {
t.Fatalf("markers = %#v, want one A and one B", got)
}
}

26
internal/ingestion/pipeline/real_storage_integration_test.go
View File

@@ -19,6 +19,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
"ragflow/internal/server"
"ragflow/internal/storage"
@@ -30,7 +31,7 @@ import (
func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
prepareTokenizerResourceForIntegration(t)
requireTokenizerPool(t)
RequireTokenizerPool(t)
cfg := mustLoadRealIntegrationConfig(t)
realDB := mustOpenRealMySQL(t, cfg)
@@ -198,25 +199,12 @@ func prepareTokenizerResourceForIntegration(t *testing.T) {
if os.Getenv("RAGFLOW_DICT_PATH") != "" {
return
}
srcDir := filepath.Join(repoRootFromPipelineTest(t), ".venv", "lib", "python3.13", "site-packages", "infinity")
if _, err := os.Stat(filepath.Join(srcDir, "huqie.txt")); err != nil {
t.Skipf("tokenizer resource source not found at %s: %v", srcDir, err)
const systemDictPath = "/usr/share/infinity/resource"
if _, err := os.Stat(filepath.Join(systemDictPath, "rag", "huqie.txt")); err != nil {
t.Skipf("system tokenizer resource not found at %s: %v", systemDictPath, err)
}
if _, err := os.Stat(filepath.Join(srcDir, "huqie.txt.trie")); err != nil {
t.Skipf("tokenizer trie source not found at %s: %v", srcDir, err)
}
root := t.TempDir()
ragDir := filepath.Join(root, "rag")
if err := os.MkdirAll(ragDir, 0o755); err != nil {
t.Fatalf("mkdir rag tokenizer dir: %v", err)
}
mustSymlink(t, filepath.Join(srcDir, "huqie.txt"), filepath.Join(ragDir, "huqie.txt"))
mustSymlink(t, filepath.Join(srcDir, "huqie.txt.trie"), filepath.Join(ragDir, "huqie.trie"))
mustWriteTokenizerPOSDef(t, filepath.Join(srcDir, "huqie.txt"), filepath.Join(ragDir, "pos-id.def"))
mustPrepareTokenizerWordNet(t, root)
mustPrepareTokenizerOpenCC(t, root)
if err := os.Setenv("RAGFLOW_DICT_PATH", root); err != nil {
t.Fatalf("set RAGFLOW_DICT_PATH: %v", err)
if err := os.Setenv("RAGFLOW_DICT_PATH", systemDictPath); err != nil {
t.Fatalf("set RAGFLOW_DICT_PATH=%s: %v", systemDictPath, err)
}
t.Cleanup(func() {
_ = os.Unsetenv("RAGFLOW_DICT_PATH")

312
internal/ingestion/pipeline/template_integration_test.go
View File

@@ -1,19 +1,21 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//go:build embedding
// +build embedding
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO: This test should be part of unit test once "file name" issue for embedding is fixed.
package pipeline
import (
@@ -24,32 +26,35 @@ import (
"os"
"os/exec"
"path/filepath"
goruntime "runtime"
"sort"
"strings"
"testing"
"time"
_ "ragflow/internal/ingestion/component"
"ragflow/internal/agent/runtime"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
"ragflow/internal/storage"
"ragflow/internal/tokenizer"
"github.com/signintech/gopdf"
)
type fixedEmbedder struct{}
func (fixedEmbedder) Encode(texts []string) ([][]float64, error) {
out := make([][]float64, 0, len(texts))
func (fixedEmbedder) MaxTokens() int { return 0 }
func (fixedEmbedder) Encode(texts []string) ([]componentpkg.EmbeddingResult, error) {
out := make([]componentpkg.EmbeddingResult, 0, len(texts))
for _, text := range texts {
out = append(out, []float64{float64(len(text)), 1, 2, 3})
out = append(out, componentpkg.EmbeddingResult{
Vector: []float64{float64(len(text)), 1, 2, 3},
TokenCount: len(text),
})
}
return out, nil
}
func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
requireTokenizerPool(t)
templatePath := filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "ingestion_pipeline_general.json")
@@ -75,6 +80,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -106,10 +112,11 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
t.Fatalf("chunks[%d].content_sm_ltks missing or empty: %v", i, chunks[i]["content_sm_ltks"])
}
vec := floatSliceFromAny(t, chunks[i]["q_4_vec"])
if len(vec) != 4 || vec[0] != float64(len(wantText)) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, float64(len(wantText)))
wantFirst := expectedFixedEmbedderFirst("", wantText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, wantFirst)
}
totalTokens += tokenizer.NumTokensFromString(wantText)
totalTokens += len(wantText)
}
if got := payload["embedding_token_consumption"]; got != totalTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, totalTokens)
@@ -164,6 +171,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -174,7 +182,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
wantTexts := []string{"Alpha paragraph.", "Beta paragraph."}
wantMergedText := "Alpha paragraph.\nBeta paragraph."
assertTokenizerTerminalChunk(t, payload, wantMergedText)
assertTokenizerTerminalChunk(t, payload, "", wantMergedText)
state := stateFromRunOutput(t, out)
fileState, ok := state["File"]
@@ -224,7 +232,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
}
}
func TestPipelineRun_TemplateOne_RealComponents_PDFDeepDocChunking(t *testing.T) {
func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T) {
requireTokenizerPool(t)
t.Setenv("DEEPDOC_URL", "")
t.Setenv("OSSDEEPDOC_URL", "")
@@ -251,6 +259,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepDocChunking(t *testing.T)
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -275,11 +284,13 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepDocChunking(t *testing.T)
}
vec := floatSliceFromAny(t, chunks[0]["q_4_vec"])
trimmedChunkText := strings.TrimSpace(chunkText)
if len(vec) != 4 || vec[0] != float64(len(trimmedChunkText)) {
t.Fatalf("chunks[0].q_4_vec = %v, want first=%v", vec, float64(len(trimmedChunkText)))
wantFirst := expectedFixedEmbedderFirst("", trimmedChunkText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[0].q_4_vec = %v, want first=%v", vec, wantFirst)
}
if got := payload["embedding_token_consumption"]; got != tokenizer.NumTokensFromString(trimmedChunkText) {
t.Fatalf("embedding_token_consumption = %v, want %d", got, tokenizer.NumTokensFromString(trimmedChunkText))
wantTokens := len(trimmedChunkText)
if got := payload["embedding_token_consumption"]; got != wantTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, wantTokens)
}
state := stateFromRunOutput(t, out)
@@ -346,6 +357,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -376,10 +388,11 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
}
vec := floatSliceFromAny(t, chunks[i]["q_4_vec"])
wantEmbedText := strings.TrimSpace(wantText)
if len(vec) != 4 || vec[0] != float64(len(wantEmbedText)) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, float64(len(wantEmbedText)))
wantFirst := expectedFixedEmbedderFirst("", wantEmbedText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, wantFirst)
}
totalTokens += tokenizer.NumTokensFromString(wantText)
totalTokens += len(wantEmbedText)
}
if got := payload["embedding_token_consumption"]; got != totalTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, totalTokens)
@@ -410,7 +423,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
}
for i, wantText := range wantChunkTexts {
if got := chunkerChunks[i]["text"]; got != wantText {
t.Fatalf("chunker chunks[%d].text = %v, want %q", i, got, wantText)
t.Fatalf("chunker chunk[%d].text = %v, want %q", i, got, wantText)
}
}
}
@@ -441,6 +454,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -470,10 +484,11 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
}
vec := floatSliceFromAny(t, chunks[i]["q_4_vec"])
wantEmbedText := strings.TrimSpace(wantText)
if len(vec) != 4 || vec[0] != float64(len(wantEmbedText)) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, float64(len(wantEmbedText)))
wantFirst := expectedFixedEmbedderFirst("", wantEmbedText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, wantFirst)
}
totalTokens += tokenizer.NumTokensFromString(wantText)
totalTokens += len(wantEmbedText)
}
if got := payload["embedding_token_consumption"]; got != totalTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, totalTokens)
@@ -490,7 +505,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
}
for i, wantText := range wantChunkTexts {
if got := chunkerChunks[i]["text"]; got != wantText {
t.Fatalf("chunker chunks[%d].text = %v, want %q", i, got, wantText)
t.Fatalf("chunker chunk[%d].text = %v, want %q", i, got, wantText)
}
}
}
@@ -521,6 +536,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -548,10 +564,11 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
}
wantEmbedText := strings.TrimSpace(wantText)
vec := floatSliceFromAny(t, chunks[i]["q_4_vec"])
if len(vec) != 4 || vec[0] != float64(len(wantEmbedText)) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, float64(len(wantEmbedText)))
wantFirst := expectedFixedEmbedderFirst("", wantEmbedText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, wantFirst)
}
totalTokens += tokenizer.NumTokensFromString(wantText)
totalTokens += len(wantEmbedText)
}
if got := payload["embedding_token_consumption"]; got != totalTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, totalTokens)
@@ -568,7 +585,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
}
for i, wantText := range wantChunkTexts {
if got := chunkerChunks[i]["text"]; got != wantText {
t.Fatalf("chunker chunks[%d].text = %v, want %q", i, got, wantText)
t.Fatalf("chunker chunk[%d].text = %v, want %q", i, got, wantText)
}
}
}
@@ -599,6 +616,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -631,10 +649,11 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
}
wantEmbedText := strings.TrimSpace(wantText)
vec := floatSliceFromAny(t, chunks[i]["q_4_vec"])
if len(vec) != 4 || vec[0] != float64(len(wantEmbedText)) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, float64(len(wantEmbedText)))
wantFirst := expectedFixedEmbedderFirst("", wantEmbedText)
if len(vec) != 4 || !approxFloat(vec[0], wantFirst) {
t.Fatalf("chunks[%d].q_4_vec = %v, want first=%v", i, vec, wantFirst)
}
totalTokens += tokenizer.NumTokensFromString(wantText)
totalTokens += len(wantEmbedText)
}
if got := payload["embedding_token_consumption"]; got != totalTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, totalTokens)
@@ -651,7 +670,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
}
for i, wantText := range wantChunkTexts {
if got := chunkerChunks[i]["text"]; got != wantText {
t.Fatalf("chunker chunks[%d].text = %v, want %q", i, got, wantText)
t.Fatalf("chunker chunk[%d].text = %v, want %q", i, got, wantText)
}
}
}
@@ -718,6 +737,7 @@ func TestPipelineRun_TemplateResume_RealComponents(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
"llm_id": model + "@openai",
@@ -789,6 +809,7 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {
if err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
})
@@ -807,13 +828,20 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {
}
}
func repoRootFromPipelineTest(t *testing.T) string {
func attachFixedEmbedderFactory(t *testing.T, pipe *Pipeline) {
t.Helper()
_, file, _, ok := goruntime.Caller(0)
if !ok {
t.Fatal("runtime.Caller(0) failed")
}
return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
pipe.WithComponentFactory(func(name string, params map[string]any) (runtime.Component, error) {
if name == componentpkg.ComponentNameTokenizer {
return componentpkg.NewTokenizerComponentWithResolver(params, func(_, _, _ string) (componentpkg.Embedder, error) {
return fixedEmbedder{}, nil
})
}
factory, _, _, ok := runtime.DefaultRegistry.Lookup(name)
if !ok {
return nil, fmt.Errorf("runtime: unknown component %q", name)
}
return factory(name, params)
})
}
func withRealTemplateDeps(t *testing.T) storage.Storage {
@@ -824,10 +852,6 @@ func withRealTemplateDeps(t *testing.T) storage.Storage {
storage.GetStorageFactory().SetStorage(mem)
t.Cleanup(func() { storage.GetStorageFactory().SetStorage(origStorage) })
origEncode := componentpkg.EncodeFunc
componentpkg.EncodeFunc = func(_, _ string) componentpkg.Embedder { return fixedEmbedder{} }
t.Cleanup(func() { componentpkg.EncodeFunc = origEncode })
refs := map[string]componentpkg.DocumentStorageRef{}
componentpkg.ResolveDocumentStorageOverride = func(docID string) (*componentpkg.DocumentStorageRef, error) {
ref, ok := refs[docID]
@@ -1048,7 +1072,7 @@ func assertExtractedMetadataContains(t *testing.T, raw any, key, want string) {
}
}
func assertTokenizerTerminalChunk(t *testing.T, payload map[string]any, wantMergedText string) {
func assertTokenizerTerminalChunk(t *testing.T, payload map[string]any, name, wantMergedText string) {
t.Helper()
if got := payload["output_format"]; got != "chunks" {
@@ -1074,180 +1098,16 @@ func assertTokenizerTerminalChunk(t *testing.T, payload map[string]any, wantMerg
if len(vec) != 4 {
t.Fatalf("chunks[0].q_4_vec len = %d, want 4", len(vec))
}
if got := vec[0]; got != float64(len(wantMergedText)) {
t.Fatalf("chunks[0].q_4_vec[0] = %v, want %v", got, float64(len(wantMergedText)))
wantFirst := expectedFixedEmbedderFirst(name, wantMergedText)
if got := vec[0]; !approxFloat(got, wantFirst) {
t.Fatalf("chunks[0].q_4_vec[0] = %v, want %v", got, wantFirst)
}
wantTokens := tokenizer.NumTokensFromString(wantMergedText)
wantTokens := len(wantMergedText)
if got := payload["embedding_token_consumption"]; got != wantTokens {
t.Fatalf("embedding_token_consumption = %v, want %d", got, wantTokens)
}
}
func stateFromRunOutput(t *testing.T, out map[string]any) map[string]map[string]any {
t.Helper()
state, ok := out["state"].(map[string]map[string]any)
if !ok {
t.Fatalf("state = %T, want map[string]map[string]any", out["state"])
}
return state
}
func requireTokenizerPool(t *testing.T) {
t.Helper()
if tokenizer.IsInitialized() {
return
}
cfg := &tokenizer.PoolConfig{
DictPath: os.Getenv("RAGFLOW_DICT_PATH"),
MinSize: 1,
MaxSize: 2,
IdleTimeout: 30 * time.Second,
AcquireTimeout: 5 * time.Second,
}
if cfg.DictPath == "" {
cfg.DictPath = "/usr/share/infinity/resource"
}
if err := tokenizer.Init(cfg); err != nil {
t.Skipf("tokenizer pool init failed: %v", err)
}
}
func floatSliceFromAny(t *testing.T, v any) []float64 {
t.Helper()
switch x := v.(type) {
case []float64:
return x
case []any:
out := make([]float64, 0, len(x))
for i, item := range x {
f, ok := item.(float64)
if !ok {
t.Fatalf("vector item %d = %T, want float64", i, item)
}
out = append(out, f)
}
return out
default:
t.Fatalf("vector = %T, want []float64 or []any", v)
return nil
}
}
func joinJSONItemTexts(items []map[string]any) string {
var parts []string
for _, item := range items {
text, _ := item["text"].(string)
text = strings.TrimSpace(text)
if text == "" {
continue
}
parts = append(parts, text)
}
return strings.Join(parts, "\n")
}
func assertNormalizedContainsAll(t *testing.T, got string, wantSubstrings ...string) {
t.Helper()
normalizedGot := normalizeTestText(got)
for _, want := range wantSubstrings {
if !strings.Contains(normalizedGot, normalizeTestText(want)) {
t.Fatalf("normalized text %q does not contain %q", normalizedGot, normalizeTestText(want))
}
}
}
func normalizeTestText(s string) string {
return strings.Join(strings.Fields(strings.ReplaceAll(s, "\u00ad", "")), " ")
}
func terminalPayloadFromRunOutput(t *testing.T, out map[string]any, terminalID string) map[string]any {
t.Helper()
if out == nil {
t.Fatal("Run returned nil output")
}
if _, ok := out["output_format"]; ok {
return out
}
if terminalID == "" {
t.Fatal("terminalID is empty")
}
nested, ok := out[terminalID].(map[string]any)
if !ok {
t.Fatalf("run output missing terminal payload %q in %v", terminalID, out)
}
return nested
}
func terminalComponentIDsFromTemplate(t *testing.T, raw []byte) []string {
t.Helper()
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
t.Fatalf("unmarshal template: %v", err)
}
dsl, ok := tpl["dsl"].(map[string]any)
if !ok {
t.Fatalf("template dsl = %T, want map[string]any", tpl["dsl"])
}
components, ok := dsl["components"].(map[string]any)
if !ok {
t.Fatalf("template components = %T, want map[string]any", dsl["components"])
}
var terminals []string
for id, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
t.Fatalf("component %q = %T, want map[string]any", id, rawComp)
}
switch ds := comp["downstream"].(type) {
case nil:
terminals = append(terminals, id)
case []any:
if len(ds) == 0 {
terminals = append(terminals, id)
}
case []string:
if len(ds) == 0 {
terminals = append(terminals, id)
}
default:
t.Fatalf("component %q downstream = %T, want []any/[]string/nil", id, comp["downstream"])
}
}
sort.Strings(terminals)
return terminals
}
func templateUsesComponent(t *testing.T, raw []byte, componentName string) bool {
t.Helper()
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
t.Fatalf("unmarshal template: %v", err)
}
dsl, ok := tpl["dsl"].(map[string]any)
if !ok {
t.Fatalf("template dsl = %T, want map[string]any", tpl["dsl"])
}
components, ok := dsl["components"].(map[string]any)
if !ok {
t.Fatalf("template components = %T, want map[string]any", dsl["components"])
}
for id, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
t.Fatalf("component %q = %T, want map[string]any", id, rawComp)
}
obj, ok := comp["obj"].(map[string]any)
if !ok {
t.Fatalf("component %q obj = %T, want map[string]any", id, comp["obj"])
}
name, _ := obj["component_name"].(string)
if name == componentName {
return true
}
}
return false
}
func TestTemplateFixtures_AreWrappedTemplates(t *testing.T) {
path := filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "ingestion_pipeline_one.json")
raw, err := os.ReadFile(path)

211
internal/ingestion/pipeline/test_helpers.go Normal file
View File

@@ -0,0 +1,211 @@
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
import (
"encoding/json"
"math"
"os"
"path/filepath"
goruntime "runtime"
"sort"
"strings"
"testing"
"time"
"ragflow/internal/tokenizer"
)
func repoRootFromPipelineTest(t *testing.T) string {
t.Helper()
_, file, _, ok := goruntime.Caller(0)
if !ok {
t.Fatal("runtime.Caller(0) failed")
}
return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
}
func RequireTokenizerPool(t *testing.T) {
t.Helper()
if tokenizer.IsInitialized() {
return
}
cfg := &tokenizer.PoolConfig{
DictPath: os.Getenv("RAGFLOW_DICT_PATH"),
MinSize: 1,
MaxSize: 2,
IdleTimeout: 30 * time.Second,
AcquireTimeout: 5 * time.Second,
}
if cfg.DictPath == "" {
cfg.DictPath = "/usr/share/infinity/resource"
}
if err := tokenizer.Init(cfg); err != nil {
t.Skipf("tokenizer pool init failed: %v", err)
}
}
func terminalComponentIDsFromTemplate(t *testing.T, raw []byte) []string {
t.Helper()
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
t.Fatalf("unmarshal template: %v", err)
}
dsl, ok := tpl["dsl"].(map[string]any)
if !ok {
t.Fatalf("template dsl = %T, want map[string]any", tpl["dsl"])
}
components, ok := dsl["components"].(map[string]any)
if !ok {
t.Fatalf("template components = %T, want map[string]any", dsl["components"])
}
var terminals []string
for id, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
t.Fatalf("component %q = %T, want map[string]any", id, rawComp)
}
switch ds := comp["downstream"].(type) {
case nil:
terminals = append(terminals, id)
case []any:
if len(ds) == 0 {
terminals = append(terminals, id)
}
case []string:
if len(ds) == 0 {
terminals = append(terminals, id)
}
default:
t.Fatalf("component %q downstream = %T, want []any/[]string/nil", id, comp["downstream"])
}
}
sort.Strings(terminals)
return terminals
}
func terminalPayloadFromRunOutput(t *testing.T, out map[string]any, terminalID string) map[string]any {
t.Helper()
if out == nil {
t.Fatal("Run returned nil output")
}
if _, ok := out["output_format"]; ok {
return out
}
if terminalID == "" {
t.Fatal("terminalID is empty")
}
nested, ok := out[terminalID].(map[string]any)
if !ok {
t.Fatalf("run output missing terminal payload %q in %v", terminalID, out)
}
return nested
}
func stateFromRunOutput(t *testing.T, out map[string]any) map[string]map[string]any {
t.Helper()
state, ok := out["state"].(map[string]map[string]any)
if !ok {
t.Fatalf("state = %T, want map[string]map[string]any", out["state"])
}
return state
}
func floatSliceFromAny(t *testing.T, v any) []float64 {
t.Helper()
switch x := v.(type) {
case []float64:
return x
case []any:
out := make([]float64, 0, len(x))
for i, item := range x {
f, ok := item.(float64)
if !ok {
t.Fatalf("vector item %d = %T, want float64", i, item)
}
out = append(out, f)
}
return out
default:
t.Fatalf("vector = %T, want []float64 or []any", v)
return nil
}
}
func joinJSONItemTexts(items []map[string]any) string {
var parts []string
for _, item := range items {
text, _ := item["text"].(string)
text = strings.TrimSpace(text)
if text == "" {
continue
}
parts = append(parts, text)
}
return strings.Join(parts, "\n")
}
func assertNormalizedContainsAll(t *testing.T, got string, wantSubstrings ...string) {
t.Helper()
normalizedGot := normalizeTestText(got)
for _, want := range wantSubstrings {
if !strings.Contains(normalizedGot, normalizeTestText(want)) {
t.Fatalf("normalized text %q does not contain %q", normalizedGot, normalizeTestText(want))
}
}
}
func normalizeTestText(s string) string {
return strings.Join(strings.Fields(strings.ReplaceAll(s, "­", "")), " ")
}
func templateUsesComponent(t *testing.T, raw []byte, componentName string) bool {
t.Helper()
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
t.Fatalf("unmarshal template: %v", err)
}
dsl, ok := tpl["dsl"].(map[string]any)
if !ok {
t.Fatalf("template dsl = %T, want map[string]any", tpl["dsl"])
}
components, ok := dsl["components"].(map[string]any)
if !ok {
t.Fatalf("template components = %T, want map[string]any", dsl["components"])
}
for id, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
t.Fatalf("component %q = %T, want map[string]any", id, rawComp)
}
obj, ok := comp["obj"].(map[string]any)
if !ok {
t.Fatalf("component %q obj = %T, want map[string]any", id, comp["obj"])
}
name, _ := obj["component_name"].(string)
if name == componentName {
return true
}
}
return false
}
func expectedFixedEmbedderFirst(name, text string) float64 {
return 0.1*float64(len(name)) + 0.9*float64(len(text))
}
func approxFloat(got, want float64) bool {
return math.Abs(got-want) < 1e-9
}

62
internal/ingestion/service/execute_task_test.go
View File

@@ -11,6 +11,64 @@ import (
"ragflow/internal/ingestion/testutil"
)
// TestExecuteTask_CheckpointParseFailureDoesNotKillProcess verifies that checkpoint
// parse failures do not call fatal exit (which would kill the whole worker process).
// Instead, the task should be marked as FAILED and return gracefully.
// This tests the fix for issue 1 from the code review.
func TestExecuteTask_CheckpointParseFailureDoesNotKillProcess(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
_, _, docID, taskID := testutil.SeedTestData(t, db,
testutil.WithPipelineID("flow-1"),
testutil.WithTenantID("tenant-1"),
)
// Create a task log with invalid checkpoint (current_step is a string instead of number)
err := db.Create(&entity.IngestionTaskLog{
TaskID: taskID,
Checkpoint: entity.JSONMap{
"current_step": "not-a-number", // intentionally wrong type
"total_step": 5,
},
}).Error
if err != nil {
t.Fatalf("create bad task log: %v", err)
}
ingestor := NewIngestor("test", 1, []string{"pdf"})
// Replace runDocumentTask to ensure it doesn't get called
var runDocumentTaskCalled bool
ingestor.runDocumentTask = func(ctx context.Context, ingestionTask *entity.IngestionTask) error {
runDocumentTaskCalled = true
return nil
}
taskCtx := taskpkg.NewTaskContextForScheduling(
context.Background(),
&entity.IngestionTask{ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING},
nil,
)
// Execute the task - this should NOT panic or fatal exit (this is our main validation!)
ingestor.executeTask(taskCtx)
// Verify runDocumentTask was not called (we returned early due to checkpoint parse failure)
if runDocumentTaskCalled {
t.Fatal("expected runDocumentTask to not be called due to checkpoint parse failure")
}
// Verify task status was set to FAILED
finalTask, err := dao.NewIngestionTaskDAO().GetByID(taskID)
if err != nil {
t.Fatalf("load final ingestion task: %v", err)
}
if finalTask.Status != common.FAILED {
t.Fatalf("final status = %s, want %s", finalTask.Status, common.FAILED)
}
}
func TestExecuteTask_DataflowRoutesToTaskHandler(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
@@ -37,10 +95,6 @@ func TestExecuteTask_DataflowRoutesToTaskHandler(t *testing.T) {
wrapped(1.0, "mock dataflow done")
return nil
}
ingestor.storageImpl = testutil.NewMockStorage(map[string][]byte{"/unused": []byte("unused")})
ingestor.documentDAO = &testutil.MockDocDAO{
Docs: map[string]*entity.Document{"doc-1": testutil.TestDoc("doc-1", "pdf", ".pdf")},
}
taskCtx := taskpkg.NewTaskContextForScheduling(
context.Background(),

86
internal/ingestion/service/ingestion_service.go
View File

@@ -30,32 +30,15 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/engine"
"ragflow/internal/entity"
taskpkg "ragflow/internal/ingestion/task"
"ragflow/internal/storage"
"github.com/google/uuid"
"google.golang.org/grpc"
"gorm.io/gorm"
)
// pdfParser is the interface for PDF parsing, made injectable for tests.
type pdfParser interface {
ParseWithDeepDoc(ctx context.Context, filename string, data []byte, config doctype.ParserConfig) ([]map[string]any, error)
}
// DAO interfaces — injectable for tests.
type docGetter interface {
GetByID(id string) (*entity.Document, error)
}
type f2dGetter interface {
GetByDocumentID(docID string) ([]*entity.File2Document, error)
}
type fileGetter interface {
GetByID(id string) (*entity.File, error)
}
type Ingestor struct {
id string
name string
@@ -87,13 +70,6 @@ type Ingestor struct {
ingestionTaskletDAO *dao.IngestionTaskletDAO
ingestionTaskletLogDAO *dao.IngestionTaskletLogDAO
// Injected dependencies for parseDocument (overridable in tests)
documentDAO docGetter
f2dDAO f2dGetter
fileDAO fileGetter
storageImpl storage.Storage
pdfParser pdfParser
// runDocumentTask dispatches to the migrated task handler path.
// Tests may override this to verify branch routing without invoking
// the full downstream stack.
@@ -248,6 +224,13 @@ func (e *Ingestor) executeTask(taskCtx *taskpkg.TaskContext) {
latestLog, err := e.ingestionTaskLogDAO.LatestLogByTaskID(task.ID)
if err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
common.Error(fmt.Sprintf("Failed to get latest task log for task %s", task.ID), err)
if uErr := e.ingestionTaskDAO.UpdateStatus(task.ID, common.FAILED); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", task.ID), uErr)
}
return
}
latestLog = &entity.IngestionTaskLog{
ID: 0,
TaskID: task.ID,
@@ -259,6 +242,9 @@ func (e *Ingestor) executeTask(taskCtx *taskpkg.TaskContext) {
err = e.ingestionTaskLogDAO.Create(latestLog)
if err != nil {
common.Error(fmt.Sprintf("Failed to create task log for task %s", task.ID), err)
if uErr := e.ingestionTaskDAO.UpdateStatus(task.ID, common.FAILED); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", task.ID), uErr)
}
return
}
}
@@ -267,12 +253,18 @@ func (e *Ingestor) executeTask(taskCtx *taskpkg.TaskContext) {
checkpointMap = latestLog.Checkpoint
currentStep, ok := common.GetInt(checkpointMap["current_step"])
if !ok {
common.Fatal(fmt.Sprintf("Failed to get current step from task log for task %s", task.ID))
common.Error(fmt.Sprintf("Failed to get current step from task log for task %s", task.ID), nil)
if uErr := e.ingestionTaskDAO.UpdateStatus(task.ID, common.FAILED); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", task.ID), uErr)
}
return
}
totalStep, ok := common.GetInt(checkpointMap["total_step"])
if !ok {
common.Fatal(fmt.Sprintf("Failed to get current step from task log for task %s", task.ID))
common.Error(fmt.Sprintf("Failed to get total step from task log for task %s", task.ID), nil)
if uErr := e.ingestionTaskDAO.UpdateStatus(task.ID, common.FAILED); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", task.ID), uErr)
}
return
}
// Bump checkpoint to signal we started processing
@@ -322,11 +314,6 @@ func (e *Ingestor) getPipelineID(tenantID string) (string, error) {
return "", err
}
templateBytes, err = disableTokenizerEmbeddingForTaskTemplate(templateBytes)
if err != nil {
return "", err
}
var templateDSL entity.JSONMap
if err := json.Unmarshal(templateBytes, &templateDSL); err != nil {
return "", err
@@ -345,41 +332,6 @@ func (e *Ingestor) getPipelineID(tenantID string) (string, error) {
return ID, nil
}
func disableTokenizerEmbeddingForTaskTemplate(raw []byte) ([]byte, error) {
var tpl map[string]any
if err := json.Unmarshal(raw, &tpl); err != nil {
return nil, fmt.Errorf("failed to unmarshal")
}
dsl, ok := tpl["dsl"].(map[string]any)
if !ok {
return nil, fmt.Errorf("failed to do dsl convert")
}
components, ok := dsl["components"].(map[string]any)
if !ok {
return nil, fmt.Errorf("template components = %T, want map[string]any", dsl["components"])
}
for _, rawComp := range components {
comp, ok := rawComp.(map[string]any)
if !ok {
continue
}
obj, ok := comp["obj"].(map[string]any)
if !ok || obj["component_name"] != "Tokenizer" {
continue
}
params, ok := obj["params"].(map[string]any)
if !ok {
continue
}
params["search_method"] = []string{"full_text"}
}
out, err := json.Marshal(tpl)
if err != nil {
return nil, fmt.Errorf("marshal modified template: %v", err)
}
return out, nil
}
func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *entity.IngestionTask) error {
docTaskCtx, err := taskpkg.LoadFromIngestionTask(ingestionTask)
if err != nil {

26
internal/ingestion/service/real_consumer_dataflow_test.go
View File

@@ -1,5 +1,5 @@
//go:build integration
// +build integration
//go:build manual
// +build manual
package service
@@ -13,6 +13,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine/nats"
"ragflow/internal/entity"
taskpkg "ragflow/internal/ingestion/task"
"ragflow/internal/ingestion/testutil"
)
@@ -98,21 +99,22 @@ func TestRealConsumer_DataflowMessageRoutesToExecuteTask(t *testing.T) {
ingestor := NewIngestor("queue-test", 1, []string{"pdf"})
var routedToDataflow bool
var progressEvents []string
taskCtx := taskpkg.NewTaskContextForScheduling(
context.Background(),
task,
taskHandle,
)
taskCtx.ProgressFunc = func(prog float64, msg string) {
taskCtx.Progress = int32(prog * 100)
progressEvents = append(progressEvents, msg)
}
ingestor.runDocumentTask = func(ctx context.Context, ingestionTask *entity.IngestionTask) error {
routedToDataflow = true
progressFn(0.82, "mock queue dataflow start")
progressFn(1.0, "mock queue dataflow done")
progressEvents = append(progressEvents, "0.82:mock queue dataflow start", "1.00:mock queue dataflow done")
taskCtx.ProgressFunc(0.82, "mock queue dataflow start")
taskCtx.ProgressFunc(1.0, "mock queue dataflow done")
return nil
}
taskCtx := &TaskContext{
Ctx: context.Background(),
CancelFunc: func() {},
Task: task,
TaskHandle: taskHandle,
}
ingestor.executeTask(taskCtx)
if !routedToDataflow {

25
internal/ingestion/service/test_helpers.go
View File

@@ -16,8 +16,6 @@
package service
import "ragflow/internal/storage"
// ──────────────────────────────────────────────────────────
// Ingestor Test Helpers
// ──────────────────────────────────────────────────────────
@@ -32,29 +30,6 @@ func NewTestIngestor() *Ingestor {
// IngestorOption configures a test Ingestor.
type IngestorOption func(*Ingestor)
// WithMockStorage sets the storageImpl to the given mock.
func WithMockStorage(mock storage.Storage) IngestorOption {
return func(i *Ingestor) {
i.storageImpl = mock
}
}
// WithMockPDFParser sets the pdfParser to the given mock.
func WithMockPDFParser(mock pdfParser) IngestorOption {
return func(i *Ingestor) {
i.pdfParser = mock
}
}
// WithMockDAOs sets the documentDAO, f2dDAO, and fileDAO.
func WithMockDAOs(docDAO docGetter, f2dDAO f2dGetter, fileDAO fileGetter) IngestorOption {
return func(i *Ingestor) {
i.documentDAO = docDAO
i.f2dDAO = f2dDAO
i.fileDAO = fileDAO
}
}
// SetupTestIngestor creates a new test Ingestor with the given options.
func SetupTestIngestor(t testingT, opts ...IngestorOption) *Ingestor {
t.Helper()

138
internal/ingestion/task/chunk_builder.go
View File

@@ -17,152 +17,14 @@
package task
import (
"encoding/base64"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/cespare/xxhash/v2"
)
// PAGERANK_FLD is the field key for pagerank feature value on a chunk,
// matching Python common.constants.PAGERANK_FLD = "pagerank_fea".
const PAGERANK_FLD = "pagerank_fea"
// Chunk represents a single document chunk ready for embedding and indexing.
// Mirrors the Python dict format produced by ChunkService._prepare_docs_and_upload().
type Chunk struct {
ID string `json:"id"` // xxhash64(content_with_weight + doc_id) hex
DocID string `json:"doc_id"` // document ID
KBID string `json:"kb_id"` // knowledge base ID
DocNameKwd string `json:"docnm_kwd"` // document file name
ContentWithWeight string `json:"content_with_weight"` // main text content
CreateTime string `json:"create_time"` // "2006-01-02 15:04:05"
CreateTimestamp float64 `json:"create_timestamp_flt"` // Unix timestamp
ImgID string `json:"img_id"` // image ID (from MinIO or inline)
PageNumInt []int `json:"page_num_int"` // page numbers
TopInt []int `json:"top_int"` // top positions
AvailableInt int `json:"available_int"` // availability flag, default 1
PageRank int `json:"pagerank_fea"` // pagerank feature value, 0 = unset
}
// prepareDocs converts ParseWithDeepDoc output ([]map[string]any) to a list of
// Chunk structs. Matches Python ChunkService._prepare_docs_and_upload().
//
// Input sections come from ParseWithDeepDoc; each map may contain:
// - "text" (string) — the section text → ContentWithWeight
// - "doc_type_kwd" (string) — document type hint (unused directly in chunk)
// - "img_id" (string) — inline image ID (empty string if none)
// - "positions" ([]float64) — [pageNum, top, left, width, height]
//
// This is a pure function with no I/O side effects. Image upload to MinIO
// is handled separately by the caller.
func PrepareDocs(sections []map[string]any, docID, kbID, docName string, pagerank int) []Chunk {
if len(sections) == 0 {
return nil
}
now := time.Now()
createTime := now.Format("2006-01-02 15:04:05")
createTimestamp := float64(now.Unix())
chunks := make([]Chunk, 0, len(sections))
for _, sec := range sections {
text, _ := sec["text"].(string)
c := Chunk{
DocID: docID,
KBID: kbID,
DocNameKwd: docName,
ContentWithWeight: text,
CreateTime: createTime,
CreateTimestamp: createTimestamp,
AvailableInt: 1,
PageRank: pagerank,
}
// Extract img_id if present
if imgID, ok := sec["img_id"].(string); ok {
c.ImgID = imgID
}
// Extract positions: [pageNum, top, left, width, height]
if positions, ok := sec["positions"].([]float64); ok && len(positions) >= 2 {
c.PageNumInt = append(c.PageNumInt, int(positions[0]))
c.TopInt = append(c.TopInt, int(positions[1]))
}
// Generate deterministic chunk ID: xxhash64(content_with_weight + doc_id) → hex
c.ID = ChunkID(text, docID)
chunks = append(chunks, c)
}
return chunks
}
// ChunkID generates a deterministic chunk identifier matching Python's:
//
// xxhash.xxh64((content_with_weight + str(doc_id)).encode("utf-8", "surrogatepass")).hexdigest()
func ChunkID(contentWithWeight, docID string) string {
return fmt.Sprintf("%016x", xxhash.Sum64String(contentWithWeight+docID))
}
// String returns a human-readable summary of the chunk for logging.
func (c *Chunk) String() string {
preview := c.ContentWithWeight
if utf8.RuneCountInString(preview) > 80 {
preview = string([]rune(preview)[:80]) + "..."
}
return fmt.Sprintf("Chunk{id=%s doc=%s page=%v top=%v text=%q}",
c.ID, c.DocID, c.PageNumInt, c.TopInt, preview)
}
// UploadChunkImages uploads base64-encoded images from sections to storage
// and sets each chunk's ImgID to the resulting storage reference.
//
// Matches Python ChunkService._prepare_docs_and_upload() image upload logic:
// - If section has "img_id" pre-set → use it directly (skip upload)
// - If section has "image" (base64) → decode → upload via putFn → set ImgID
// - Otherwise → ImgID stays empty
//
// putFn is typically storage.Put(bucket, fnm, data) where bucket=kbID, fnm=chunkID.
// sections and chunks must correspond 1:1 by index.
func UploadChunkImages(sections []map[string]any, chunks []Chunk, kbID string, putFn func(bucket, fnm string, binary []byte) error) error {
for i := range chunks {
sec := sections[i]
// Already has img_id from parser — use as-is
if imgID, ok := sec["img_id"].(string); ok && imgID != "" {
chunks[i].ImgID = imgID
continue
}
// No image data — skip
imgRaw, ok := sec["image"].(string)
if !ok || imgRaw == "" {
continue
}
imgBytes, err := decodeChunkImagePayload(imgRaw)
if err != nil {
return fmt.Errorf("decode image for chunk %s: %w", chunks[i].ID, err)
}
// Upload to storage (bucket=kbID, fnm=chunkID)
if err := putFn(kbID, chunks[i].ID, imgBytes); err != nil {
return fmt.Errorf("upload image for chunk %s: %w", chunks[i].ID, err)
}
// Set ImgID to storage reference (matching Python f"{bucket}-{objname}")
chunks[i].ImgID = fmt.Sprintf("%s-%s", kbID, chunks[i].ID)
}
return nil
}
func decodeChunkImagePayload(raw string) ([]byte, error) {
if idx := strings.Index(raw, ","); strings.HasPrefix(raw, "data:image/") && idx >= 0 {
raw = raw[idx+1:]
}
return base64.StdEncoding.DecodeString(raw)
}

417
internal/ingestion/task/chunk_builder_test.go
View File

@@ -17,398 +17,13 @@
package task
import (
"encoding/base64"
"errors"
"regexp"
"testing"
)
const (
testDocID = "doc-1"
testKBID = "kb-1"
testDocName = "test.pdf"
testDocID = "doc-1"
)
func TestPrepareDocs_Basic(t *testing.T) {
sections := []map[string]any{
{"text": "Hello World", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
c := chunks[0]
if c.ContentWithWeight != "Hello World" {
t.Errorf("ContentWithWeight = %q, want %q", c.ContentWithWeight, "Hello World")
}
if c.DocID != testDocID {
t.Errorf("DocID = %q, want %q", c.DocID, testDocID)
}
if c.KBID != testKBID {
t.Errorf("KBID = %q, want %q", c.KBID, testKBID)
}
if c.DocNameKwd != testDocName {
t.Errorf("DocNameKwd = %q, want %q", c.DocNameKwd, testDocName)
}
if c.AvailableInt != 1 {
t.Errorf("AvailableInt = %d, want 1", c.AvailableInt)
}
}
func TestPrepareDocs_Multiple(t *testing.T) {
sections := []map[string]any{
{"text": "First", "doc_type_kwd": "text", "img_id": ""},
{"text": "Second", "doc_type_kwd": "text", "img_id": ""},
{"text": "Third", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
if len(chunks) != 3 {
t.Fatalf("expected 3 chunks, got %d", len(chunks))
}
// All IDs should be different (different content)
ids := make(map[string]bool)
for _, c := range chunks {
if c.ContentWithWeight == "" {
t.Error("chunk has empty ContentWithWeight")
}
ids[c.ID] = true
}
if len(ids) != 3 {
t.Errorf("expected 3 unique IDs, got %d", len(ids))
}
}
func TestPrepareDocs_EmptyInput(t *testing.T) {
chunks := PrepareDocs(nil, testDocID, testKBID, testDocName, 0)
if chunks != nil {
t.Errorf("expected nil for empty input, got %d chunks", len(chunks))
}
chunks = PrepareDocs([]map[string]any{}, testDocID, testKBID, testDocName, 0)
if chunks != nil {
t.Errorf("expected nil for empty input, got %d chunks", len(chunks))
}
}
func TestPrepareDocs_IDFormat(t *testing.T) {
sections := []map[string]any{
{"text": "Content for ID test", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
id := chunks[0].ID
// xxhash64 hex digest is 16 hex chars (64 bits = 16 hex digits)
matched, _ := regexp.MatchString(`^[0-9a-f]{16}$`, id)
if !matched {
t.Errorf("ID %q does not match hex format", id)
}
}
func TestPrepareDocs_IDDeterministic(t *testing.T) {
sections := []map[string]any{
{"text": "Deterministic test content", "doc_type_kwd": "text", "img_id": ""},
}
chunks1 := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
chunks2 := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
if chunks1[0].ID != chunks2[0].ID {
t.Errorf("ID not deterministic: %q vs %q", chunks1[0].ID, chunks2[0].ID)
}
// Different docID should produce different ID
chunks3 := PrepareDocs(sections, "doc-2", testKBID, testDocName, 0)
if chunks1[0].ID == chunks3[0].ID {
t.Error("different docIDs produced the same chunk ID")
}
}
func TestPrepareDocs_Timestamps(t *testing.T) {
sections := []map[string]any{
{"text": "Timestamp test", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
c := chunks[0]
// CreateTime should match the expected format
matched, _ := regexp.MatchString(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$`, c.CreateTime)
if !matched {
t.Errorf("CreateTime %q does not match format YYYY-MM-DD HH:MM:SS", c.CreateTime)
}
if c.CreateTimestamp <= 0 {
t.Errorf("CreateTimestamp = %f, want > 0", c.CreateTimestamp)
}
}
func TestPrepareDocs_Positions(t *testing.T) {
sections := []map[string]any{
{
"text": "Positioned content",
"doc_type_kwd": "text",
"img_id": "",
"positions": []float64{3, 150.5, 20, 400, 60},
},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
c := chunks[0]
if len(c.PageNumInt) != 1 || c.PageNumInt[0] != 3 {
t.Errorf("PageNumInt = %v, want [3]", c.PageNumInt)
}
if len(c.TopInt) != 1 || c.TopInt[0] != 150 {
t.Errorf("TopInt = %v, want [150]", c.TopInt)
}
}
func TestPrepareDocs_NoPositions(t *testing.T) {
sections := []map[string]any{
{"text": "No positions", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
c := chunks[0]
if c.PageNumInt != nil {
t.Errorf("PageNumInt = %v, want nil", c.PageNumInt)
}
if c.TopInt != nil {
t.Errorf("TopInt = %v, want nil", c.TopInt)
}
}
func TestPrepareDocs_ImgID(t *testing.T) {
sections := []map[string]any{
{
"text": "Content with image",
"doc_type_kwd": "text",
"img_id": "some_image_id_123",
},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
c := chunks[0]
if c.ImgID != "some_image_id_123" {
t.Errorf("ImgID = %q, want %q", c.ImgID, "some_image_id_123")
}
}
func TestPrepareDocs_EmptyImgID(t *testing.T) {
sections := []map[string]any{
{"text": "No image", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
c := chunks[0]
if c.ImgID != "" {
t.Errorf("ImgID = %q, want empty string", c.ImgID)
}
}
func TestPrepareDocs_MixedContent(t *testing.T) {
sections := []map[string]any{
{"text": "Page 1 content", "doc_type_kwd": "text", "img_id": "",
"positions": []float64{1, 0, 0, 100, 50}},
{"text": "Page 2 with image", "doc_type_kwd": "text", "img_id": "img_abc",
"positions": []float64{2, 30, 10, 200, 80}},
{"text": "No position, no image", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
if len(chunks) != 3 {
t.Fatalf("expected 3 chunks, got %d", len(chunks))
}
// First: has position
if len(chunks[0].PageNumInt) != 1 || chunks[0].PageNumInt[0] != 1 {
t.Errorf("chunk[0] PageNumInt = %v, want [1]", chunks[0].PageNumInt)
}
// Second: has position + img_id
if chunks[1].ImgID != "img_abc" {
t.Errorf("chunk[1] ImgID = %q, want %q", chunks[1].ImgID, "img_abc")
}
if len(chunks[1].PageNumInt) != 1 || chunks[1].PageNumInt[0] != 2 {
t.Errorf("chunk[1] PageNumInt = %v, want [2]", chunks[1].PageNumInt)
}
// Third: no position, no image
if chunks[2].PageNumInt != nil {
t.Errorf("chunk[2] PageNumInt should be nil, got %v", chunks[2].PageNumInt)
}
if chunks[2].ImgID != "" {
t.Errorf("chunk[2] ImgID should be empty, got %q", chunks[2].ImgID)
}
}
func TestPrepareDocs_PageRank(t *testing.T) {
sections := []map[string]any{
{"text": "Ranked content", "doc_type_kwd": "text", "img_id": ""},
}
// pagerank=0 → PageRank should be 0 (unset)
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
if chunks[0].PageRank != 0 {
t.Errorf("PageRank = %d, want 0", chunks[0].PageRank)
}
// pagerank=5 → PageRank should be 5
chunks = PrepareDocs(sections, testDocID, testKBID, testDocName, 5)
if chunks[0].PageRank != 5 {
t.Errorf("PageRank = %d, want 5", chunks[0].PageRank)
}
}
func TestUploadChunkImages_WithImage(t *testing.T) {
imgBase64 := base64.StdEncoding.EncodeToString([]byte("fake image"))
sections := []map[string]any{
{"text": "Has image", "doc_type_kwd": "text", "img_id": "", "image": imgBase64},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
var putBucket, putFnm string
putFn := func(bucket, fnm string, binary []byte) error {
putBucket = bucket
putFnm = fnm
if len(binary) == 0 {
t.Error("Put called with empty binary")
}
return nil
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err != nil {
t.Fatalf("UploadChunkImages failed: %v", err)
}
expectedImgID := testKBID + "-" + chunks[0].ID
if chunks[0].ImgID != expectedImgID {
t.Errorf("ImgID = %q, want %q", chunks[0].ImgID, expectedImgID)
}
if putBucket != testKBID {
t.Errorf("put bucket = %q, want %q", putBucket, testKBID)
}
if putFnm != chunks[0].ID {
t.Errorf("put fnm = %q, want %q", putFnm, chunks[0].ID)
}
}
func TestUploadChunkImages_PresetImgID(t *testing.T) {
sections := []map[string]any{
{"text": "Has preset img_id", "doc_type_kwd": "text", "img_id": "preset-id-123", "image": "should-be-ignored"},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
callCount := 0
putFn := func(bucket, fnm string, binary []byte) error {
callCount++
return nil
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err != nil {
t.Fatalf("UploadChunkImages failed: %v", err)
}
if chunks[0].ImgID != "preset-id-123" {
t.Errorf("ImgID = %q, want %q", chunks[0].ImgID, "preset-id-123")
}
if callCount != 0 {
t.Errorf("expected 0 Put calls (preset img_id), got %d", callCount)
}
}
func TestUploadChunkImages_NoImage(t *testing.T) {
sections := []map[string]any{
{"text": "No image", "doc_type_kwd": "text", "img_id": ""},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
callCount := 0
putFn := func(bucket, fnm string, binary []byte) error {
callCount++
return nil
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err != nil {
t.Fatalf("UploadChunkImages failed: %v", err)
}
if chunks[0].ImgID != "" {
t.Errorf("ImgID = %q, want empty string", chunks[0].ImgID)
}
if callCount != 0 {
t.Errorf("expected 0 Put calls, got %d", callCount)
}
}
func TestUploadChunkImages_Mixed(t *testing.T) {
imgBase64 := base64.StdEncoding.EncodeToString([]byte("img1"))
sections := []map[string]any{
{"text": "Img1", "doc_type_kwd": "text", "img_id": "", "image": imgBase64},
{"text": "No img", "doc_type_kwd": "text", "img_id": ""},
{"text": "Preset", "doc_type_kwd": "text", "img_id": "preset-001"},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
var callCount int
putFn := func(bucket, fnm string, binary []byte) error {
callCount++
return nil
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err != nil {
t.Fatalf("UploadChunkImages failed: %v", err)
}
if callCount != 1 {
t.Errorf("expected 1 Put call, got %d", callCount)
}
if chunks[0].ImgID != testKBID+"-"+chunks[0].ID {
t.Errorf("chunk[0] ImgID = %q, want %q", chunks[0].ImgID, testKBID+"-"+chunks[0].ID)
}
if chunks[1].ImgID != "" {
t.Errorf("chunk[1] ImgID = %q, want empty", chunks[1].ImgID)
}
if chunks[2].ImgID != "preset-001" {
t.Errorf("chunk[2] ImgID = %q, want 'preset-001'", chunks[2].ImgID)
}
}
func TestUploadChunkImages_InvalidBase64(t *testing.T) {
sections := []map[string]any{
{"text": "Bad image", "doc_type_kwd": "text", "img_id": "", "image": "!!!not-base64!!!"},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
putFn := func(bucket, fnm string, binary []byte) error {
return nil
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err == nil {
t.Fatal("expected error for invalid base64, got nil")
}
}
func TestUploadChunkImages_PutError(t *testing.T) {
imgBase64 := base64.StdEncoding.EncodeToString([]byte("img"))
sections := []map[string]any{
{"text": "Put fails", "doc_type_kwd": "text", "img_id": "", "image": imgBase64},
}
chunks := PrepareDocs(sections, testDocID, testKBID, testDocName, 0)
putFn := func(bucket, fnm string, binary []byte) error {
return errors.New("storage unavailable")
}
if err := UploadChunkImages(sections, chunks, testKBID, putFn); err == nil {
t.Fatal("expected error for put failure, got nil")
}
}
func TestChunkID_NotPanic(t *testing.T) {
// Edge cases: empty content
_ = ChunkID("", testDocID)
@@ -429,33 +44,3 @@ func TestChunkID_PreservesLeadingZero(t *testing.T) {
t.Fatalf("ChunkID length = %d, want 16", len(got))
}
}
func TestChunk_String(t *testing.T) {
c := Chunk{
ID: "abc123",
DocID: "doc-1",
ContentWithWeight: "Short text",
PageNumInt: []int{1},
TopInt: []int{0},
}
s := c.String()
if len(s) == 0 {
t.Error("String() returned empty")
}
}
func TestChunk_StringTruncates(t *testing.T) {
long := ""
for i := 0; i < 200; i++ {
long += "x"
}
c := Chunk{
ID: "abc123",
DocID: "doc-1",
ContentWithWeight: long,
}
s := c.String()
if len(s) >= 200 {
t.Error("String() should truncate long content")
}
}

2
internal/ingestion/task/chunk_process.go
View File

@@ -243,4 +243,4 @@ func processChunkPositions(ck map[string]any) {
AddPositions(ck, positions)
}
delete(ck, "positions")
}
}

36
internal/ingestion/task/chunk_process_test.go
View File

@@ -362,39 +362,3 @@ func TestProcessChunksForDataflow_PreservesContentWithWeight(t *testing.T) {
t.Errorf("content_with_weight = %q, want \"already set\"", chunks[0]["content_with_weight"])
}
}
func TestProcessChunksForDataflow_MetadataExtraction(t *testing.T) {
chunks := []map[string]any{
{"text": "hello", "metadata": map[string]any{"author": "Alice", "year": "2024"}},
}
meta := ProcessChunksForDataflow(chunks, "doc-1", "kb-1", "test-doc.pdf", time.Now())
if _, exists := chunks[0]["metadata"]; exists {
t.Error("metadata key should be removed from chunk")
}
if meta["author"] != "Alice" {
t.Errorf("metadata[\"author\"] = %q, want \"Alice\"", meta["author"])
}
}
func TestProcessChunksForDataflow_MetadataMergeAcrossChunks(t *testing.T) {
chunks := []map[string]any{
{"text": "chunk1", "metadata": map[string]any{"author": "Alice"}},
{"text": "chunk2", "metadata": map[string]any{"year": "2024"}},
}
meta := ProcessChunksForDataflow(chunks, "doc-1", "kb-1", "test-doc.pdf", time.Now())
if meta["author"] != "Alice" {
t.Errorf("author = %q, want \"Alice\"", meta["author"])
}
if meta["year"] != "2024" {
t.Errorf("year = %v, want \"2024\"", meta["year"])
}
}
func TestProcessChunksForDataflow_NilChunks(t *testing.T) {
meta := ProcessChunksForDataflow(nil, "doc-1", "kb-1", "test-doc.pdf", time.Now())
if len(meta) != 0 {
t.Errorf("metadata should be empty for nil chunks, got %v", meta)
}
}

20
internal/ingestion/task/chunk_utils.go
View File

@@ -142,23 +142,3 @@ func MustGetChunkTextString(chunk map[string]any, where string) string {
common.Error(msg, nil)
panic(msg)
}
// AttachVectors attaches embedding vectors to chunks in-place.
// Each chunk gets a key like "q_{dim}_vec" with the vector as []float64.
// Mirrors Python: EmbeddingUtils.attach_vectors()
func AttachVectors(chunks []map[string]any, vectors [][]float64) int {
if len(chunks) == 0 && len(vectors) == 0 {
return 0
}
if len(vectors) != len(chunks) {
panic(fmt.Sprintf("vectors/chunks length mismatch: %d != %d", len(vectors), len(chunks)))
}
vectorSize := 0
for i, doc := range chunks {
vec := vectors[i]
vectorSize = len(vec)
key := fmt.Sprintf("q_%d_vec", vectorSize)
doc[key] = vec
}
return vectorSize
}

94
internal/ingestion/task/chunk_utils_test.go
View File

@@ -331,97 +331,3 @@ func TestMustGetChunkTextString_PanicOnStringSlice(t *testing.T) {
}()
_ = MustGetChunkTextString(chunk, "unit-test")
}
// =============================================================================
// AttachVectors
// =============================================================================
func TestAttachVectors_Basic(t *testing.T) {
chunks := []map[string]any{
{"text": "hello"},
{"text": "world"},
}
vectors := [][]float64{
{0.1, 0.2, 0.3},
{0.4, 0.5, 0.6},
}
dim := AttachVectors(chunks, vectors)
if dim != 3 {
t.Errorf("dim = %d, want 3", dim)
}
if len(chunks[0]) == 0 || chunks[0]["q_3_vec"] == nil {
t.Errorf("expected q_3_vec in chunk[0], got %v", chunks[0])
}
vec0 := chunks[0]["q_3_vec"].([]float64)
if len(vec0) != 3 || vec0[0] != 0.1 {
t.Errorf("chunk[0] vector = %v, want [0.1 0.2 0.3]", vec0)
}
vec1 := chunks[1]["q_3_vec"].([]float64)
if len(vec1) != 3 || vec1[2] != 0.6 {
t.Errorf("chunk[1] vector = %v, want [0.4 0.5 0.6]", vec1)
}
}
func TestAttachVectors_EmptyChunks(t *testing.T) {
result := AttachVectors(nil, nil)
if result != 0 {
t.Errorf("expected 0 for empty chunks, got %d", result)
}
}
func TestAttachVectors_DifferentDimensions(t *testing.T) {
chunks := []map[string]any{
{"text": "a"},
{"text": "b"},
}
vectors := [][]float64{
{0.1, 0.2},
{0.3, 0.4, 0.5},
}
// Each chunk gets its own vector with key based on its dimension
AttachVectors(chunks, vectors)
if chunks[0]["q_2_vec"] == nil {
t.Errorf("chunk[0] should have q_2_vec, got keys: %v", chunkKeys(chunks[0]))
}
if chunks[1]["q_3_vec"] == nil {
t.Errorf("chunk[1] should have q_3_vec, got keys: %v", chunkKeys(chunks[1]))
}
}
func TestAttachVectors_MismatchedCount(t *testing.T) {
chunks := []map[string]any{
{"text": "hello"},
}
vectors := [][]float64{
{0.1},
{0.2},
}
defer func() {
if r := recover(); r == nil {
t.Error("expected panic for mismatched count")
}
}()
AttachVectors(chunks, vectors)
}
func TestAttachVectors_KeyFormat(t *testing.T) {
chunks := []map[string]any{
{"text": "hello"},
}
vectors := [][]float64{
{0.1, 0.2, 0.3, 0.4, 0.5},
}
AttachVectors(chunks, vectors)
key := "q_5_vec"
if chunks[0][key] == nil {
t.Errorf("expected key %q in chunk, got keys: %v", key, chunkKeys(chunks[0]))
}
}
func chunkKeys(m map[string]any) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}

44
internal/ingestion/task/dataflow_real_pipeline_integration_test.go
View File

@@ -107,7 +107,12 @@ func TestDataflowService_Run_RealCanvasDSL_UsesGeneralPipeline(t *testing.T) {
})
taskCtx := &TaskContext{
Task: entity.Task{ID: "task-real-canvas-1", DocID: docID, TaskType: "dataflow"},
IngestionTask: &entity.IngestionTask{
ID: "task-real-canvas-1",
DocumentID: docID,
DatasetID: kbID,
},
TaskType: "dataflow",
Doc: entity.Document{
ID: docID,
KbID: kbID,
@@ -241,7 +246,12 @@ func TestDataflowService_Run_RealPDF_WritesAndReadsBackFromElasticsearch(t *test
})
taskCtx := &TaskContext{
Task: entity.Task{ID: "task-real-pdf-es-1", DocID: docID, TaskType: "dataflow"},
IngestionTask: &entity.IngestionTask{
ID: "task-real-pdf-es-1",
DocumentID: docID,
DatasetID: kbID,
},
TaskType: "dataflow",
Doc: entity.Document{
ID: docID,
KbID: kbID,
@@ -360,7 +370,12 @@ func TestRunDataflow_RealPipelineOutput_ProducesIndexFields(t *testing.T) {
pipelineOut = taskTerminalPayloadFromRunOutput(t, pipelineOut, "Tokenizer:LegalReadersDecide")
taskCtx := &TaskContext{
Task: entity.Task{ID: "task-real-1", DocID: docID},
IngestionTask: &entity.IngestionTask{
ID: "task-real-1",
DocumentID: docID,
DatasetID: kbID,
},
TaskType: "dataflow",
Doc: entity.Document{
ID: docID,
KbID: kbID,
@@ -507,25 +522,12 @@ func prepareTokenizerResourceForTaskIntegration(t *testing.T) {
if os.Getenv("RAGFLOW_DICT_PATH") != "" {
return
}
srcDir := filepath.Join(taskRepoRoot(t), ".venv", "lib", "python3.13", "site-packages", "infinity")
if _, err := os.Stat(filepath.Join(srcDir, "huqie.txt")); err != nil {
t.Skipf("tokenizer resource source not found at %s: %v", srcDir, err)
const systemDictPath = "/usr/share/infinity/resource"
if _, err := os.Stat(filepath.Join(systemDictPath, "rag", "huqie.txt")); err != nil {
t.Skipf("system tokenizer resource not found at %s: %v", systemDictPath, err)
}
if _, err := os.Stat(filepath.Join(srcDir, "huqie.txt.trie")); err != nil {
t.Skipf("tokenizer trie source not found at %s: %v", srcDir, err)
}
root := t.TempDir()
ragDir := filepath.Join(root, "rag")
if err := os.MkdirAll(ragDir, 0o755); err != nil {
t.Fatalf("mkdir rag tokenizer dir: %v", err)
}
taskMustSymlink(t, filepath.Join(srcDir, "huqie.txt"), filepath.Join(ragDir, "huqie.txt"))
taskMustSymlink(t, filepath.Join(srcDir, "huqie.txt.trie"), filepath.Join(ragDir, "huqie.trie"))
taskMustWriteTokenizerPOSDef(t, filepath.Join(srcDir, "huqie.txt"), filepath.Join(ragDir, "pos-id.def"))
taskMustPrepareTokenizerWordNet(t, root)
taskMustPrepareTokenizerOpenCC(t, root)
if err := os.Setenv("RAGFLOW_DICT_PATH", root); err != nil {
t.Fatalf("set RAGFLOW_DICT_PATH: %v", err)
if err := os.Setenv("RAGFLOW_DICT_PATH", systemDictPath); err != nil {
t.Fatalf("set RAGFLOW_DICT_PATH=%s: %v", systemDictPath, err)
}
t.Cleanup(func() {
_ = os.Unsetenv("RAGFLOW_DICT_PATH")

157
internal/ingestion/task/dataflow_service.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
"ragflow/internal/agent/runtime"
componentpkg "ragflow/internal/ingestion/component"
"ragflow/internal/utility"
"regexp"
@@ -40,15 +41,22 @@ type embedder struct {
model *models.EmbeddingModel
}
func (e *embedder) Encode(texts []string) ([][]float64, error) {
func (e *embedder) MaxTokens() int {
if e == nil || e.model == nil {
return 0
}
return e.model.MaxTokens
}
func (e *embedder) Encode(texts []string) ([]componentpkg.EmbeddingResult, error) {
config := &models.EmbeddingConfig{Dimension: 0}
embeds, err := e.model.ModelDriver.Embed(e.model.ModelName, texts, e.model.APIConfig, config)
if err != nil {
return nil, err
}
vecs := make([][]float64, len(embeds))
vecs := make([]componentpkg.EmbeddingResult, len(embeds))
for i, v := range embeds {
vecs[i] = v.Embedding
vecs[i] = componentpkg.EmbeddingResult{Vector: v.Embedding, TokenCount: v.TokenCount}
}
return vecs, nil
}
@@ -107,13 +115,14 @@ type PipelineExecutor struct {
docBulkSize int
progressFunc ProgressFunc
docSvc docService
chunkCounter chunkCounter
insertChunksFunc func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error)
logCreateFunc func(log *entity.PipelineOperationLog) error
getEmbeddingModelFunc func(tenantID, embdID string) (*models.EmbeddingModel, error)
loadDSLFunc func(ctx context.Context, dataflowID string) (string, string, error)
runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
docSvc docService
chunkCounter chunkCounter
insertChunksFunc func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error)
logCreateFunc func(log *entity.PipelineOperationLog) error
getEmbeddingModelFunc func(tenantID, embdID string) (*models.EmbeddingModel, error)
getKnowledgebaseByIDFunc func(kbID string) (*entity.Knowledgebase, error)
loadDSLFunc func(ctx context.Context, dataflowID string) (string, string, error)
runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
}
func validateDataflowTaskContext(taskCtx *TaskContext) error {
@@ -132,9 +141,6 @@ func validateDataflowTaskContext(taskCtx *TaskContext) error {
if taskCtx.KB.ID == "" {
return fmt.Errorf("dataflow service: empty knowledgebase id")
}
if taskCtx.KB.EmbdID == "" {
return fmt.Errorf("dataflow service: empty embedding model id")
}
if taskCtx.Tenant.ID == "" {
return fmt.Errorf("dataflow service: empty tenant id")
}
@@ -157,6 +163,7 @@ func NewDataflowService(
if taskCtx != nil && taskCtx.ProgressFunc != nil {
progressFn = taskCtx.ProgressFunc
}
modelProvider := service.NewModelProviderService()
svc := &PipelineExecutor{
taskCtx: taskCtx,
dataflowID: dataflowID,
@@ -168,8 +175,9 @@ func NewDataflowService(
insertChunksFunc: func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
return engine.Get().InsertChunks(ctx, chunks, baseName, datasetID)
},
logCreateFunc: dao.NewPipelineOperationLogDAO().Create,
getEmbeddingModelFunc: service.NewModelProviderService().GetEmbeddingModel,
logCreateFunc: dao.NewPipelineOperationLogDAO().Create,
getEmbeddingModelFunc: modelProvider.GetEmbeddingModel,
getKnowledgebaseByIDFunc: dao.NewKnowledgebaseDAO().GetByID,
}
svc.loadDSLFunc = svc.defaultLoadDSL
svc.runPipelineFunc = svc.defaultRunPipeline
@@ -300,59 +308,6 @@ func (s *PipelineExecutor) normalizeChunks(output map[string]any) []map[string]a
return NormalizeChunks(output)
}
func (s *PipelineExecutor) embedChunks(ctx context.Context, chunks []map[string]any, tokenConsumption int) ([]map[string]any, int, error) {
if len(chunks) == 0 {
return nil, 0, nil
}
s.progress(0.82, "\n-------------------------------------\nStart to embedding...")
model, err := s.getEmbeddingModel(s.taskCtx.Tenant.ID, s.taskCtx.KB.EmbdID)
if err != nil {
s.progress(-1, fmt.Sprintf("[ERROR]: %v", err))
return nil, tokenConsumption, err
}
texts := PrepareTextsForDataflowEmbedding(chunks)
batchSize := s.embeddingBatchSize
if batchSize <= 0 {
batchSize = 16
}
delta := 0.20 / float64(len(texts)/batchSize+1)
prog := 0.8
var allVects [][]float64
for i := 0; i < len(texts); i += batchSize {
end := i + batchSize
if end > len(texts) {
end = len(texts)
}
batch := texts[i:end]
if lim := s.taskCtx.EmbedLimiter; lim != nil {
if err := lim.Acquire(ctx, 1); err != nil {
s.progress(-1, fmt.Sprintf("[ERROR]: %v", err))
return nil, tokenConsumption, err
}
}
vecs, tc, err := encodeTexts(model, batch)
if err != nil {
if lim := s.taskCtx.EmbedLimiter; lim != nil {
lim.Release(1)
}
s.progress(-1, fmt.Sprintf("[ERROR]: %v", err))
return nil, tokenConsumption, err
}
if lim := s.taskCtx.EmbedLimiter; lim != nil {
lim.Release(1)
}
allVects = append(allVects, vecs...)
tokenConsumption += tc
prog += delta
s.progress(prog, fmt.Sprintf("%d / %d", i+1, len(texts)/batchSize))
}
if len(allVects) != len(chunks) {
panic(fmt.Sprintf("vector count mismatch: %d vs %d", len(allVects), len(chunks)))
}
AttachVectors(chunks, allVects)
return chunks, tokenConsumption, nil
}
func (s *PipelineExecutor) processChunks(chunks []map[string]any) map[string]any {
return ProcessChunksForDataflow(
chunks,
@@ -496,21 +451,41 @@ func (s *PipelineExecutor) defaultLoadDSL(ctx context.Context, dataflowID string
return string(raw), correctedID, nil
}
func (s *PipelineExecutor) tokenizerEmbedderResolver() componentpkg.EmbedderResolver {
return func(tenantID, kbID, modelID string) (componentpkg.Embedder, error) {
if strings.TrimSpace(tenantID) == "" && s != nil && s.taskCtx != nil {
tenantID = s.taskCtx.Tenant.ID
}
if strings.TrimSpace(kbID) == "" && s != nil && s.taskCtx != nil {
kbID = s.taskCtx.KB.ID
}
model, err := s.resolveEmbeddingModel(tenantID, kbID, modelID)
if err != nil {
return nil, err
}
return &embedder{model: model}, nil
}
}
func (s *PipelineExecutor) taskScopedComponentFactory() runtime.ComponentFactory {
resolver := s.tokenizerEmbedderResolver()
return func(name string, params map[string]any) (runtime.Component, error) {
if strings.EqualFold(name, componentpkg.ComponentNameTokenizer) {
return componentpkg.NewTokenizerComponentWithResolver(params, resolver)
}
factory, _, _, ok := runtime.DefaultRegistry.Lookup(name)
if !ok {
return nil, fmt.Errorf("runtime: unknown component %q", name)
}
return factory(name, params)
}
}
func (s *PipelineExecutor) defaultRunPipeline(ctx context.Context, dsl string) (map[string]any, string, error) {
if s == nil || s.taskCtx == nil {
return nil, dsl, fmt.Errorf("dataflow service: nil task context")
}
prevEncode := componentpkg.EncodeFunc
componentpkg.EncodeFunc = func(tenantID, embdID string) componentpkg.Embedder {
model, err := s.getEmbeddingModelFunc(tenantID, embdID)
if err != nil {
return nil
}
return &embedder{model: model}
}
defer func() { componentpkg.EncodeFunc = prevEncode }()
// Use doc ID as pipeline ID if available, otherwise a placeholder
pipelineID := "pipeline_" + s.taskCtx.Doc.ID
if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {
@@ -520,6 +495,7 @@ func (s *PipelineExecutor) defaultRunPipeline(ctx context.Context, dsl string) (
if err != nil {
return nil, dsl, fmt.Errorf("compile pipeline dsl: %w", err)
}
pipe.WithComponentFactory(s.taskScopedComponentFactory())
inputs := map[string]any{}
if s.taskCtx.Doc.ID != "" {
inputs["doc_id"] = s.taskCtx.Doc.ID
@@ -528,7 +504,7 @@ func (s *PipelineExecutor) defaultRunPipeline(ctx context.Context, dsl string) (
inputs["file"] = s.taskCtx.File
}
inputs["tenant_id"] = s.taskCtx.Tenant.ID
inputs["model_id"] = s.taskCtx.KB.EmbdID
inputs["kb_id"] = s.taskCtx.KB.ID
output, err := pipe.Run(ctx, inputs)
if err != nil {
@@ -541,6 +517,29 @@ func (s *PipelineExecutor) defaultRunPipeline(ctx context.Context, dsl string) (
return payload, dsl, nil
}
func (s *PipelineExecutor) resolveEmbeddingModel(tenantID, kbID, modelID string) (*models.EmbeddingModel, error) {
_ = modelID
if strings.TrimSpace(kbID) != "" {
if s.taskCtx != nil && s.taskCtx.KB.ID == kbID && strings.TrimSpace(s.taskCtx.KB.EmbdID) != "" {
return s.getEmbeddingModelFunc(tenantID, s.taskCtx.KB.EmbdID)
}
if s.getKnowledgebaseByIDFunc == nil {
return nil, fmt.Errorf("knowledgebase resolver unavailable")
}
kb, err := s.getKnowledgebaseByIDFunc(kbID)
if err != nil {
return nil, err
}
if kb != nil && strings.TrimSpace(kb.EmbdID) != "" {
return s.getEmbeddingModelFunc(tenantID, kb.EmbdID)
}
}
if s.taskCtx != nil && strings.TrimSpace(s.taskCtx.KB.EmbdID) != "" {
return s.getEmbeddingModelFunc(tenantID, s.taskCtx.KB.EmbdID)
}
return nil, fmt.Errorf("embedding requested but dataset has no embd_id configured")
}
func extractDataflowPipelinePayload(dsl string, out map[string]any) (map[string]any, error) {
if out == nil {
return nil, nil

212
internal/ingestion/task/dataflow_service_test.go
View File

@@ -3,7 +3,6 @@ package task
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
@@ -185,7 +184,6 @@ func TestNewDataflowService_RejectsIncompleteTaskContext(t *testing.T) {
{name: "missing kb id", mutate: func(ctx *TaskContext) { ctx.Doc.KbID = "" }},
{name: "missing doc name", mutate: func(ctx *TaskContext) { ctx.Doc.Name = nil }},
{name: "missing knowledgebase id", mutate: func(ctx *TaskContext) { ctx.KB.ID = "" }},
{name: "missing embedding model id", mutate: func(ctx *TaskContext) { ctx.KB.EmbdID = "" }},
{name: "missing tenant id", mutate: func(ctx *TaskContext) { ctx.Tenant.ID = "" }},
}
@@ -256,71 +254,6 @@ func TestDataflowService_ProcessChunks_WrapsProcessChunksForDataflow(t *testing.
}
}
// =============================================================================
// embedChunks — Python: _embed_chunks (line 247)
// =============================================================================
func TestEmbedChunks_Success(t *testing.T) {
stub := &stubDriver{}
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0).WithGetEmbeddingModelFunc(
func(tenantID, embdID string) (*models.EmbeddingModel, error) {
return makeTestEmbeddingModel(stub, 100), nil
},
)
chunks := []map[string]any{
{"text": "hello"},
{"text": "world"},
}
result, tc, err := svc.embedChunks(context.Background(), chunks, 5)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(result) != 2 {
t.Fatalf("len = %d, want 2", len(result))
}
// vectors should be attached (q_*_vec keys)
if result[0]["q_2_vec"] == nil {
t.Errorf("expected q_2_vec in chunk[0], got keys: %v", chunkKeys(result[0]))
}
// token consumption should include initial + new tokens
if tc < 5 {
t.Errorf("token consumption should be at least the initial 5, got %d", tc)
}
}
func TestEmbedChunks_EmptyChunks(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
result, _, err := svc.embedChunks(context.Background(), nil, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result != nil {
t.Errorf("expected nil for empty chunks, got %v", result)
}
}
func TestEmbedChunks_ModelError(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0).WithGetEmbeddingModelFunc(
func(tenantID, embdID string) (*models.EmbeddingModel, error) {
return nil, errors.New("model not found")
},
)
chunks := []map[string]any{{"text": "hello"}}
result, tc, err := svc.embedChunks(context.Background(), chunks, 10)
if err == nil {
t.Fatal("expected error on model error")
}
if !strings.Contains(err.Error(), "model not found") {
t.Errorf("expected error containing 'model not found', got %v", err)
}
if result != nil {
t.Errorf("expected nil result on model error, got %v", result)
}
if tc != 10 {
t.Errorf("token consumption should be preserved on error, got %d", tc)
}
}
// =============================================================================
// progress
// =============================================================================
@@ -695,30 +628,6 @@ func TestDataflowService_Run_MainFlowWithStubs(t *testing.T) {
}
}
func TestEmbedChunks_ReportsIntermediateProgress(t *testing.T) {
stub := &stubDriver{}
var progressCalls []float64
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 1, 0).
WithGetEmbeddingModelFunc(func(tenantID, embdID string) (*models.EmbeddingModel, error) {
return makeTestEmbeddingModel(stub, 100), nil
}).
WithProgressFunc(func(prog float64, msg string) {
progressCalls = append(progressCalls, prog)
})
chunks := []map[string]any{
{"text": "hello"},
{"text": "world"},
}
_, _, err := svc.embedChunks(context.Background(), chunks, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(progressCalls) < 2 {
t.Fatalf("expected start and intermediate progress calls, got %v", progressCalls)
}
}
func TestInsertChunks_ReportsBatchProgress(t *testing.T) {
var progressCalls []float64
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 1).
@@ -742,26 +651,6 @@ func TestInsertChunks_ReportsBatchProgress(t *testing.T) {
}
}
func TestEmbedChunks_ErrorReportsNegativeProgressAndReturnsError(t *testing.T) {
var progressCalls []float64
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 1, 0).
WithGetEmbeddingModelFunc(func(tenantID, embdID string) (*models.EmbeddingModel, error) {
return nil, errors.New("model unavailable")
}).
WithProgressFunc(func(prog float64, msg string) {
progressCalls = append(progressCalls, prog)
})
chunks := []map[string]any{{"text": "hello"}}
_, _, err := svc.embedChunks(context.Background(), chunks, 0)
if err == nil {
t.Fatal("expected error")
}
if len(progressCalls) == 0 || progressCalls[len(progressCalls)-1] != -1 {
t.Fatalf("expected final error progress -1, got %v", progressCalls)
}
}
// =============================================================================
// Stub implementations for testing
// =============================================================================
@@ -824,3 +713,104 @@ var (
_ docService = (*stubDocService)(nil)
_ chunkCounter = (*stubChunkCounter)(nil)
)
func makeEmbeddingModelForResolver() *models.EmbeddingModel {
return models.NewEmbeddingModel(&stubDriver{}, strPtr("embed"), &models.APIConfig{}, 128)
}
func TestPipelineExecutor_ResolveEmbeddingModel_IgnoresModelIDAndUsesDatasetEmbedding(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
var gotTenantID, gotModelID string
svc.getEmbeddingModelFunc = func(tenantID, embdID string) (*models.EmbeddingModel, error) {
gotTenantID, gotModelID = tenantID, embdID
return makeEmbeddingModelForResolver(), nil
}
model, err := svc.resolveEmbeddingModel("tenant-1", "kb-1", "override-model")
if err != nil {
t.Fatalf("resolveEmbeddingModel: %v", err)
}
if model == nil {
t.Fatal("expected model")
}
if gotTenantID != "tenant-1" || gotModelID != "embd-1" {
t.Fatalf("resolver args = (%q, %q), want (tenant-1, embd-1)", gotTenantID, gotModelID)
}
}
func TestPipelineExecutor_ResolveEmbeddingModel_KnowledgebaseModel(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
var gotModelID string
svc.getEmbeddingModelFunc = func(_ string, embdID string) (*models.EmbeddingModel, error) {
gotModelID = embdID
return makeEmbeddingModelForResolver(), nil
}
_, err := svc.resolveEmbeddingModel("tenant-1", "kb-1", "")
if err != nil {
t.Fatalf("resolveEmbeddingModel: %v", err)
}
if gotModelID != "embd-1" {
t.Fatalf("got model id %q, want embd-1", gotModelID)
}
}
func TestPipelineExecutor_ResolveEmbeddingModel_KnowledgebaseLookupFallback(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
svc.taskCtx.KB.ID = "other-kb"
var gotModelID string
svc.getEmbeddingModelFunc = func(_ string, embdID string) (*models.EmbeddingModel, error) {
gotModelID = embdID
return makeEmbeddingModelForResolver(), nil
}
svc.getKnowledgebaseByIDFunc = func(kbID string) (*entity.Knowledgebase, error) {
if kbID != "kb-2" {
t.Fatalf("kb lookup id = %q, want kb-2", kbID)
}
return &entity.Knowledgebase{ID: "kb-2", EmbdID: "lookup-embd"}, nil
}
_, err := svc.resolveEmbeddingModel("tenant-1", "kb-2", "")
if err != nil {
t.Fatalf("resolveEmbeddingModel: %v", err)
}
if gotModelID != "lookup-embd" {
t.Fatalf("got model id %q, want lookup-embd", gotModelID)
}
}
func TestPipelineExecutor_ResolveEmbeddingModel_MissingDatasetEmbeddingReturnsError(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
svc.taskCtx.KB.EmbdID = ""
svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: "kb-1", EmbdID: ""}, nil
}
svc.getEmbeddingModelFunc = func(_ string, _ string) (*models.EmbeddingModel, error) {
t.Fatal("knowledgebase resolver should not be called")
return nil, nil
}
_, err := svc.resolveEmbeddingModel("tenant-1", "kb-1", "")
if err == nil {
t.Fatal("expected error when dataset embd_id is missing, got nil")
}
if !strings.Contains(err.Error(), "dataset has no embd_id configured") {
t.Fatalf("err = %v, want dataset has no embd_id configured", err)
}
}
func TestPipelineExecutor_TokenizerEmbedderResolver_FallsBackToTaskContext(t *testing.T) {
svc := mustNewDataflowService(t, makeTaskCtx(), "flow-1", 0, 0)
var gotTenantID, gotModelID string
svc.getEmbeddingModelFunc = func(tenantID, embdID string) (*models.EmbeddingModel, error) {
gotTenantID, gotModelID = tenantID, embdID
return makeEmbeddingModelForResolver(), nil
}
resolver := svc.tokenizerEmbedderResolver()
emb, err := resolver("", "", "")
if err != nil {
t.Fatalf("resolver: %v", err)
}
if emb == nil {
t.Fatal("expected embedder")
}
if gotTenantID != "tenant-1" || gotModelID != "embd-1" {
t.Fatalf("resolver args = (%q, %q), want (tenant-1, embd-1)", gotTenantID, gotModelID)
}
}

26
internal/ingestion/task/embedding_test.go
View File

@@ -148,32 +148,6 @@ func TestEncodeTexts_ReturnsVectors(t *testing.T) {
}
}
// =============================================================================
// GetEmbeddingDimension
// =============================================================================
func TestTestEncodeForDim_ReturnsDimension(t *testing.T) {
stub := &stubDriver{}
model := makeTestEmbeddingModel(stub, 100)
dim, err := getEmbeddingDimension(model)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if dim != 2 {
t.Errorf("dim = %d, want 2 (stubDriver returns 2-element vectors)", dim)
}
}
func TestTestEncodeForDim_ModelError(t *testing.T) {
stub := &stubDriver{}
model := makeTestEmbeddingModel(stub, 100)
model.ModelDriver = &errDriver{}
_, err := getEmbeddingDimension(model)
if err == nil {
t.Error("expected error from failing driver")
}
}
type errDriver struct{}
func (d *errDriver) Embed(modelName *string, texts []string, apiConfig *models.APIConfig, embeddingConfig *models.EmbeddingConfig) ([]models.EmbeddingData, error) {

269
internal/ingestion/task/naive_merge.go
View File

@@ -1,269 +0,0 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package task
import (
"regexp"
"sort"
"strings"
"ragflow/internal/tokenizer"
)
// MergedSegment is a merged text chunk produced by NaiveMerge.
// SectionIndices tracks which input sections contributed to this segment,
// enabling the caller to associate images from those sections.
type MergedSegment struct {
Text string `json:"text"`
SectionIndices []int `json:"section_indices"`
Position string `json:"position,omitempty"`
}
// indexedText pairs text content with its original index and position.
type indexedText struct {
text string
idx int
pos string
}
// NaiveMerge merges texts into chunks by token count with delimiter-based
// splitting and overlap. Matches Python naive_merge_with_images().
//
// Parameters:
// - texts: input text segments (one per parsed section)
// - positions: optional position strings (one per section, may be nil)
// - chunkTokenNum: max token count per chunk (default 128 in Python)
// - delimiter: delimiter pattern, supports backtick-quoted custom delimiters
// - overlappedPercent: overlap percentage (0-100)
func NaiveMerge(texts []string, positions []string, chunkTokenNum int, delimiter string, overlappedPercent float64) []MergedSegment {
if len(texts) == 0 {
return nil
}
// Filter out empty texts but track original indices and positions
var filtered []indexedText
for i, t := range texts {
if strings.TrimSpace(t) != "" {
pos := ""
if positions != nil && i < len(positions) {
pos = positions[i]
}
filtered = append(filtered, indexedText{text: t, idx: i, pos: pos})
}
}
if len(filtered) == 0 {
return nil
}
// Parse delimiter: extract custom (backtick-quoted) and normal delimiters
customDels, normalPattern := parseDelimiters(delimiter)
// Custom delimiter mode: each segment is its own chunk, no merging by token count
if len(customDels) > 0 {
return mergeByCustomDelimiters(filtered, customDels, chunkTokenNum)
}
// Normal mode: split oversized texts at delimiters, merge by token count
return mergeByTokenCount(filtered, normalPattern, chunkTokenNum, overlappedPercent)
}
// ── Delimiter parsing ───────────────────────────────────────────────────────
// parseDelimiters extracts custom (backtick-quoted) and normal delimiters.
// Matches Python get_delimiters().
func parseDelimiters(delimiter string) (custom []string, pattern string) {
re := regexp.MustCompile("`([^`]+)`")
matches := re.FindAllStringSubmatchIndex(delimiter, -1)
var normalChars []rune
lastEnd := 0
for _, m := range matches {
start, end := m[0], m[1]
normalChars = append(normalChars, []rune(delimiter[lastEnd:start])...)
custom = append(custom, delimiter[m[2]:m[3]])
lastEnd = end
}
if lastEnd < len(delimiter) {
normalChars = append(normalChars, []rune(delimiter[lastEnd:])...)
}
var dels []string
seen := make(map[string]bool)
for _, r := range normalChars {
s := string(r)
if !seen[s] && s != "" {
dels = append(dels, regexp.QuoteMeta(s))
seen[s] = true
}
}
sort.Slice(dels, func(i, j int) bool {
return len(dels[i]) > len(dels[j])
})
if len(dels) > 0 {
pattern = strings.Join(dels, "|")
}
return
}
// ── Custom delimiter mode ───────────────────────────────────────────────────
func mergeByCustomDelimiters(texts []indexedText, customDels []string, chunkTokenNum int) []MergedSegment {
sort.Slice(customDels, func(i, j int) bool {
return len(customDels[i]) > len(customDels[j])
})
var escaped []string
for _, d := range customDels {
escaped = append(escaped, regexp.QuoteMeta(d))
}
pattern := regexp.MustCompile("(" + strings.Join(escaped, "|") + ")")
var result []MergedSegment
for _, it := range texts {
parts := pattern.Split(it.text, -1)
for _, part := range parts {
if part == "" {
continue
}
isDelimiter := false
for _, d := range customDels {
if part == d {
isDelimiter = true
break
}
}
if isDelimiter {
continue
}
textSeg := "\n" + part
pos := resolvePosition(textSeg, it.pos)
result = append(result, MergedSegment{
Text: textSeg,
SectionIndices: []int{it.idx},
Position: pos,
})
}
}
return result
}
// ── Normal (token-count-based) mode ─────────────────────────────────────────
func mergeByTokenCount(texts []indexedText, delimPattern string, chunkTokenNum int, overlappedPercent float64) []MergedSegment {
var result []MergedSegment
threshold := float64(chunkTokenNum) * (100.0 - overlappedPercent) / 100.0
for _, it := range texts {
var subSegments []indexedText
if delimPattern != "" && tokenizer.NumTokensFromString(it.text) >= chunkTokenNum {
re := regexp.MustCompile("(" + delimPattern + ")")
parts := re.Split(it.text, -1)
for _, part := range parts {
if part == "" {
continue
}
if matched, _ := regexp.MatchString("^("+delimPattern+")$", part); matched {
continue
}
subSegments = append(subSegments, indexedText{
text: "\n" + part,
idx: it.idx,
pos: it.pos,
})
}
} else {
subSegments = append(subSegments, indexedText{
text: "\n" + it.text,
idx: it.idx,
pos: it.pos,
})
}
for _, sub := range subSegments {
tk := tokenizer.NumTokensFromString(sub.text)
pos := resolvePosition(sub.text, sub.pos)
if len(result) == 0 {
result = append(result, MergedSegment{
Text: sub.text,
SectionIndices: []int{sub.idx},
Position: pos,
})
continue
}
last := &result[len(result)-1]
lastTk := tokenizer.NumTokensFromString(last.Text)
if lastTk > int(threshold) {
newText := sub.text
if overlappedPercent > 0 && lastTk > 0 {
overlapChars := int(float64(len([]rune(last.Text))) * (100.0 - overlappedPercent) / 100.0)
if overlapChars > 0 && overlapChars < len([]rune(last.Text)) {
overlap := string([]rune(last.Text)[overlapChars:])
newText = overlap + sub.text
}
}
result = append(result, MergedSegment{
Text: newText,
SectionIndices: []int{sub.idx},
Position: pos,
})
} else {
last.Text += sub.text
last.SectionIndices = appendUnique(last.SectionIndices, sub.idx)
// Position: keep the first non-empty position
if last.Position == "" && pos != "" {
last.Position = pos
}
}
_ = tk
}
}
return result
}
// ── Helpers ─────────────────────────────────────────────────────────────────
// resolvePosition applies Python position rules:
// - if token count < 8 → clear position
// - if position is not already in text → append to text (done by caller)
// Returns the resolved position string.
func resolvePosition(text, pos string) string {
if pos == "" {
return ""
}
tk := tokenizer.NumTokensFromString(text)
if tk < 8 {
return ""
}
return pos
}
func appendUnique(slice []int, val int) []int {
for _, v := range slice {
if v == val {
return slice
}
}
return append(slice, val)
}

417
internal/ingestion/task/naive_merge_test.go
View File

@@ -1,417 +0,0 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package task
import (
"strings"
"testing"
"ragflow/internal/tokenizer"
)
// naiveMergeTests defines a table-driven test case for NaiveMerge.
type naiveMergeTests struct {
name string
texts []string
chunkTokenNum int
delimiter string
overlappedPct float64
// Expectations
minChunks int // at least this many chunks
maxChunks int // at most this many chunks
maxTokens int // no single chunk should exceed this token count
contains []string // each chunk should contain these substrings
notContains []string // no chunk should contain these
}
func TestNaiveMerge_TableDriven(t *testing.T) {
tests := []naiveMergeTests{
// ── Empty & nil inputs ──────────────────────────────────────────
{
name: "nil texts",
texts: nil,
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
maxChunks: 0,
},
{
name: "empty texts",
texts: []string{},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
maxChunks: 0,
},
// ── Single short text ───────────────────────────────────────────
{
name: "single short text",
texts: []string{"Hello World"},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 1,
maxChunks: 1,
contains: []string{"Hello World"},
},
// ── Multiple short texts merged into one chunk ──────────────────
{
name: "multiple short texts merge into one",
texts: []string{"Short A", "Short B", "Short C"},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 1,
maxChunks: 1,
contains: []string{"Short A", "Short B", "Short C"},
},
// ── Large text split at delimiters ──────────────────────────────
{
name: "long text split at delimiter",
texts: []string{strings.Repeat("A。", 100)}, // 100 Chinese sentences
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 2, // should be split at '。'
maxChunks: 20,
},
// ── Chinese delimiters ──────────────────────────────────────────
{
name: "chinese sentence delimiters",
texts: []string{"第一句话。第二句话！第三句话？"},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 1,
maxChunks: 3,
},
// ── Overlap ─────────────────────────────────────────────────────
{
name: "overlap between chunks",
texts: []string{
strings.Repeat("A", 100) + "。",
strings.Repeat("B", 100) + "。",
strings.Repeat("C", 100) + "。",
},
chunkTokenNum: 50, // small enough to force multiple chunks
delimiter: "\n。；！？",
overlappedPct: 20, // 20% overlap
minChunks: 2,
maxChunks: 10,
},
// ── Custom delimiter (backtick-quoted) ──────────────────────────
{
name: "custom backtick delimiter",
texts: []string{"Section 1 `CHAPTER` Section 2"},
chunkTokenNum: 128,
delimiter: "`CHAPTER`\n。；！？",
overlappedPct: 0,
minChunks: 2,
maxChunks: 2,
contains: []string{"Section 1", "Section 2"},
notContains: []string{"CHAPTER"},
},
// ── Empty delimiter — no splitting ─────────────────────────────
{
name: "empty delimiter",
texts: []string{"Keep As Is Without Split"},
chunkTokenNum: 128,
delimiter: "",
overlappedPct: 0,
minChunks: 1,
maxChunks: 1,
},
// ── Multiple texts, some oversized, some not ────────────────────
{
name: "mixed sizes",
texts: []string{
"Tiny。",
strings.Repeat("Long text that exceeds the token limit and should be split。", 20),
"Another tiny。",
},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 2, // oversized text split, small ones may merge
maxChunks: 30,
},
// ── All texts empty strings ─────────────────────────────────────
{
name: "all empty strings",
texts: []string{"", "", ""},
chunkTokenNum: 128,
delimiter: "\n。；！？",
overlappedPct: 0,
maxChunks: 0,
},
// ── Very small chunk_token_num forces many chunks ───────────────
{
name: "tiny token limit",
texts: []string{"一。二。三。四。五。六。"},
chunkTokenNum: 5,
delimiter: "\n。；！？",
overlappedPct: 0,
minChunks: 2, // should be split into multiple chunks
maxChunks: 10,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
chunks := NaiveMerge(tt.texts, nil, tt.chunkTokenNum, tt.delimiter, tt.overlappedPct)
// Check min/max count
if len(chunks) < tt.minChunks {
t.Errorf("got %d chunks, want at least %d", len(chunks), tt.minChunks)
}
if len(chunks) > tt.maxChunks {
t.Errorf("got %d chunks, want at most %d", len(chunks), tt.maxChunks)
}
// Check max tokens per chunk
if tt.maxTokens > 0 {
for i, c := range chunks {
tk := tokenizer.NumTokensFromString(c.Text)
if tk > tt.maxTokens {
t.Errorf("chunk[%d] has %d tokens, exceeds max %d: %q", i, tk, tt.maxTokens, c.Text)
}
}
}
// Check contains
for _, substr := range tt.contains {
found := false
for _, c := range chunks {
if strings.Contains(c.Text, substr) {
found = true
break
}
}
if !found {
t.Errorf("no chunk contains %q", substr)
}
}
// Check not contains
for _, substr := range tt.notContains {
for _, c := range chunks {
if strings.Contains(c.Text, substr) {
t.Errorf("chunk should not contain %q: %q", substr, c.Text)
}
}
}
})
}
}
// ── Specific behavior tests ────────────────────────────────────────────────
func TestNaiveMerge_OverlapContent(t *testing.T) {
// Create texts where we can verify overlap behavior.
// Each text is a complete sentence with delimiter.
texts := []string{
"AAAAA。",
"BBBBB。",
"CCCCC。",
"DDDDD。",
}
chunks := NaiveMerge(texts, nil, 10, "\n。；！？", 30) // 30% overlap
// With token limit 10 and 30% overlap, chunks should reuse end of previous.
// Verify we got chunks and they look reasonable.
if len(chunks) < 2 {
t.Fatalf("expected at least 2 chunks with overlap, got %d", len(chunks))
}
// Verify chunks are non-empty and contain content
for i, c := range chunks {
if c.Text == "" {
t.Errorf("chunk[%d] is empty", i)
}
}
}
func TestNaiveMerge_DelimiterAsSplitPoint(t *testing.T) {
// Delimiters are split points for oversized texts (>chunkTokenNum).
// For texts under the token limit, no splitting occurs.
// This test verifies that a large text IS split at delimiters.
texts := []string{strings.Repeat("A。", 100)} // many sentences, over token limit
chunks := NaiveMerge(texts, nil, 20, "\n。；！？", 0)
// Should be split into multiple chunks (token limit 20 is small)
if len(chunks) < 2 {
t.Fatalf("expected at least 2 chunks, got %d", len(chunks))
}
// No chunk should contain '。' (delimiter is removed)
for _, c := range chunks {
if strings.Contains(c.Text, "。") {
t.Errorf("chunk should not contain delimiter '。': %q", c.Text)
}
}
}
func TestNaiveMerge_AllBelowTokenThreshold(t *testing.T) {
// All texts below chunk_token_num → should merge into one chunk.
texts := []string{"A", "B", "C", "D", "E"}
chunks := NaiveMerge(texts, nil, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
// Should contain all
for _, letter := range []string{"A", "B", "C", "D", "E"} {
if !strings.Contains(chunks[0].Text, letter) {
t.Errorf("chunk missing %q", letter)
}
}
}
func TestNaiveMerge_PositionInfo(t *testing.T) {
// Texts with position info should have positions carried through.
texts := []string{"Page 1 content。", "Page 2 content。"}
chunks := NaiveMerge(texts, nil, 128, "\n。；！？", 0)
// Verify SectionIndices are tracked
for _, c := range chunks {
if len(c.SectionIndices) == 0 {
t.Error("chunk has no SectionIndices")
}
// Section indices should be within range
for _, idx := range c.SectionIndices {
if idx < 0 || idx >= len(texts) {
t.Errorf("SectionIndex %d out of range [0, %d)", idx, len(texts))
}
}
}
}
func TestNaiveMerge_CustomDelimiterExclusive(t *testing.T) {
// Custom delimiter mode: each segment is its own chunk, no merging.
texts := []string{"Part A `---` Part B `---` Part C"}
chunks := NaiveMerge(texts, nil, 128, "`---`", 0)
// Should produce 3 chunks (one per segment between ---)
if len(chunks) != 3 {
t.Fatalf("expected 3 chunks from custom delimiter split, got %d", len(chunks))
}
// No chunk should contain "---"
for _, c := range chunks {
if strings.Contains(c.Text, "---") {
t.Errorf("chunk should not contain custom delimiter: %q", c.Text)
}
}
}
func TestNaiveMerge_LeadingNewline(t *testing.T) {
// Python adds "\n" prefix to each text before adding as chunk.
texts := []string{"Hello World"}
chunks := NaiveMerge(texts, nil, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
// Should start with "\n" (matching Python behavior)
if !strings.HasPrefix(chunks[0].Text, "\n") {
t.Errorf("expected chunk to start with newline, got %q", chunks[0].Text)
}
}
func TestNaiveMerge_SectionIndicesCorrect(t *testing.T) {
// Verify that merged chunks track which input sections they came from.
texts := []string{"First。", "Second。", "Third。", "Fourth。"}
chunks := NaiveMerge(texts, nil, 40, "\n。；！？", 0)
// Collect all section indices across all chunks
allIndices := make(map[int]bool)
for _, c := range chunks {
for _, idx := range c.SectionIndices {
allIndices[idx] = true
}
}
// Every input section should appear in exactly one chunk
for i := range texts {
if !allIndices[i] {
t.Errorf("section %d (%q) not found in any chunk", i, texts[i])
}
}
}
// ── Position-aware tests (gap: Python carries pos through add_chunk) ──────────
func TestNaiveMerge_PositionCarriedThrough(t *testing.T) {
texts := []string{"Long enough text with sufficient tokens for position to stay。", "Another long sentence with enough tokens。"}
positions := []string{"p1", "p2"}
chunks := NaiveMerge(texts, positions, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
if chunks[0].Position == "" {
t.Error("position should not be empty for merged chunk")
}
}
func TestNaiveMerge_PositionClearedWhenBelowThreshold(t *testing.T) {
// Token count < 8 → position should be cleared (Python: tnum < 8 → pos = "")
texts := []string{"Hi"}
positions := []string{"page1_top100"}
chunks := NaiveMerge(texts, positions, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
if chunks[0].Position != "" {
t.Errorf("position should be empty for short text (< 8 tokens), got %q", chunks[0].Position)
}
}
func TestNaiveMerge_PositionAppendedWhenNotPresent(t *testing.T) {
texts := []string{"Long enough text that exceeds eight tokens easily。"}
positions := []string{"[page1]"}
chunks := NaiveMerge(texts, positions, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
if chunks[0].Position != "[page1]" {
t.Errorf("position = %q, want %q", chunks[0].Position, "[page1]")
}
}
func TestNaiveMerge_NilPositions(t *testing.T) {
texts := []string{"Text A。", "Text B。"}
chunks := NaiveMerge(texts, nil, 128, "\n。；！？", 0)
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
if chunks[0].Position != "" {
t.Errorf("position should be empty when positions is nil, got %q", chunks[0].Position)
}
}

19
internal/ingestion/task/real_consumer_test.go
View File

@@ -1,3 +1,6 @@
//go:build manual
// +build manual
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
@@ -14,14 +17,13 @@
// limitations under the License.
//
//go:build integration
// +build integration
package task
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"ragflow/internal/common"
@@ -57,7 +59,16 @@ func TestRealProducerConsumer(t *testing.T) {
}
// ── 2. SQLite DB ──
db, _ := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("failed to get sql DB: %v", err)
}
sqlDB.SetMaxOpenConns(1)
db.AutoMigrate(
&entity.IngestionTask{}, &entity.Task{},
&entity.Document{}, &entity.Knowledgebase{}, &entity.Tenant{},

32
internal/ingestion/task/task_context.go
View File

@@ -119,38 +119,6 @@ func LoadFromIngestionTask(ingestionTask *entity.IngestionTask) (*TaskContext, e
}, nil
}
// Load loads the full task context following the FK chain: document → knowledgebase → tenant.
// Kept for backward compatibility.
func Load(docID string) (*TaskContext, error) {
doc, err := dao.NewDocumentDAO().GetByID(docID)
if err != nil || doc == nil {
return nil, fmt.Errorf("error when load document %s : %w", docID, err)
}
kb, err := dao.NewKnowledgebaseDAO().GetByID(doc.KbID)
if err != nil || kb == nil {
return nil, fmt.Errorf("error when load knowledgebase %s: %w", doc.KbID, err)
}
tenant, err := dao.NewTenantDAO().GetByID(kb.TenantID)
if err != nil || tenant == nil {
return nil, fmt.Errorf("error when load tenant %s: %w", kb.TenantID, err)
}
pipelineID := ""
if doc.PipelineID != nil {
pipelineID = *doc.PipelineID
}
return &TaskContext{
TaskType: "dataflow",
PipelineID: pipelineID,
Doc: *doc,
KB: *kb,
Tenant: *tenant,
}, nil
}
// LoadTaskContext loads the full task context following the FK chain: task → document → knowledgebase → tenant.
// Kept for backward compatibility.
func LoadTaskContext(taskID string) (*TaskContext, error) {

144
internal/ingestion/task/task_context_test.go
View File

@@ -17,6 +17,8 @@
package task
import (
"fmt"
"strings"
"testing"
"ragflow/internal/dao"
@@ -28,10 +30,16 @@ import (
func setupTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("failed to get sql DB: %v", err)
}
sqlDB.SetMaxOpenConns(1)
if err := db.AutoMigrate(
&entity.Task{},
&entity.Document{},
@@ -51,137 +59,3 @@ func pushDB(t *testing.T, db *gorm.DB) {
}
func ptr(s string) *string { return &s }
func TestLoadTaskContext(t *testing.T) {
db := setupTestDB(t)
pushDB(t, db)
// Prepare test data in FK dependency order: tenant → kb → document → task
tenant := &entity.Tenant{
ID: "tenant-1",
LLMID: "gpt-4",
ASRID: "whisper-1",
Img2TxtID: "gpt-4-vision",
Status: ptr("1"),
}
kb := &entity.Knowledgebase{
ID: "kb-1",
TenantID: "tenant-1",
Language: ptr("Chinese"),
EmbdID: "embd-model-1",
Pagerank: 0,
Status: ptr(string(entity.StatusValid)),
ParserConfig: entity.JSONMap{
"chunk_token_size": float64(512),
},
}
doc := &entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{"pages": []any{[]any{float64(1), float64(10)}}},
Name: ptr("test.pdf"),
Type: "pdf",
Location: ptr("bucket/test.pdf"),
Size: 1024,
}
task := &entity.Task{
ID: "task-1",
DocID: "doc-1",
FromPage: 0,
ToPage: 100000,
}
db.Create(tenant)
db.Create(kb)
db.Create(doc)
db.Create(task)
got, err := LoadTaskContext("task-1")
if err != nil {
t.Fatalf("LoadTaskContext: %v", err)
}
// === task table fields (no longer stored in TaskContext) ===
// Task field removed from TaskContext, skipping tests
// === document table fields ===
if got.Doc.ID != "doc-1" {
t.Errorf("Doc.ID = %v, want doc-1", got.Doc.ID)
}
if got.Doc.ParserID != "naive" {
t.Errorf("Doc.ParserID = %v, want naive", got.Doc.ParserID)
}
if got.Doc.Name == nil || *got.Doc.Name != "test.pdf" {
t.Errorf("Doc.Name = %v, want test.pdf", got.Doc.Name)
}
if got.Doc.Type != "pdf" {
t.Errorf("Doc.Type = %v, want pdf", got.Doc.Type)
}
if got.Doc.Location == nil || *got.Doc.Location != "bucket/test.pdf" {
t.Errorf("Doc.Location = %v, want bucket/test.pdf", got.Doc.Location)
}
if got.Doc.Size != 1024 {
t.Errorf("Doc.Size = %v, want 1024", got.Doc.Size)
}
if got.Doc.ParserConfig == nil {
t.Error("Doc.ParserConfig is nil")
} else if got.Doc.ParserConfig["pages"] == nil {
t.Error("Doc.ParserConfig.pages is nil")
}
// === knowledgebase table fields ===
if got.KB.ID != "kb-1" {
t.Errorf("KB.ID = %v, want kb-1", got.KB.ID)
}
if got.KB.TenantID != "tenant-1" {
t.Errorf("KB.TenantID = %v, want tenant-1", got.KB.TenantID)
}
if got.KB.Language == nil || *got.KB.Language != "Chinese" {
t.Errorf("KB.Language = %v, want Chinese", got.KB.Language)
}
if got.KB.EmbdID != "embd-model-1" {
t.Errorf("KB.EmbdID = %v, want embd-model-1", got.KB.EmbdID)
}
if got.KB.Pagerank != 0 {
t.Errorf("KB.Pagerank = %v, want 0", got.KB.Pagerank)
}
if got.KB.ParserConfig == nil {
t.Error("KB.ParserConfig is nil")
} else if got.KB.ParserConfig["chunk_token_size"] != float64(512) {
t.Errorf("KB.ParserConfig.chunk_token_size = %v, want 512",
got.KB.ParserConfig["chunk_token_size"])
}
// === tenant table fields ===
if got.Tenant.LLMID != "gpt-4" {
t.Errorf("Tenant.LLMID = %v, want gpt-4", got.Tenant.LLMID)
}
}
func TestLoadTaskContext_TaskNotFound(t *testing.T) {
db := setupTestDB(t)
pushDB(t, db)
_, err := LoadTaskContext("nonexistent")
if err == nil {
t.Fatal("expected error for nonexistent task")
}
}
func TestLoadTaskContext_DocNotFound(t *testing.T) {
db := setupTestDB(t)
pushDB(t, db)
db.Create(&entity.Task{
ID: "task-1",
DocID: "nonexistent-doc",
FromPage: 0,
ToPage: 100000,
})
_, err := LoadTaskContext("task-1")
if err == nil {
t.Fatal("expected error when document not found")
}
}

58
internal/ingestion/task/task_handler.go
View File

@@ -20,10 +20,6 @@ import (
"context"
"fmt"
"strings"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
"ragflow/internal/service"
)
// TaskHandler dispatches document processing tasks by task_type.
@@ -99,7 +95,11 @@ func (h *TaskHandler) handleDataflow() error {
if err != nil {
return err
}
return svc.Run(context.Background())
runCtx := h.ctx.Ctx
if runCtx == nil {
runCtx = context.Background()
}
return svc.Run(runCtx)
}
func (h *TaskHandler) handleRaptor() error {
@@ -117,51 +117,3 @@ func (h *TaskHandler) handleStub(name string) error {
func (h *TaskHandler) handleStandard() error {
return nil // stub
}
// BindEmbeddingModel creates an embedding model for the task's tenant.
// Returns the model and its vector dimension.
// Mirrors Python: _bind_embedding_model (task_handler.py:204)
func (h *TaskHandler) BindEmbeddingModel() (*models.EmbeddingModel, int, error) {
var model *models.EmbeddingModel
var err error
if embdID := h.ctx.KB.EmbdID; embdID != "" {
modelSvc := service.NewModelProviderService()
model, err = modelSvc.GetEmbeddingModel(h.ctx.Tenant.ID, embdID)
} else {
// Use tenant's default embedding model
model, err = defaultBindEmbeddingModel(h.ctx.Tenant.ID)
}
if err != nil {
return nil, 0, fmt.Errorf("bind embedding model: %w", err)
}
dim, err := getEmbeddingDimension(model)
if err != nil {
return nil, 0, fmt.Errorf("bind embedding model: %w", err)
}
return model, dim, err
}
// defaultBindEmbeddingModel returns the tenant's default embedding model and its vector dimension.
func defaultBindEmbeddingModel(tenantID string) (*models.EmbeddingModel, error) {
modelSvc := service.NewModelProviderService()
driver, modelName, apiConfig, maxTokens, err := modelSvc.GetTenantDefaultModelByType(tenantID, entity.ModelTypeEmbedding)
if err != nil {
return nil, fmt.Errorf("bind default embedding model: %w", err)
}
model := models.NewEmbeddingModel(driver, &modelName, apiConfig, maxTokens)
return model, nil
}
// getEmbeddingDimension encodes a test string to determine vector dimension.
func getEmbeddingDimension(model *models.EmbeddingModel) (int, error) {
embeds, err := model.ModelDriver.Embed(model.ModelName, []string{"ok"}, model.APIConfig, &models.EmbeddingConfig{Dimension: 0})
if err != nil {
return 0, fmt.Errorf("test encode failed: %w", err)
}
if len(embeds) == 0 {
return 0, fmt.Errorf("test encode returned no embeddings")
}
return len(embeds[0].Embedding), nil
}

29
internal/ingestion/task/task_handler_test.go
View File

@@ -136,6 +136,35 @@ func TestTaskHandler_DefaultDataflowServiceInjectsProgress(t *testing.T) {
}
}
func TestTaskHandler_Dataflow_UsesTaskContext(t *testing.T) {
ctx := makeTaskHandlerTestContext("dataflow")
type ctxKey string
const key ctxKey = "trace"
ctx.Ctx = context.WithValue(context.Background(), key, "task-ctx")
handler := NewTaskHandler(ctx).WithDataflowServiceFactory(func(ctx *TaskContext, dataflowID string) (*PipelineExecutor, error) {
return mustNewDataflowService(t, ctx, dataflowID, 0, 0).
WithLoadDSLFunc(func(ctx context.Context, dataflowID string) (string, string, error) {
return `{"nodes":[{"id":"stub-node"}],"edges":[]}`, dataflowID, nil
}).
WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
if got := runCtx.Value(key); got != "task-ctx" {
t.Fatalf("runCtx value = %v, want task-ctx", got)
}
return map[string]any{"chunks": []map[string]any{{"text": "stub", "q_2_vec": []float64{0.1, 0.2}}}}, dsl, nil
}).
WithInsertChunksFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
}).
WithLogCreateFunc(func(log *entity.PipelineOperationLog) error { return nil }).
WithDocService(&stubDocService{}).
WithChunkCounter(&stubChunkCounter{}), nil
})
if err := handler.Handle(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTaskHandler_Dataflow_ShowsProgressAndPipelineLog(t *testing.T) {
ctx := makeTaskHandlerTestContext("dataflow")
ctx.Doc.PipelineID = testStrPtr("flow-1")

10
internal/ingestion/testutil/helpers.go
View File

@@ -17,6 +17,8 @@
package testutil
import (
"fmt"
"strings"
"testing"
"ragflow/internal/common"
@@ -45,12 +47,18 @@ func StrPtr(s string) *string {
// It auto-migrates the given tables (or all common tables if none provided).
func SetupTestDB(t *testing.T, tables ...any) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("failed to get sql DB: %v", err)
}
sqlDB.SetMaxOpenConns(1)
if len(tables) == 0 {
tables = []any{

2
internal/service/document.go
View File

@@ -1313,7 +1313,7 @@ func (s *DocumentService) Ingest(userID string, req *IngestDocumentRequest) (com
kb, err := s.kbDAO.GetByID(doc.KbID)
if err != nil {
return common.CodeDataError, fmt.Errorf("tenant not found")
return common.CodeDataError, fmt.Errorf("dataset not found")
}
if !s.kbDAO.Accessible(kb.ID, userID) {

2
internal/service/model_service.go
View File

@@ -2127,7 +2127,7 @@ func (m *ModelProviderService) GetTenantDefaultModelByType(tenantID string, mode
tenantSvc := NewTenantService()
modelName, err := tenantSvc.GetDefaultModelName(tenantID, modelType)
if err != nil {
return nil, "", nil, 0, fmt.Errorf("failed to get default model name for type %s: %w", modelType, err)
return nil, "", nil, 0, fmt.Errorf("failed to get default model name for tenant: %s type %s: %w", tenantID, modelType, err)
}
if modelName == "" {
return nil, "", nil, 0, fmt.Errorf("no default %s model is set", modelType)