Refactor: drop dead canvas runtime selector and tokenizer embedding wiring (#16809)

Two refactors on the Go port (agent-go-port):

- Remove the dead per-tenant canvas runtime selector (write-only Redis
scaffolding with no runtime callers) and its dependent metrics/admin
code.
- Move the tokenizer embedding-model id from the shared ingestion
globals into a Tokenizer-scoped setup, and wire the production embedder
resolver in the ingestion task package.

32 files changed, 861 insertions, 1228 deletions.

internal/agent/canvas/compile.go

@@ -75,6 +75,15 @@ type CompileOptions struct {
 	// graph does not pause on completion and force an extra, needless
 	// ResumeWithData round.
 	InterruptAfterNonTerminal bool
+	// SetupOverrides is a run-level override map keyed by cpnID. Each
+	// component's `params["setups"]` is merged only with its own entry
+	// (an arbitrary string-keyed map); the override wins on top-level key
+	// collision (see node_body.go mergeSetups). Components absent from the
+	// map are left untouched. Used by the ingestion pipeline so a single
+	// Pipeline.Run can override the DSL-baked component setups without
+	// mutating the shared *Canvas (see node_body.go applySetupOverrides /
+	// mergeSetups).
+	SetupOverrides map[string]any
 }
 
 // CompileOption mutates a CompileOptions before the compile runs.
@@ -121,6 +130,14 @@ func WithInterruptAfterNonTerminalCpn() CompileOption {
 	return func(o *CompileOptions) { o.InterruptAfterNonTerminal = true }
 }
 
+// WithSetupOverrides attaches a run-level setups override map (keyed by
+// cpnID) to the compile. Each component's `params["setups"]` is merged with
+// its own entry at compile time (run-level wins on key collision, see
+// node_body.go mergeSetups). Passing nil is a no-op.
+func WithSetupOverrides(m map[string]any) CompileOption {
+	return func(o *CompileOptions) { o.SetupOverrides = m }
+}
+
 // Compile builds the eino Workflow from the Canvas and returns the
 // compiled Runnable. State pre/post handlers are wired inside BuildWorkflow
 // (see scheduler.go). Checkpoint store + serializer are wired here as
@@ -196,6 +213,14 @@ func Compile(ctx context.Context, c *Canvas, opts ...CompileOption) (*CompiledCa
 		}
 	}
 
+	// Thread the run-level setups override (if any) into ctx so each
+	// component's `params["setups"]` is merged with its own entry inside
+	// buildNodeBody. The override is keyed by cpnID; the canvas package
+	// never imports ingestion.
+	if cfg.SetupOverrides != nil {
+		ctx = withSetupOverrides(ctx, cfg.SetupOverrides)
+	}
+
 	wf, err := BuildWorkflow(ctx, c)
 	if err != nil {
 		return nil, fmt.Errorf("canvas: build workflow: %w", err)

internal/agent/canvas/compile_setup_override_test.go

@@ -0,0 +1,128 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package canvas
+
+import (
+	"context"
+	"testing"
+
+	"ragflow/internal/agent/runtime"
+)
+
+// TestCompile_SetupOverrides exercises the full canvas-level wiring:
+//
+//	canvas.Compile(ctx, dsl, WithSetupOverrides(override))
+//
+// threads the cpnID-keyed override through ctx into
+// BuildWorkflow → buildNodeBody → applySetupOverrides → mergeSetups, so
+// each component's factory receives its own merged params["setups"]. Only
+// the entry for a component's own cpnID applies; components absent from the
+// override map keep their base params (no spurious "setups" injected).
+func TestCompile_SetupOverrides(t *testing.T) {
+	captured := map[string]map[string]any{} // component_name -> params received by factory
+	factory := func(name string, params map[string]any) (runtime.Component, error) {
+		// Deep-shallow copy so later mutations don't hide what the
+		// factory actually received.
+		cp := make(map[string]any, len(params))
+		for k, v := range params {
+			cp[k] = v
+		}
+		captured[name] = cp
+		return &stubComponent{params: cp}, nil
+	}
+
+	dsl := &Canvas{
+		Components: map[string]CanvasComponent{
+			"parser_0": {
+				Obj: CanvasComponentObj{
+					ComponentName: "Parser",
+					Params: map[string]any{
+						"setups": map[string]any{
+							"pdf": map[string]any{
+								"output_format": "one",
+								"parse_method":  "naive",
+							},
+							"doc": map[string]any{
+								"output_format": "one",
+							},
+						},
+					},
+				},
+				Upstream:   []string{},
+				Downstream: []string{"sink_0"},
+			},
+			"sink_0": {
+				Obj: CanvasComponentObj{
+					ComponentName: "Sink",
+					Params:        map[string]any{"name": "sink"},
+				},
+				Upstream:   []string{"parser_0"},
+				Downstream: []string{},
+			},
+		},
+		Path: []string{"parser_0", "sink_0"},
+	}
+
+	// Override is keyed by cpnID. Only "parser_0" is present, so its
+	// "pdf" entry is fully replaced (parse_method dropped) and "docx" is
+	// injected; "doc" survives from the base. "sink_0" is absent and must
+	// keep its base params untouched.
+	override := map[string]any{
+		"parser_0": map[string]any{
+			"pdf": map[string]any{
+				"output_format": "detailed",
+			},
+			"docx": map[string]any{
+				"output_format": "one",
+			},
+		},
+	}
+
+	ctx := WithComponentFactory(context.Background(), factory)
+	if _, err := Compile(ctx, dsl, WithSetupOverrides(override)); err != nil {
+		t.Fatalf("Compile: %v", err)
+	}
+
+	// parser_0: override merged into params["setups"] before the factory.
+	got, ok := captured["Parser"]["setups"].(map[string]any)
+	if !ok {
+		t.Fatalf("Parser factory did not receive a setups map: %#v", captured["Parser"])
+	}
+	// doc preserved from base.
+	if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" {
+		t.Errorf("doc should be preserved from base: %#v", got["doc"])
+	}
+	// docx injected by the override (was absent in base).
+	if v, _ := got["docx"].(map[string]any); v == nil || v["output_format"] != "one" {
+		t.Errorf("docx injection missing: %#v", got["docx"])
+	}
+	// pdf: override fully replaces the base entry (shallow merge).
+	pdf, _ := got["pdf"].(map[string]any)
+	if pdf == nil {
+		t.Fatalf("pdf setup missing: %#v", got)
+	}
+	if pdf["output_format"] != "detailed" {
+		t.Errorf("pdf.output_format not overridden: %#v", pdf)
+	}
+	if _, ok := pdf["parse_method"]; ok {
+		t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)
+	}
+
+	// sink_0: absent from the override → no setups key injected.
+	if _, ok := captured["Sink"]["setups"]; ok {
+		t.Errorf("Sink should not receive a setups key: %#v", captured["Sink"])
+	}
+}

internal/agent/canvas/node_body.go

@@ -74,7 +74,75 @@ type nodeBodyFn = func(ctx context.Context, in map[string]any) (map[string]any,
 // Outputs bucket. UserFillUpNodeBody tags its output itself so the
 // interrupt-driven branch still attributes the resume payload to the
 // right cpn.
+// ctxKeySetupOverrides carries the run-level setups override map into
+// BuildWorkflow so a component's `params["setups"]` can be merged with it
+// at compile time. The map is keyed by cpnID; each component only sees the
+// entry for its own id (an arbitrary string-keyed map). It mirrors the ctx
+// plumbing used for the per-run component factory
+// (componentFactoryFromContext): the override is threaded through
+// canvas.Compile → BuildWorkflow → buildNodeBody without the canvas
+// package ever importing the ingestion layer.
+const ctxKeySetupOverrides ctxKey = "canvas_setup_overrides"
+
+// withSetupOverrides attaches a run-level setups override map to ctx. It is
+// a no-op when m is nil so callers can pass a possibly-nil run parameter
+// straight through.
+func withSetupOverrides(ctx context.Context, m map[string]any) context.Context {
+	if m == nil {
+		return ctx
+	}
+	return context.WithValue(ctx, ctxKeySetupOverrides, m)
+}
+
+func setupOverridesFromContext(ctx context.Context) map[string]any {
+	m, _ := ctx.Value(ctxKeySetupOverrides).(map[string]any)
+	return m
+}
+
+// applySetupOverrides returns a clone of params with the per-component
+// setups override (already resolved for this cpnID by the caller) merged
+// into params["setups"]. The override wins on top-level key collisions. The
+// original params map is never mutated — the merge result is a fresh map —
+// because the params come from the shared *Canvas and a per-run override
+// must not leak into the next Run on the same Pipeline.
+func applySetupOverrides(params, cpnOverride map[string]any) map[string]any {
+	if len(cpnOverride) == 0 {
+		return params
+	}
+	out := make(map[string]any, len(params)+1)
+	for k, v := range params {
+		out[k] = v
+	}
+	base, _ := out["setups"].(map[string]any)
+	out["setups"] = mergeSetups(base, cpnOverride)
+	return out
+}
+
+// mergeSetups merges a component-level setups map (base) with a run-level
+// override map. The maps are arbitrary string-keyed maps; when the same
+// top-level key exists in both, the override value wins (a full replacement
+// of that entry). The merge is shallow: only the top-level key-value pairs
+// are considered. (The Parser component happens to use file-type keys such
+// as "pdf"/"docx" as one example, but that is not required by this merge.)
+func mergeSetups(base, override map[string]any) map[string]any {
+	merged := make(map[string]any, len(base)+len(override))
+	for k, v := range base {
+		merged[k] = v
+	}
+	for k, ov := range override {
+		merged[k] = ov
+	}
+	return merged
+}
+
 func buildNodeBody(ctx context.Context, cpnID, name string, params map[string]any) (nodeBodyFn, error) {
+	if overrides := setupOverridesFromContext(ctx); len(overrides) > 0 {
+		// overrides is keyed by cpnID; a component only sees its own
+		// entry. Components absent from the map are left untouched.
+		if cpnOverride, ok := overrides[cpnID].(map[string]any); ok && len(cpnOverride) > 0 {
+			params = applySetupOverrides(params, cpnOverride)
+		}
+	}
 	if isLegacyNoOp(name) {
 		return legacyNoOpBody(cpnID), nil
 	}

internal/agent/canvas/node_body_setup_override_test.go

@@ -0,0 +1,152 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package canvas
+
+import (
+	"context"
+	"testing"
+
+	"ragflow/internal/agent/runtime"
+)
+
+// stubComponent records the params it was constructed with so a test can
+// assert that buildNodeBody forwarded the merged setups. Its Invoke is a
+// no-op echo.
+type stubComponent struct {
+	params map[string]any
+}
+
+func (s *stubComponent) Invoke(_ context.Context, in map[string]any) (map[string]any, error) {
+	return in, nil
+}
+
+// TestBuildNodeBody_SetupOverrides asserts that a run-level setups override
+// (threaded via ctx, keyed by cpnID) is merged into the component's
+// `params["setups"]` before the factory is called. Only the entry for the
+// component's own cpnID applies. The merge is shallow: a top-level key
+// present in the override fully replaces the base entry for that key
+// (no inner deep-merge), while base keys absent from the override survive.
+func TestBuildNodeBody_SetupOverrides(t *testing.T) {
+	captured := map[string]any{}
+	factory := func(name string, params map[string]any) (runtime.Component, error) {
+		// Deep-shallow copy so later mutations by the builder don't
+		// hide what the factory actually received.
+		cp := make(map[string]any, len(params))
+		for k, v := range params {
+			cp[k] = v
+		}
+		captured = cp
+		return &stubComponent{params: cp}, nil
+	}
+
+	baseParams := map[string]any{
+		"setups": map[string]any{
+			"pdf": map[string]any{
+				"output_format": "one",
+				"parse_method":  "naive",
+			},
+			"doc": map[string]any{
+				"output_format": "one",
+			},
+		},
+	}
+	// Run-level override is keyed by cpnID. For "cpn-parser": the whole
+	// "pdf" entry is replaced (parse_method is dropped), and a new "docx"
+	// entry is injected. "doc" is untouched because it is absent from the
+	// override. The entry for a different cpnID must not leak in.
+	override := map[string]any{
+		"cpn-parser": map[string]any{
+			"pdf": map[string]any{
+				"output_format": "detailed",
+			},
+			"docx": map[string]any{
+				"output_format": "one",
+			},
+		},
+		"cpn-other": map[string]any{
+			"pdf": map[string]any{
+				"output_format": "should-not-apply",
+			},
+		},
+	}
+
+	ctx := WithComponentFactory(context.Background(), factory)
+	ctx = withSetupOverrides(ctx, override)
+
+	body, err := buildNodeBody(ctx, "cpn-parser", "Parser", baseParams)
+	if err != nil {
+		t.Fatalf("buildNodeBody: %v", err)
+	}
+	if _, err := body(context.Background(), map[string]any{"x": 1}); err != nil {
+		t.Fatalf("body: %v", err)
+	}
+
+	got, ok := captured["setups"].(map[string]any)
+	if !ok {
+		t.Fatalf("factory did not receive a setups map: %#v", captured["setups"])
+	}
+	// doc is untouched (absent from the override).
+	if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" {
+		t.Errorf("doc should be preserved from base: %#v", got["doc"])
+	}
+	// docx injected by the override (was absent in base).
+	if v, _ := got["docx"].(map[string]any); v == nil || v["output_format"] != "one" {
+		t.Errorf("docx injection missing: %#v", got["docx"])
+	}
+	// pdf: the override fully replaces the base entry, so output_format is
+	// overridden and the base-only parse_method is dropped (shallow merge).
+	pdf, _ := got["pdf"].(map[string]any)
+	if pdf == nil {
+		t.Fatalf("pdf setup missing: %#v", got)
+	}
+	if pdf["output_format"] != "detailed" {
+		t.Errorf("pdf.output_format not overridden: %#v", pdf)
+	}
+	if _, ok := pdf["parse_method"]; ok {
+		t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)
+	}
+}
+
+// TestBuildNodeBody_SetupOverridesNilIsNoOp asserts that with no override in
+// ctx the component receives exactly its base params (no spurious setups key
+// injected, and the original map is untouched).
+func TestBuildNodeBody_SetupOverridesNilIsNoOp(t *testing.T) {
+	captured := map[string]any{}
+	factory := func(name string, params map[string]any) (runtime.Component, error) {
+		cp := make(map[string]any, len(params))
+		for k, v := range params {
+			cp[k] = v
+		}
+		captured = cp
+		return &stubComponent{params: cp}, nil
+	}
+	baseParams := map[string]any{"name": "x"}
+
+	ctx := WithComponentFactory(context.Background(), factory)
+	body, err := buildNodeBody(ctx, "cpn", "Parser", baseParams)
+	if err != nil {
+		t.Fatalf("buildNodeBody: %v", err)
+	}
+	if _, err := body(context.Background(), map[string]any{}); err != nil {
+		t.Fatalf("body: %v", err)
+	}
+	if _, ok := captured["setups"]; ok {
+		t.Errorf("setups key should not be injected when no override is present: %#v", captured)
+	}
+	if _, ok := baseParams["setups"]; ok {
+		t.Errorf("base params map must not be mutated: %#v", baseParams)
+	}
+}

internal/agent/runtime/metrics.go

@@ -1,102 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-//
-
-package runtime
-
-import (
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-// outcomeSuccess / outcomeError / outcomeCancelled are the only outcome
-// label values the canvas-run metric emits. Keeping the set closed lets
-// downstream alerts reason about the cardinality.
-const (
-	OutcomeSuccess   = "success"
-	OutcomeError     = "error"
-	OutcomeCancelled = "cancelled"
-)
-
-var (
-	canvasRunsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ragflow_canvas_runs_total",
-			Help: "Total canvas runs by runtime mode and outcome.",
-		},
-		[]string{"runtime", "outcome"},
-	)
-
-	canvasRunDuration = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "ragflow_canvas_run_duration_seconds",
-			Help:    "Canvas run latency in seconds.",
-			Buckets: prometheus.DefBuckets,
-		},
-		[]string{"runtime"},
-	)
-
-	registerOnce sync.Once
-)
-
-// init registers the package's metrics with the default Prometheus
-// registry. The sync.Once guard means tests that re-import this package
-// (or any future entry point that also imports it) won't trigger
-// "duplicate metrics collector registration" panics. If a test wants a
-// clean registry, call ResetMetrics().
-func init() {
-	registerOnce.Do(func() {
-		prometheus.MustRegister(canvasRunsTotal, canvasRunDuration)
-	})
-}
-
-// ResetMetricsForTesting unregisters the package metrics from the default
-// Prometheus registry, clears any recorded samples, and re-registers them
-// so the next ObserveRun call sees a clean slate. Intended for unit tests
-// that need to assert on freshly-registered metrics.
-func ResetMetricsForTesting() {
-	prometheus.DefaultRegisterer.Unregister(canvasRunsTotal)
-	prometheus.DefaultRegisterer.Unregister(canvasRunDuration)
-	canvasRunsTotal.Reset()
-	canvasRunDuration.Reset()
-	registerOnce = sync.Once{}
-	registerOnce.Do(func() {
-		prometheus.MustRegister(canvasRunsTotal, canvasRunDuration)
-	})
-}
-
-// ObserveRun emits a counter + histogram observation for one canvas run.
-// The runtime label is the string form of RuntimeMode; the outcome label
-// must be one of the Outcome* constants. Negative or zero durations are
-// dropped from the histogram to avoid skewing percentile math.
-//
-// An empty runtime defaults to the process-wide Default() so the metric
-// tag matches what Selector.Select would have returned for the same
-// tenant (review follow-up M4). Callers that need to record the
-// Python-routed-fallback case explicitly should pass RuntimePython.
-func ObserveRun(runtime RuntimeMode, outcome string, duration time.Duration) {
-	if runtime == "" {
-		runtime = Default()
-	}
-	if outcome == "" {
-		outcome = OutcomeError
-	}
-	canvasRunsTotal.WithLabelValues(string(runtime), outcome).Inc()
-	if duration > 0 {
-		canvasRunDuration.WithLabelValues(string(runtime)).Observe(duration.Seconds())
-	}
-}

internal/agent/runtime/metrics_test.go

@@ -1,116 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-
-package runtime
-
-import (
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/testutil"
-	dto "github.com/prometheus/client_model/go"
-)
-
-// labelsMatch reports whether the metric carries a label named "runtime"
-// with the supplied value.
-func labelsMatch(pairs []*dto.LabelPair, wantRuntime string) bool {
-	for _, p := range pairs {
-		if p.GetName() == "runtime" && p.GetValue() == wantRuntime {
-			return true
-		}
-	}
-	return false
-}
-
-func TestObserveRun_IncrementsCounter(t *testing.T) {
-	// ResetMetricsForTesting re-registers the metrics on the default
-	// registry so testutil can read them.
-	ResetMetricsForTesting()
-
-	ObserveRun(RuntimeGo, OutcomeSuccess, 250*time.Millisecond)
-	ObserveRun(RuntimeGo, OutcomeSuccess, 500*time.Millisecond)
-	ObserveRun(RuntimePython, OutcomeError, 750*time.Millisecond)
-
-	if got := testutil.ToFloat64(canvasRunsTotal.WithLabelValues("go", "success")); got != 2 {
-		t.Errorf("go/success counter = %v, want 2", got)
-	}
-	if got := testutil.ToFloat64(canvasRunsTotal.WithLabelValues("python", "error")); got != 1 {
-		t.Errorf("python/error counter = %v, want 1", got)
-	}
-	if got := testutil.ToFloat64(canvasRunsTotal.WithLabelValues("python", "success")); got != 0 {
-		t.Errorf("python/success counter = %v, want 0", got)
-	}
-}
-
-func TestObserveRun_RecordsDuration(t *testing.T) {
-	ResetMetricsForTesting()
-
-	ObserveRun(RuntimeGo, OutcomeSuccess, time.Second)
-	ObserveRun(RuntimeGo, OutcomeSuccess, 2*time.Second)
-
-	gathered, err := prometheus.DefaultGatherer.Gather()
-	if err != nil {
-		t.Fatalf("Gather: %v", err)
-	}
-
-	var found bool
-	for _, mf := range gathered {
-		if !strings.Contains(mf.GetName(), "canvas_run_duration_seconds") {
-			continue
-		}
-		for _, m := range mf.Metric {
-			if labelsMatch(m.Label, "go") {
-				found = true
-				if m.Histogram == nil || m.Histogram.GetSampleCount() != 2 {
-					t.Errorf("go histogram count = %v, want 2", m.Histogram.GetSampleCount())
-				}
-			}
-		}
-	}
-	if !found {
-		t.Fatal("did not find canvas_run_duration_seconds metric for runtime=go")
-	}
-}
-
-func TestObserveRun_NormalisesEmptyArgs(t *testing.T) {
-	ResetMetricsForTesting()
-
-	// Pin the env-driven default to Python so the test is hermetic
-	// regardless of the host environment. The assertion is "the
-	// empty-runtime label equals Default()", so we set the env to
-	// make the expected value explicit.
-	t.Setenv("RAGFLOW_CANVAS_DEFAULT_RUNTIME", string(RuntimePython))
-	ResetDefaultCache()
-
-	ObserveRun("", "", 0) // all empty — should default to python/error and not observe histogram
-
-	if got := testutil.ToFloat64(canvasRunsTotal.WithLabelValues("python", "error")); got != 1 {
-		t.Errorf("defaulted counter = %v, want 1", got)
-	}
-}
-
-// TestObserveRun_DefaultFallsToGoAfterPhase7 documents the
-// empty-runtime fallback: the metric's fallback now follows
-// selector.Default() (which is RuntimeGo), so the runtime label
-// is consistent with what Selector.Select would have returned
-// for the same tenant.
-func TestObserveRun_DefaultFallsToGoAfterPhase7(t *testing.T) {
-	ResetMetricsForTesting()
-	t.Setenv("RAGFLOW_CANVAS_DEFAULT_RUNTIME", "")
-	ResetDefaultCache()
-
-	ObserveRun("", OutcomeError, 0)
-
-	if got := testutil.ToFloat64(canvasRunsTotal.WithLabelValues(string(Default()), "error")); got != 1 {
-		t.Errorf("defaulted counter = %v, want 1 for runtime=%s outcome=error", got, Default())
-	}
-}

internal/agent/runtime/selector.go

@@ -1,168 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-//
-
-// Package runtime implements per-tenant runtime selection for the
-// agent canvas port.
-//
-// Two pieces live in this package:
-//
-//   - Selector (this file): reads/writes the per-tenant runtime
-//     override in Redis. The default is RuntimeGo; per-tenant
-//     overrides still let operators force a tenant back to Python
-//     during the agent_api.py deprecation window.
-//   - Metrics (metrics.go): Prometheus counter + histogram for
-//     per-run observation, keyed by runtime mode.
-package runtime
-
-import (
-	"context"
-	"fmt"
-	"ragflow/internal/common"
-	"sync"
-
-	"github.com/redis/go-redis/v9"
-	"go.uber.org/zap"
-)
-
-// RuntimeMode identifies which agent-canvas runtime implementation
-// serves a given tenant. Supports "go" and "python"; "auto" is
-// reserved for future adaptive policies.
-type RuntimeMode string
-
-const (
-	// RuntimeGo routes the tenant to the Go-side eino
-	// implementation. This is the process-wide default.
-	RuntimeGo RuntimeMode = "go"
-	// RuntimePython routes the tenant to the legacy Python
-	// agent_api.py implementation. Retained for the 1-release
-	// deprecation window; per-tenant overrides via Selector.Set
-	// can still force a tenant to this mode.
-	RuntimePython RuntimeMode = "python"
-	// RuntimeAuto defers to the per-tenant override, then to the
-	// process-wide Default(). It exists as a sentinel for clients that
-	// want explicit "I don't care, pick for me" semantics.
-	RuntimeAuto RuntimeMode = "auto"
-)
-
-// defaultEnvKey is the environment variable consulted by Default() when no
-// override is registered for a tenant.
-const defaultEnvKey = "RAGFLOW_CANVAS_DEFAULT_RUNTIME"
-
-// overrideKeyPrefix is the Redis key namespace for per-tenant runtime
-// overrides. Final keys look like "tenant_canvas_runtime:<tenantID>".
-const overrideKeyPrefix = "tenant_canvas_runtime:"
-
-var (
-	defaultOnce sync.Once
-	defaultMode RuntimeMode
-)
-
-// Default returns the process-wide default runtime mode.
-//
-// The default is Go. The per-tenant override (via Selector.Set)
-// can still force a tenant back to Python for the 1-release
-// deprecation window of agent_api.py.
-//
-// The value is read once from the RAGFLOW_CANVAS_DEFAULT_RUNTIME env var;
-// subsequent calls return the cached result. Unknown env values fall back
-// to RuntimeGo (the new default) so a misconfig still lands on the Go path.
-func Default() RuntimeMode {
-	defaultOnce.Do(func() {
-		raw := common.GetEnv(defaultEnvKey)
-		switch RuntimeMode(raw) {
-		case RuntimeGo, RuntimePython, RuntimeAuto:
-			defaultMode = RuntimeMode(raw)
-		default:
-			defaultMode = RuntimeGo
-		}
-	})
-	return defaultMode
-}
-
-// ResetDefaultCache clears the cached default-mode value. Test-only helper.
-func ResetDefaultCache() {
-	defaultOnce = sync.Once{}
-	defaultMode = ""
-}
-
-// Selector resolves the runtime mode for a tenant at request time. It is
-// safe for concurrent use.
-type Selector struct {
-	redis  *redis.Client
-	logger *zap.Logger
-}
-
-// NewSelector constructs a Selector backed by the supplied Redis client. A
-// nil logger is replaced with zap.NewNop() so callers in tests can omit it.
-func NewSelector(rdb *redis.Client, logger *zap.Logger) *Selector {
-	if logger == nil {
-		logger = zap.NewNop()
-	}
-	return &Selector{redis: rdb, logger: logger}
-}
-
-// overrideKey returns the Redis key for a tenant's runtime override.
-func overrideKey(tenantID string) string {
-	return overrideKeyPrefix + tenantID
-}
-
-// Select returns the runtime mode registered for tenantID. The lookup
-// order is:
-//
-//  1. The Redis key "tenant_canvas_runtime:<tenantID>" if present.
-//  2. The process-wide Default() (env RAGFLOW_CANVAS_DEFAULT_RUNTIME,
-//     falling back to RuntimeGo).
-//
-// A nil Redis client short-circuits to the default and never errors.
-func (s *Selector) Select(ctx context.Context, tenantID string) (RuntimeMode, error) {
-	if s == nil || s.redis == nil {
-		return Default(), nil
-	}
-	raw, err := s.redis.Get(ctx, overrideKey(tenantID)).Result()
-	if err == redis.Nil {
-		return Default(), nil
-	}
-	if err != nil {
-		s.logger.Warn("runtime selector: redis get failed, falling back to default",
-			zap.String("tenant_id", tenantID), zap.Error(err))
-		return Default(), err
-	}
-	mode := RuntimeMode(raw)
-	switch mode {
-	case RuntimeGo, RuntimePython, RuntimeAuto:
-		return mode, nil
-	default:
-		s.logger.Warn("runtime selector: unrecognized value, falling back to default",
-			zap.String("tenant_id", tenantID), zap.String("value", raw))
-		return Default(), fmt.Errorf("unrecognized runtime mode %q for tenant %q", raw, tenantID)
-	}
-}
-
-// Set overrides the runtime mode for a tenant. The override has no TTL
-// (it is permanent until explicitly changed) so the operator does not have
-// to remember to re-set it after a Redis flush of short-lived keys. Used
-// by the admin runtime endpoint and tests.
-func (s *Selector) Set(ctx context.Context, tenantID string, mode RuntimeMode) error {
-	if s == nil || s.redis == nil {
-		return fmt.Errorf("runtime selector: no redis client configured")
-	}
-	switch mode {
-	case RuntimeGo, RuntimePython, RuntimeAuto:
-	default:
-		return fmt.Errorf("runtime selector: refusing to set invalid mode %q", mode)
-	}
-	return s.redis.Set(ctx, overrideKey(tenantID), string(mode), 0).Err()
-}

internal/agent/runtime/selector_test.go

@@ -1,187 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-
-package runtime
-
-import (
-	"context"
-	"testing"
-
-	"github.com/alicebob/miniredis/v2"
-	"github.com/redis/go-redis/v9"
-)
-
-// newTestRedis spins up an in-process miniredis and returns a connected
-// client plus a teardown. The miniredis instance is reachable only from
-// the test goroutine that called this helper.
-func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
-	t.Helper()
-	mr := miniredis.RunT(t)
-	rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
-	t.Cleanup(func() {
-		_ = rdb.Close()
-	})
-	return rdb, mr
-}
-
-// TestDefault_ReturnsGo is the default-runtime acceptance
-// assertion: with no env var override, Default() must return
-// RuntimeGo. The env var is explicitly cleared so the test is
-// hermetic.
-func TestDefault_ReturnsGo(t *testing.T) {
-	t.Setenv(defaultEnvKey, "")
-	ResetDefaultCache()
-	if got := Default(); got != RuntimeGo {
-		t.Fatalf("Default() = %q, want %q (Go default)", got, RuntimeGo)
-	}
-}
-
-// TestDefault_RespectsEnv exercises the env-var override path through
-// Default(). The Go default holds for unset/unknown values; explicit
-// "python" / "auto" still win.
-func TestDefault_RespectsEnv(t *testing.T) {
-	t.Setenv(defaultEnvKey, string(RuntimePython))
-	ResetDefaultCache()
-	if got := Default(); got != RuntimePython {
-		t.Fatalf("Default() with python env = %q, want %q", got, RuntimePython)
-	}
-
-	t.Setenv(defaultEnvKey, string(RuntimeAuto))
-	ResetDefaultCache()
-	if got := Default(); got != RuntimeAuto {
-		t.Fatalf("Default() with auto env = %q, want %q", got, RuntimeAuto)
-	}
-
-	t.Setenv(defaultEnvKey, "bogus")
-	ResetDefaultCache()
-	if got := Default(); got != RuntimeGo {
-		t.Fatalf("Default() with invalid env = %q, want %q (Go fallback)", got, RuntimeGo)
-	}
-}
-
-// TestSelector_PerTenantOverride verifies the per-tenant override
-// mechanism still works with the Go default: even when Default()
-// would return Go, a tenant explicitly Set to Python must be
-// routed there.
-func TestSelector_PerTenantOverride(t *testing.T) {
-	t.Setenv(defaultEnvKey, "")
-	ResetDefaultCache()
-
-	rdb, _ := newTestRedis(t)
-	s := NewSelector(rdb, nil)
-	ctx := context.Background()
-
-	// Sanity: the global default is Go now.
-	if got := Default(); got != RuntimeGo {
-		t.Fatalf("setup: Default() = %q, want %q", got, RuntimeGo)
-	}
-
-	// Tenant with no override -> Go.
-	got, err := s.Select(ctx, "tenant_unset")
-	if err != nil {
-		t.Fatalf("Select() unset tenant: %v", err)
-	}
-	if got != RuntimeGo {
-		t.Fatalf("Select() unset tenant = %q, want %q", got, RuntimeGo)
-	}
-
-	// Force tenant_force_python to Python even though default is Go.
-	if err := s.Set(ctx, "tenant_force_python", RuntimePython); err != nil {
-		t.Fatalf("Set() unexpected error: %v", err)
-	}
-	got, err = s.Select(ctx, "tenant_force_python")
-	if err != nil {
-		t.Fatalf("Select() force-python tenant: %v", err)
-	}
-	if got != RuntimePython {
-		t.Fatalf("Select() force-python tenant = %q, want %q (per-tenant override)", got, RuntimePython)
-	}
-}
-
-// TestSelector_Select_FallbackToDefault covers the "no Redis key" path:
-// Select must return the process-wide default with no error.
-func TestSelector_Select_FallbackToDefault(t *testing.T) {
-	t.Setenv(defaultEnvKey, string(RuntimeGo))
-	ResetDefaultCache()
-
-	rdb, _ := newTestRedis(t)
-	s := NewSelector(rdb, nil)
-
-	got, err := s.Select(context.Background(), "tenant_42")
-	if err != nil {
-		t.Fatalf("Select() unexpected error: %v", err)
-	}
-	if got != RuntimeGo {
-		t.Fatalf("Select() = %q, want %q", got, RuntimeGo)
-	}
-}
-
-// TestSelector_SetAndSelectOverride proves that Set makes the override
-// visible to subsequent Select calls, even if the env default would have
-// been different.
-func TestSelector_SetAndSelectOverride(t *testing.T) {
-	t.Setenv(defaultEnvKey, string(RuntimeGo))
-	ResetDefaultCache()
-
-	rdb, _ := newTestRedis(t)
-	s := NewSelector(rdb, nil)
-	ctx := context.Background()
-
-	if err := s.Set(ctx, "tenant_a", RuntimePython); err != nil {
-		t.Fatalf("Set() unexpected error: %v", err)
-	}
-	got, err := s.Select(ctx, "tenant_a")
-	if err != nil {
-		t.Fatalf("Select() unexpected error: %v", err)
-	}
-	if got != RuntimePython {
-		t.Fatalf("Select() after Set = %q, want %q", got, RuntimePython)
-	}
-
-	// Different tenant still falls back to the Go default.
-	got, err = s.Select(ctx, "tenant_b")
-	if err != nil {
-		t.Fatalf("Select() for other tenant: %v", err)
-	}
-	if got != RuntimeGo {
-		t.Fatalf("Select() for other tenant = %q, want %q", got, RuntimeGo)
-	}
-}
-
-// TestSelector_SetRejectsInvalidMode makes sure a bad mode cannot poison
-// the override key.
-func TestSelector_SetRejectsInvalidMode(t *testing.T) {
-	rdb, _ := newTestRedis(t)
-	s := NewSelector(rdb, nil)
-	if err := s.Set(context.Background(), "tenant_x", RuntimeMode("rust")); err == nil {
-		t.Fatal("Set() with invalid mode should error, got nil")
-	}
-}
-
-// TestSelector_NilRedis_NoError covers the "Redis not initialised" path:
-// Select returns the default and never errors; Set errors so the caller
-// knows the override wasn't applied.
-func TestSelector_NilRedis_NoError(t *testing.T) {
-	t.Setenv(defaultEnvKey, string(RuntimeGo))
-	ResetDefaultCache()
-
-	s := NewSelector(nil, nil)
-	got, err := s.Select(context.Background(), "tenant_1")
-	if err != nil {
-		t.Fatalf("Select() with nil redis errored: %v", err)
-	}
-	if got != RuntimeGo {
-		t.Fatalf("Select() with nil redis = %q, want default %q", got, RuntimeGo)
-	}
-
-	if err := s.Set(context.Background(), "tenant_1", RuntimePython); err == nil {
-		t.Fatal("Set() with nil redis should error")
-	}
-}

internal/agent/runtime/state.go

@@ -304,6 +304,35 @@ func (s *CanvasState) RecordOutput(cpnID, bucket string, payload any) {
 	b[bucket] = payload
 }
 
+// GetGlobal returns a value from the workflow-wide Globals bag. Globals is a
+// generic, cross-component scratch space owned by CanvasState; the set of
+// keys an ingestion pipeline elects to store there is ingestion-specific and
+// therefore lives in the ingestion component package, not here.
+func (s *CanvasState) GetGlobal(key string) (any, bool) {
+	if s == nil {
+		return nil, false
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	v, ok := s.Globals[key]
+	return v, ok
+}
+
+// SetGlobal writes a value into the workflow-wide Globals bag. It is the
+// single, lock-safe mutation point for Globals so callers never touch the map
+// field directly.
+func (s *CanvasState) SetGlobal(key string, val any) {
+	if s == nil {
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.Globals == nil {
+		s.Globals = make(map[string]any)
+	}
+	s.Globals[key] = val
+}
+
 // GetRetrievalChunks returns a snapshot of the chunks recorded in
 // state.Retrieval["chunks"]. The Retrieval map is the canvas-level
 // aggregate that the Retrieval tool populates during the ReAct loop;