feat(go-agent): Ported retrieval node, added Keenable web search tool (#16396)

Ported retrieval node, added Keenable web search tool
- [x] New Feature (non-breaking change which adds functionality)

internal/agent/component/production_chain_fixes_test.go

@@ -34,7 +34,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/glebarez/sqlite"
+	"gorm.io/gorm"
+
+	"ragflow/internal/agent/runtime"
 	agenttool "ragflow/internal/agent/tool"
+	"ragflow/internal/dao"
+	"ragflow/internal/entity"
 )
 
 type codeExecSandboxRecorder struct {
@@ -190,6 +196,203 @@ func TestRetrieval_KbIDsTranslatedToDatasetIDs(t *testing.T) {
 	}
 }
 
+func TestRetrieval_LegacyQueryStringNormalized(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{TranslateError: true})
+	if err != nil {
+		t.Fatalf("failed to open sqlite: %v", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("failed to unwrap sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+	if err := db.AutoMigrate(&entity.Knowledgebase{}); err != nil {
+		t.Fatalf("failed to migrate knowledgebase: %v", err)
+	}
+	if err := db.AutoMigrate(&entity.UserTenant{}); err != nil {
+		t.Fatalf("failed to migrate user_tenant: %v", err)
+	}
+	origDB := dao.DB
+	dao.DB = db
+	t.Cleanup(func() { dao.DB = origDB })
+	activeStatus := "1"
+	if err := db.Create(&entity.UserTenant{
+		ID:        "ut-1",
+		UserID:    "user-1",
+		TenantID:  "tenant-1",
+		Role:      "owner",
+		InvitedBy: "user-1",
+		Status:    &activeStatus,
+	}).Error; err != nil {
+		t.Fatalf("failed to seed user_tenant: %v", err)
+	}
+
+	if err := db.Create(&entity.Knowledgebase{
+		ID:         "kb-da1",
+		Name:       "da1",
+		TenantID:   "tenant-1",
+		EmbdID:     "BAAI/bge-m3@yy2@SILICONFLOW",
+		Permission: "me",
+		CreatedBy:  "user-1",
+		Status:     func() *string { s := string(entity.StatusValid); return &s }(),
+	}).Error; err != nil {
+		t.Fatalf("failed to seed kb: %v", err)
+	}
+
+	c, err := newRetrievalComponent(nil)
+	if err != nil {
+		t.Fatalf("newRetrievalComponent: %v", err)
+	}
+	rc := c.(*retrievalComponent)
+	merged := rc.applyDefaults(map[string]any{
+		"query": "UserFillUp:   da1\nInput diamond necklace\n",
+	})
+	state := runtime.NewCanvasState("run-1", "task-1")
+	state.Sys["user_id"] = "user-1"
+	normalizeLegacyRetrievalInputs(runtime.WithState(context.Background(), state), merged)
+
+	if got, _ := merged["query"].(string); got != "diamond necklace" {
+		t.Fatalf("query = %q, want diamond necklace", got)
+	}
+	ds, ok := merged["dataset_ids"].([]string)
+	if !ok || len(ds) != 1 || ds[0] != "kb-da1" {
+		t.Fatalf("dataset_ids = %#v, want []string{\"kb-da1\"}", merged["dataset_ids"])
+	}
+}
+
+func TestRetrieval_StructuredUserFillInputNormalized(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{TranslateError: true})
+	if err != nil {
+		t.Fatalf("failed to open sqlite: %v", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("failed to unwrap sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+	if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.UserTenant{}); err != nil {
+		t.Fatalf("failed to migrate tables: %v", err)
+	}
+	origDB := dao.DB
+	dao.DB = db
+	t.Cleanup(func() { dao.DB = origDB })
+
+	activeStatus := "1"
+	if err := db.Create(&entity.UserTenant{
+		ID:        "ut-1",
+		UserID:    "user-1",
+		TenantID:  "tenant-1",
+		Role:      "owner",
+		InvitedBy: "user-1",
+		Status:    &activeStatus,
+	}).Error; err != nil {
+		t.Fatalf("failed to seed user_tenant: %v", err)
+	}
+	if err := db.Create(&entity.Knowledgebase{
+		ID:         "kb-da1",
+		Name:       "da1",
+		TenantID:   "tenant-1",
+		EmbdID:     "BAAI/bge-m3@yy2@SILICONFLOW",
+		Permission: "me",
+		CreatedBy:  "user-1",
+		Status:     func() *string { s := string(entity.StatusValid); return &s }(),
+	}).Error; err != nil {
+		t.Fatalf("failed to seed kb: %v", err)
+	}
+
+	c, err := newRetrievalComponent(nil)
+	if err != nil {
+		t.Fatalf("newRetrievalComponent: %v", err)
+	}
+	rc := c.(*retrievalComponent)
+	merged := rc.applyDefaults(map[string]any{
+		"state": map[string]any{
+			"UserFillUp:KBInput": map[string]any{
+				"kb":    "da1",
+				"query": "合同",
+			},
+		},
+	})
+	state := runtime.NewCanvasState("run-1", "task-1")
+	state.Sys["user_id"] = "user-1"
+	normalizeLegacyRetrievalInputs(runtime.WithState(context.Background(), state), merged)
+
+	if got, _ := merged["query"].(string); got != "合同" {
+		t.Fatalf("query = %q, want 合同", got)
+	}
+	ds, ok := merged["dataset_ids"].([]string)
+	if !ok || len(ds) != 1 || ds[0] != "kb-da1" {
+		t.Fatalf("dataset_ids = %#v, want []string{\"kb-da1\"}", merged["dataset_ids"])
+	}
+}
+
+func TestRetrieval_ResolveDatasetIDByTenantName(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{TranslateError: true})
+	if err != nil {
+		t.Fatalf("failed to open sqlite: %v", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("failed to unwrap sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+	if err := db.AutoMigrate(&entity.Knowledgebase{}); err != nil {
+		t.Fatalf("failed to migrate knowledgebase: %v", err)
+	}
+	origDB := dao.DB
+	dao.DB = db
+	t.Cleanup(func() { dao.DB = origDB })
+
+	if err := db.Create(&entity.Knowledgebase{
+		ID:         "kb-da1",
+		Name:       "da1",
+		TenantID:   "tenant-1",
+		EmbdID:     "BAAI/bge-m3@yy2@SILICONFLOW",
+		Permission: "me",
+		CreatedBy:  "user-1",
+		Status:     func() *string { s := string(entity.StatusValid); return &s }(),
+	}).Error; err != nil {
+		t.Fatalf("failed to seed kb: %v", err)
+	}
+
+	state := runtime.NewCanvasState("run-1", "task-1")
+	state.Sys["tenant_id"] = "tenant-1"
+	ctx := runtime.WithState(context.Background(), state)
+
+	if got := resolveRetrievalDatasetID(ctx, "da1"); got != "kb-da1" {
+		t.Fatalf("resolveRetrievalDatasetID = %q, want kb-da1", got)
+	}
+}
+
+func TestRetrieval_StructuredInputPreservesQueryWhenDatasetIDsAlreadyPresent(t *testing.T) {
+	c, err := newRetrievalComponent(nil)
+	if err != nil {
+		t.Fatalf("newRetrievalComponent: %v", err)
+	}
+	rc := c.(*retrievalComponent)
+	merged := rc.applyDefaults(map[string]any{
+		"dataset_ids": []string{"kb-fixed"},
+		"state": map[string]any{
+			"UserFillUp:KBInput": map[string]any{
+				"kb":    "da1",
+				"query": "合同",
+			},
+		},
+	})
+
+	consumed := normalizeStructuredRetrievalInputs(context.Background(), merged)
+	if !consumed {
+		t.Fatal("normalizeStructuredRetrievalInputs should consume structured query")
+	}
+	if got, _ := merged["query"].(string); got != "合同" {
+		t.Fatalf("query = %q, want 合同", got)
+	}
+	ds, ok := merged["dataset_ids"].([]string)
+	if !ok || len(ds) != 1 || ds[0] != "kb-fixed" {
+		t.Fatalf("dataset_ids = %#v, want []string{\"kb-fixed\"}", merged["dataset_ids"])
+	}
+}
+
 // TestRetrieval_KbIDsEndToEndThroughTool is the wire-level
 // companion to TestRetrieval_KbIDsTranslatedToDatasetIDs: it
 // installs the simple retrieval service, builds a wrapper with

internal/agent/component/switch.go

@@ -353,6 +353,22 @@ func evaluateClause(clause map[string]any, state *runtime.CanvasState) (bool, er
 	right := clause["right"]
 	lv := leftValue(left, state)
 
+	// Port of python PR #16320: for the four string operators,
+	// coerce nil on either side to "". In Python this avoids
+	// AttributeError on `.lower()`; in Go there's no crash (fmt
+	// renders nil as "<nil>"), but the Python post-fix semantic —
+	// where "foo" contains None is True — diverges from Go's
+	// "<nil>" rendering. Coercing to "" aligns the Go port with
+	// the Python workflow.
+	if op == "contains" || op == "not contains" || op == "start with" || op == "end with" {
+		if lv == nil {
+			lv = ""
+		}
+		if right == nil {
+			right = ""
+		}
+	}
+
 	switch op {
 	case "==":
 		return equalFoldValues(lv, right), nil

internal/agent/component/switch_test.go

@@ -241,6 +241,149 @@ func TestSwitch_LegacyConditionsAndArrayTo(t *testing.T) {
 	}
 }
 
+// TestSwitch_NilUpstreamContainsEmptyNeedleMatches ports the
+// regression covered by python PR #16320: when an upstream
+// component yields nil and the configured value is the empty
+// string, the "contains" operator must match (Python semantics
+// after the fix: "" in "anything"). Pre-fix Python crashed with
+// AttributeError; pre-port Go returned false because fmt rendered
+// nil as "<nil>" instead of "". The fix coerces nil → "" before
+// formatting, restoring parity with the Python workflow.
+func TestSwitch_NilUpstreamContainsEmptyNeedleMatches(t *testing.T) {
+	s, _ := NewSwitchComponent(nil)
+	state := canvas.NewCanvasState("run-nil-contains", "task-nil-contains")
+	state.Sys["answer"] = nil
+	ctx := withStateForTest(context.Background(), state)
+
+	inputs := map[string]any{
+		"conditions": []any{
+			map[string]any{
+				"op": "and",
+				"to": []any{"case_target"},
+				"clauses": []any{
+					map[string]any{"left": "{{sys.answer}}", "op": "contains", "right": ""},
+				},
+			},
+		},
+		"default": "else_target",
+	}
+	out, err := s.Invoke(ctx, inputs)
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	targets := nextTargets(out)
+	if len(targets) != 1 || targets[0] != "case_target" {
+		t.Errorf("_next: got %v, want [\"case_target\"] (nil coerced to \"\" should match empty needle)", targets)
+	}
+}
+
+// TestSwitch_NilUpstreamContainsNonEmptyDoesNotMatch verifies
+// the inverse: nil coerced to "" still must NOT match a
+// non-empty needle (we only coerce, we don't synthesize a match).
+func TestSwitch_NilUpstreamContainsNonEmptyDoesNotMatch(t *testing.T) {
+	s, _ := NewSwitchComponent(nil)
+	state := canvas.NewCanvasState("run-nil-needle", "task-nil-needle")
+	state.Sys["answer"] = nil
+	ctx := withStateForTest(context.Background(), state)
+
+	inputs := map[string]any{
+		"conditions": []any{
+			map[string]any{
+				"op": "and",
+				"to": []any{"case_target"},
+				"clauses": []any{
+					map[string]any{"left": "{{sys.answer}}", "op": "contains", "right": "foo"},
+				},
+			},
+		},
+		"default": "else_target",
+	}
+	out, err := s.Invoke(ctx, inputs)
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	targets := nextTargets(out)
+	if len(targets) != 1 || targets[0] != "else_target" {
+		t.Errorf("_next: got %v, want [\"else_target\"] (\"\" does not contain \"foo\")", targets)
+	}
+}
+
+// TestSwitch_NilValueContainsDoesNotRaise mirrors python test
+// "test_switch_none_value_contains_does_not_raise": the configured
+// value can also be nil and the operator must not crash. With
+// nil coerced to "" on both sides, "foobar" contains "" matches.
+func TestSwitch_NilValueContainsDoesNotRaise(t *testing.T) {
+	s, _ := NewSwitchComponent(nil)
+	state := canvas.NewCanvasState("run-nil-value", "task-nil-value")
+	state.Sys["answer"] = "foobar"
+	ctx := withStateForTest(context.Background(), state)
+
+	inputs := map[string]any{
+		"conditions": []any{
+			map[string]any{
+				"op": "and",
+				"to": []any{"case_target"},
+				"clauses": []any{
+					map[string]any{"left": "{{sys.answer}}", "op": "contains", "right": nil},
+				},
+			},
+		},
+		"default": "else_target",
+	}
+	out, err := s.Invoke(ctx, inputs)
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	targets := nextTargets(out)
+	if len(targets) != 1 || targets[0] != "case_target" {
+		t.Errorf("_next: got %v, want [\"case_target\"] (nil value coerced to \"\" matches any string)", targets)
+	}
+}
+
+// TestSwitch_NilUpstreamStartWithEndWithDoNotCrash guards the
+// remaining two string operators covered by PR #16320. They were
+// crash-prone in Python for the same reason; in Go they don't
+// crash but the nil → "" coercion still applies, so a nil
+// upstream with an empty prefix/suffix must match (rather than
+// being rendered as "<nil>" and silently missing).
+func TestSwitch_NilUpstreamStartWithEndWithDoNotCrash(t *testing.T) {
+	s, _ := NewSwitchComponent(nil)
+	state := canvas.NewCanvasState("run-nil-start-end", "task-nil-start-end")
+	state.Sys["answer"] = nil
+	ctx := withStateForTest(context.Background(), state)
+
+	for _, tc := range []struct {
+		name string
+		op   string
+	}{
+		{name: "start with", op: "start with"},
+		{name: "end with", op: "end with"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			inputs := map[string]any{
+				"conditions": []any{
+					map[string]any{
+						"op": "and",
+						"to": []any{"case_target"},
+						"clauses": []any{
+							map[string]any{"left": "{{sys.answer}}", "op": tc.op, "right": ""},
+						},
+					},
+				},
+				"default": "else_target",
+			}
+			out, err := s.Invoke(ctx, inputs)
+			if err != nil {
+				t.Fatalf("Invoke: %v", err)
+			}
+			targets := nextTargets(out)
+			if len(targets) != 1 || targets[0] != "case_target" {
+				t.Errorf("_next: got %v, want [\"case_target\"] (nil coerced to \"\" %s \"\")", targets, tc.op)
+			}
+		})
+	}
+}
+
 // TestSwitch_MultiTargetTo verifies that Switch returns all cpn_ids
 // from a multi-element "to" field. This mirrors Python's behavior
 // where a condition can route to multiple downstream nodes

internal/agent/component/universe_a_wrappers.go

@@ -31,16 +31,22 @@ package component
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
 	einotool "github.com/cloudwego/eino/components/tool"
 
+	"ragflow/internal/agent/runtime"
 	agenttool "ragflow/internal/agent/tool"
 	"ragflow/internal/common"
+	"ragflow/internal/dao"
+	"ragflow/internal/entity"
 
 	"go.uber.org/zap"
+	"gorm.io/gorm"
 )
 
 // tavilySearchComponent delegates to internal/agent/tool/TavilyTool.
@@ -153,6 +159,8 @@ type retrievalComponent struct {
 	params retrievalParams
 }
 
+var legacyRetrievalQueryPattern = regexp.MustCompile(`(?s)^\s*UserFillUp:\s*(.*?)\s+Input\s+(.*?)\s*$`)
+
 func newRetrievalComponent(params map[string]any) (Component, error) {
 	return &retrievalComponent{
 		inner:  agenttool.NewRetrievalTool(),
@@ -180,11 +188,19 @@ func (c *retrievalComponent) Outputs() map[string]string {
 
 func (c *retrievalComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
 	merged := c.applyDefaults(inputs)
+	normalizeLegacyRetrievalInputs(ctx, merged)
+	common.Debug("agent retrieval component: invoke",
+		zap.Any("inputs", inputs),
+		zap.Any("merged", merged),
+	)
 	argsJSON, _ := json.Marshal(merged)
 	out, err := c.inner.InvokableRun(ctx, string(argsJSON))
 	if err != nil {
 		return nil, fmt.Errorf("canvas: Retrieval: %w", err)
 	}
+	common.Debug("agent retrieval component: output",
+		zap.String("tool_output", out),
+	)
 	return parseToolEnvelope(out), nil
 }
 
@@ -261,6 +277,148 @@ func (c *retrievalComponent) applyDefaults(inputs map[string]any) map[string]any
 	return out
 }
 
+func normalizeLegacyRetrievalInputs(ctx context.Context, out map[string]any) {
+	if normalizeStructuredRetrievalInputs(ctx, out) {
+		return
+	}
+	rawQuery, _ := out["query"].(string)
+	rawQuery = strings.TrimSpace(rawQuery)
+	if rawQuery == "" {
+		return
+	}
+	matches := legacyRetrievalQueryPattern.FindStringSubmatch(rawQuery)
+	if len(matches) != 3 {
+		return
+	}
+	kbName := strings.TrimSpace(matches[1])
+	queryText := strings.TrimSpace(matches[2])
+	if queryText != "" {
+		out["query"] = queryText
+	}
+	if _, hasDatasetIDs := out["dataset_ids"]; hasDatasetIDs {
+		return
+	}
+	if kbName == "" {
+		return
+	}
+	if datasetID := resolveRetrievalDatasetID(ctx, kbName); datasetID != "" {
+		out["dataset_ids"] = []string{datasetID}
+	}
+}
+
+func normalizeStructuredRetrievalInputs(ctx context.Context, out map[string]any) bool {
+	_, hasDatasetIDs := out["dataset_ids"]
+	candidateMaps := []map[string]any{}
+	if stateMap, ok := out["state"].(map[string]any); ok {
+		if raw, ok := stateMap["UserFillUp:KBInput"].(map[string]any); ok {
+			candidateMaps = append(candidateMaps, raw)
+		}
+	}
+	candidateMaps = append(candidateMaps, out)
+
+	consumed := false
+	for _, candidate := range candidateMaps {
+		kbName, _ := candidate["kb"].(string)
+		queryText, _ := candidate["query"].(string)
+		if kbName == "" && legacyRetrievalQueryPattern.MatchString(strings.TrimSpace(queryText)) {
+			continue
+		}
+		if kbName == "" && queryText == "" {
+			continue
+		}
+		consumed = true
+		if queryText != "" {
+			out["query"] = queryText
+		}
+		if kbName != "" && !hasDatasetIDs {
+			if datasetID := resolveRetrievalDatasetID(ctx, strings.TrimSpace(kbName)); datasetID != "" {
+				out["dataset_ids"] = []string{datasetID}
+				common.Debug("agent retrieval component: resolved dataset id",
+					zap.String("kb", strings.TrimSpace(kbName)),
+					zap.String("dataset_id", datasetID))
+			}
+		}
+		if queryText != "" {
+			return true
+		}
+		if kbName != "" && out["dataset_ids"] != nil {
+			return true
+		}
+	}
+	return consumed
+}
+
+func resolveRetrievalDatasetID(ctx context.Context, kbName string) string {
+	if kbName == "" {
+		return ""
+	}
+	if kb, err := dao.NewKnowledgebaseDAO().GetByID(kbName); err == nil && kb != nil {
+		common.Debug("agent retrieval component: resolved dataset id by direct id",
+			zap.String("kb", kbName),
+			zap.String("dataset_id", kb.ID))
+		return kb.ID
+	} else if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+		common.Warn("agent retrieval component: resolve dataset id by id failed",
+			zap.String("kb", kbName),
+			zap.Error(err))
+	}
+	if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {
+		common.Debug("agent retrieval component: resolve dataset id context",
+			zap.String("kb", kbName),
+			zap.Any("sys_query", state.Sys["query"]),
+			zap.Any("tenant_id", state.Sys["tenant_id"]),
+			zap.Any("user_id", state.Sys["user_id"]))
+		if tenantID, _ := state.Sys["tenant_id"].(string); tenantID != "" {
+			if kb, lookupErr := dao.NewKnowledgebaseDAO().GetByName(kbName, tenantID); lookupErr == nil && kb != nil {
+				common.Debug("agent retrieval component: resolved dataset id by tenant",
+					zap.String("kb", kbName),
+					zap.String("tenant_id", tenantID),
+					zap.String("dataset_id", kb.ID))
+				return kb.ID
+			} else if lookupErr != nil && !errors.Is(lookupErr, gorm.ErrRecordNotFound) {
+				common.Warn("agent retrieval component: resolve dataset id by tenant failed",
+					zap.String("kb", kbName),
+					zap.String("tenant_id", tenantID),
+					zap.Error(lookupErr))
+			} else {
+				common.Debug("agent retrieval component: tenant lookup missed",
+					zap.String("kb", kbName),
+					zap.String("tenant_id", tenantID))
+			}
+		}
+		if userID, _ := state.Sys["user_id"].(string); userID != "" {
+			if kbs, lookupErr := dao.NewKnowledgebaseDAO().GetKBByNameAndUserID(kbName, userID); lookupErr == nil && len(kbs) > 0 {
+				for _, kb := range kbs {
+					if kb == nil || kb.Status == nil || *kb.Status != string(entity.StatusValid) {
+						continue
+					}
+					common.Debug("agent retrieval component: resolved dataset id by user visibility",
+						zap.String("kb", kbName),
+						zap.String("user_id", userID),
+						zap.String("dataset_id", kb.ID))
+					return kb.ID
+				}
+			} else if lookupErr != nil {
+				common.Warn("agent retrieval component: resolve dataset id by name failed",
+					zap.String("kb", kbName),
+					zap.String("user_id", userID),
+					zap.Error(lookupErr))
+			} else {
+				common.Debug("agent retrieval component: user visibility lookup missed",
+					zap.String("kb", kbName),
+					zap.String("user_id", userID))
+			}
+		}
+	} else {
+		common.Debug("agent retrieval component: resolve dataset id missing canvas state",
+			zap.String("kb", kbName),
+			zap.Error(err))
+	}
+	common.Debug("agent retrieval component: dataset id unresolved",
+		zap.String("kb", kbName))
+	return ""
+}
+
 // exesqlComponent delegates to internal/agent/tool/ExeSQLTool. The
 // connection params (db_type, host, port, database, username,
 // password) are passed via the canvas node's params map at build