mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-24 17:36:47 +08:00
a7da78d0d7
## Summary
This PR consolidates Eino-backed Agent tools behind the shared
ToolBackedComponent implementation and aligns their Canvas
configuration,
runtime inputs, output conversion, validation, and registration.
## What changed
- Migrated these Canvas components to the unified tool-backed path:
- Tavily Search and Extract
- Execute SQL
- Google
- Yahoo Finance
- Email
- DuckDuckGo
- Wikipedia
- Google Scholar
- ArXiv
- PubMed
- BGPT
- GitHub
- WenCai
- SearXNG
- Keenable Search
- Removed superseded component wrappers and their duplicate tests.
- Added dedicated registry builders for node-level configuration and
validation.
- Kept model-emitted runtime inputs separate from Canvas node
configuration.
- Moved Email defaults, template resolution, recipient parsing, SMTP
execution, and output conversion into the owning tool.
- Added complete ToolComponent specifications for Canvas inputs,
outputs, and input forms.
- Preserved raw upstream fields where downstream workflows may depend on
them.
- Added workflow registration coverage for all migrated component names.
- Kept HTTP Request, Docs Generator, and Browser as standalone
components because they are not Eino-backed tools.
## Testing
Passed:
bash build.sh --test ./internal/agent/tool/...
bash build.sh --test ./internal/agent/component/...
bash build.sh --test ./internal/agent/runtime/...
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/c728d7a3-9d15-4c5c-b0eb-6b77ad0e41ac"
/>
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/ef13119a-ef92-4b43-9061-eee9511bf492"
/>
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/f50f2c30-eab0-463c-b0b3-0a02523219f1"
/>
383 lines
12 KiB
Go
383 lines
12 KiB
Go
//
|
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
package tool
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
einotool "github.com/cloudwego/eino/components/tool"
|
|
"github.com/cloudwego/eino/compose"
|
|
"github.com/cloudwego/eino/flow/agent/react"
|
|
"github.com/cloudwego/eino/schema"
|
|
)
|
|
|
|
func TestDuckDuckGo_BuildSearchURL(t *testing.T) {
|
|
got := buildDuckDuckGoSearchURL("rag flow", 7)
|
|
u, err := url.Parse(got)
|
|
if err != nil {
|
|
t.Fatalf("url.Parse(%q): %v", got, err)
|
|
}
|
|
if u.Host != "duckduckgo.com" {
|
|
t.Errorf("host = %q, want duckduckgo.com", u.Host)
|
|
}
|
|
if u.Path != "/html/" {
|
|
t.Errorf("path = %q, want /html/", u.Path)
|
|
}
|
|
q := u.Query()
|
|
if q.Get("q") != "rag flow" {
|
|
t.Errorf("q = %q, want rag flow", q.Get("q"))
|
|
}
|
|
if q.Get("dc") != "8" {
|
|
t.Errorf("dc = %q, want 8", q.Get("dc"))
|
|
}
|
|
if got := q.Get("vqd"); got != "" {
|
|
t.Errorf("vqd = %q, want omitted", got)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_BuildNewsURL(t *testing.T) {
|
|
got := buildDuckDuckGoNewsURL("rag flow", 3)
|
|
u, err := url.Parse(got)
|
|
if err != nil {
|
|
t.Fatalf("url.Parse(%q): %v", got, err)
|
|
}
|
|
if u.Path != "/news.js" {
|
|
t.Errorf("path = %q, want /news.js", u.Path)
|
|
}
|
|
q := u.Query()
|
|
if q.Get("o") != "json" {
|
|
t.Fatalf("o = %q, want json", q.Get("o"))
|
|
}
|
|
if q.Get("l") != "wt-wt" {
|
|
t.Fatalf("l = %q, want wt-wt", q.Get("l"))
|
|
}
|
|
if q.Get("dc") != "3" {
|
|
t.Errorf("dc = %q, want 3", q.Get("dc"))
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_BuildNewsURLWithVQD(t *testing.T) {
|
|
got := buildDuckDuckGoNewsURLWithVQD("rag flow", 3, "vqd-1")
|
|
u, err := url.Parse(got)
|
|
if err != nil {
|
|
t.Fatalf("url.Parse(%q): %v", got, err)
|
|
}
|
|
q := u.Query()
|
|
if q.Get("vqd") != "vqd-1" {
|
|
t.Fatalf("vqd = %q, want vqd-1", q.Get("vqd"))
|
|
}
|
|
if q.Get("u") != "bing" {
|
|
t.Fatalf("u = %q, want bing", q.Get("u"))
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_ParseGeneralResults(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><body>
|
|
<div class="results">
|
|
<div class="result">
|
|
<a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fragflow.io">RAGFlow</a>
|
|
<a class="result__snippet">Open source RAG engine</a>
|
|
</div>
|
|
<div class="result">
|
|
<a class="result__a" href="https://github.com/infiniflow/ragflow">GitHub</a>
|
|
<div class="result__snippet">Source code repository</div>
|
|
</div>
|
|
</div>
|
|
</body></html>`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
prevSearch := duckduckgoSearchEndpoint
|
|
duckduckgoSearchEndpoint = srv.URL
|
|
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
|
|
|
|
tool := NewDuckDuckGoTool()
|
|
out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","top_n":5}`)
|
|
if err != nil {
|
|
t.Fatalf("InvokableRun: %v", err)
|
|
}
|
|
|
|
var env duckduckgoEnvelope
|
|
if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {
|
|
t.Fatalf("output is not valid JSON: %v (raw=%s)", jerr, out)
|
|
}
|
|
if env.Error != "" {
|
|
t.Errorf("Error = %q, want empty", env.Error)
|
|
}
|
|
if len(env.Results) != 2 {
|
|
t.Fatalf("Results len = %d, want 2", len(env.Results))
|
|
}
|
|
if env.Results[0].Title != "RAGFlow" {
|
|
t.Errorf("Results[0].Title = %q, want RAGFlow", env.Results[0].Title)
|
|
}
|
|
if env.Results[0].URL != "https://ragflow.io" {
|
|
t.Errorf("Results[0].URL = %q, want https://ragflow.io", env.Results[0].URL)
|
|
}
|
|
if env.Results[0].Body != "Open source RAG engine" {
|
|
t.Errorf("Results[0].Body = %q, want Open source RAG engine", env.Results[0].Body)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_ParseNewsResults(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/bootstrap":
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><head><script>var x=""; var vqd="test-vqd-1";</script></head><body></body></html>`))
|
|
case "/news.js":
|
|
if got := r.URL.Query().Get("vqd"); got != "test-vqd-1" {
|
|
t.Fatalf("vqd = %q, want test-vqd-1", got)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{
|
|
"results": [
|
|
{"title":"Story One","url":"https://news.example.com/story-1","excerpt":"Breaking update one"},
|
|
{"title":"Story Two","url":"https://news.example.com/story-2","excerpt":"Breaking & update two"}
|
|
]
|
|
}`))
|
|
default:
|
|
t.Fatalf("unexpected path: %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
|
|
prevNews := duckduckgoNewsEndpoint
|
|
prevBootstrap := duckduckgoNewsBootstrapEndpoint
|
|
duckduckgoNewsEndpoint = srv.URL + "/news.js"
|
|
duckduckgoNewsBootstrapEndpoint = srv.URL + "/bootstrap"
|
|
t.Cleanup(func() { duckduckgoNewsEndpoint = prevNews })
|
|
t.Cleanup(func() { duckduckgoNewsBootstrapEndpoint = prevBootstrap })
|
|
|
|
tool := NewDuckDuckGoTool()
|
|
out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","channel":"news","top_n":1}`)
|
|
if err != nil {
|
|
t.Fatalf("InvokableRun: %v", err)
|
|
}
|
|
|
|
var env duckduckgoEnvelope
|
|
if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {
|
|
t.Fatalf("output is not valid JSON: %v", jerr)
|
|
}
|
|
if len(env.Results) != 1 {
|
|
t.Fatalf("Results len = %d, want 1", len(env.Results))
|
|
}
|
|
if env.Results[0].Title != "Story One" {
|
|
t.Errorf("Results[0].Title = %q, want Story One", env.Results[0].Title)
|
|
}
|
|
if env.Results[0].URL != "https://news.example.com/story-1" {
|
|
t.Errorf("Results[0].URL = %q, want https://news.example.com/story-1", env.Results[0].URL)
|
|
}
|
|
if env.Results[0].Body != "Breaking update one" {
|
|
t.Errorf("Results[0].Body = %q, want Breaking update one", env.Results[0].Body)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_DefaultChannelUsesGeneralSearch(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got := r.URL.Path; got != "/" {
|
|
// keep old behavior impossible to hit if search endpoint override works incorrectly
|
|
}
|
|
if got := r.URL.Query().Get("o"); got != "" {
|
|
t.Fatalf("o = %q, want empty for general search html endpoint", got)
|
|
}
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><body>
|
|
<div class="result"><a class="result__a" href="https://n.example/a">A</a></div>
|
|
</body></html>`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
prevSearch := duckduckgoSearchEndpoint
|
|
duckduckgoSearchEndpoint = srv.URL
|
|
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
|
|
|
|
tool := NewDuckDuckGoTool()
|
|
_, err := tool.InvokableRun(context.Background(), `{"query":"ragflow"}`)
|
|
if err != nil {
|
|
t.Fatalf("InvokableRun: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_Info(t *testing.T) {
|
|
tool := NewDuckDuckGoTool()
|
|
info, err := tool.Info(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Info: %v", err)
|
|
}
|
|
if info.Name != "duckduckgo_search" {
|
|
t.Errorf("Name = %q, want duckduckgo_search", info.Name)
|
|
}
|
|
if !strings.Contains(info.Desc, "DuckDuckGo") {
|
|
t.Errorf("Desc = %q, want to mention DuckDuckGo", info.Desc)
|
|
}
|
|
if info.ParamsOneOf == nil {
|
|
t.Fatal("ParamsOneOf = nil, want schema definition")
|
|
}
|
|
schema, err := info.ParamsOneOf.ToJSONSchema()
|
|
if err != nil {
|
|
t.Fatalf("ToJSONSchema: %v", err)
|
|
}
|
|
raw, err := json.Marshal(schema)
|
|
if err != nil {
|
|
t.Fatalf("marshal params schema: %v", err)
|
|
}
|
|
params := string(raw)
|
|
if !strings.Contains(params, `"channel"`) {
|
|
t.Fatalf("schema missing channel param: %s", params)
|
|
}
|
|
if strings.Contains(params, `"top_n"`) {
|
|
t.Fatalf("schema should not expose top_n param: %s", params)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_EmptyQuery(t *testing.T) {
|
|
tool := NewDuckDuckGoTool()
|
|
out, err := tool.InvokableRun(context.Background(), `{"query":""}`)
|
|
if err != nil {
|
|
t.Fatalf("InvokableRun(empty): %v", err)
|
|
}
|
|
var envelope duckduckgoEnvelope
|
|
if err := json.Unmarshal([]byte(out), &envelope); err != nil || len(envelope.Results) != 0 {
|
|
t.Fatalf("empty result = %s / %v", out, err)
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) {
|
|
var hitCount int
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hitCount++
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><body>
|
|
<div class="result">
|
|
<a class="result__a" href="https://ragflow.io">RAGFlow</a>
|
|
<div class="result__snippet">RAGFlow is an open-source RAG engine.</div>
|
|
</div>
|
|
</body></html>`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
prevSearch := duckduckgoSearchEndpoint
|
|
duckduckgoSearchEndpoint = srv.URL
|
|
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
|
|
|
|
realTool := NewDuckDuckGoTool()
|
|
|
|
mdl := newReactScriptedModel(
|
|
"duckduckgo_search",
|
|
`{"query":"ragflow"}`,
|
|
"RAGFlow is an open-source RAG engine.",
|
|
)
|
|
|
|
agent, err := react.NewAgent(context.Background(), &react.AgentConfig{
|
|
ToolCallingModel: mdl,
|
|
ToolsConfig: compose.ToolsNodeConfig{
|
|
Tools: []einotool.BaseTool{realTool},
|
|
},
|
|
MaxStep: 5,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("react.NewAgent: %v", err)
|
|
}
|
|
|
|
out, err := agent.Generate(context.Background(), []*schema.Message{
|
|
schema.UserMessage("What is RAGFlow?"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("agent.Generate: %v", err)
|
|
}
|
|
if got, want := out.Content, "RAGFlow is an open-source RAG engine."; got != want {
|
|
t.Errorf("Content = %q, want %q", got, want)
|
|
}
|
|
if mdl.turn != 2 {
|
|
t.Errorf("Generate called %d times, want 2 (tool_call + final)", mdl.turn)
|
|
}
|
|
if len(mdl.boundTools) != 1 || mdl.boundTools[0].Name != "duckduckgo_search" {
|
|
names := make([]string, 0, len(mdl.boundTools))
|
|
for _, ti := range mdl.boundTools {
|
|
names = append(names, ti.Name)
|
|
}
|
|
t.Errorf("tools bound to model = %v, want [duckduckgo_search]", names)
|
|
}
|
|
if len(mdl.rounds) < 2 {
|
|
t.Fatalf("only %d rounds captured, want >= 2", len(mdl.rounds))
|
|
}
|
|
var sawToolResult bool
|
|
for _, msg := range mdl.rounds[1] {
|
|
if msg.Role == schema.Tool && strings.Contains(msg.Content, "RAGFlow is an open-source RAG engine") {
|
|
sawToolResult = true
|
|
break
|
|
}
|
|
}
|
|
if !sawToolResult {
|
|
t.Errorf("round 2 input did not contain a ToolMessage carrying the upstream result")
|
|
}
|
|
if hitCount == 0 {
|
|
t.Error("test server was never hit; the tool did not actually call the upstream")
|
|
}
|
|
}
|
|
|
|
func TestDuckDuckGo_ComponentReferencesAndDefaults(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
built, err := BuildByName("duckduckgo", map[string]any{
|
|
"top_n": float64(4),
|
|
"channel": "news",
|
|
"outputs": map[string]any{"json": map[string]any{}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("BuildByName: %v", err)
|
|
}
|
|
duck := built.(*DuckDuckGoTool)
|
|
if duck.defaults.TopN != 4 || duck.defaults.Channel != "news" {
|
|
t.Fatalf("defaults = %+v", duck.defaults)
|
|
}
|
|
for _, params := range []map[string]any{{"top_n": 0}, {"top_n": 1.5}, {"channel": "images"}} {
|
|
if _, err := BuildByName("duckduckgo", params); err == nil {
|
|
t.Fatalf("BuildByName(%#v) succeeded", params)
|
|
}
|
|
}
|
|
spec := duck.ComponentSpec()
|
|
if channel, ok := spec.InputForm["channel"].(map[string]any); !ok || channel["value"] != "general" {
|
|
t.Fatalf("channel input form = %#v", spec.InputForm["channel"])
|
|
}
|
|
envelope := map[string]any{"results": []any{map[string]any{
|
|
"title": "Story", "url": "https://news.example/story", "body": "Breaking update",
|
|
}}}
|
|
chunks, docAggs := duck.BuildReferences(context.Background(), envelope)
|
|
if len(chunks) != 1 || len(docAggs) != 1 || chunks[0]["content"] != "Breaking update" {
|
|
t.Fatalf("references = %#v / %#v", chunks, docAggs)
|
|
}
|
|
outputs := duck.BuildComponentOutputs(envelope)
|
|
if results, ok := outputs["json"].([]any); !ok || len(results) != 1 {
|
|
t.Fatalf("json output = %#v", outputs["json"])
|
|
}
|
|
if !strings.Contains(outputs["formalized_content"].(string), "Breaking update") {
|
|
t.Fatalf("formalized_content = %q", outputs["formalized_content"])
|
|
}
|
|
if _, exists := envelope["chunks"]; exists {
|
|
t.Fatalf("output conversion mutated envelope: %#v", envelope)
|
|
}
|
|
}
|