zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-10 21:55:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(go-agent): align Go DuckDuckGo component with canvas input form (#16775)

Browse Source 
## Summary

- register the Go `DuckDuckGo` canvas component and restore its dynamic
input form metadata
- align the Go component input/output surface with the current canvas
usage for `query`, `channel`, and `top_n`
- fix DuckDuckGo news search in Go by fetching the required `vqd` token
before calling `news.js`, and add targeted regression tests

## Testing

Passed:
- `bash build.sh --test ./internal/agent/tool/... -run 'DuckDuckGo'`
- `bash build.sh --test ./internal/agent/component/... -run
'DuckDuckGo|TestVerifyRegistration_P1'`
- `bash build.sh --test ./internal/agent/component/... -run
'DuckDuckGo'`

Not run:
- frontend tests
- frontend build
- full Go test suite

<img width="1776" height="1092" alt="image"
src="https://github.com/user-attachments/assets/9f3f8e4b-f6b4-4915-b96c-3c5b8c7b8b30"
/>
This commit is contained in:
Hz_
2026-07-10 10:11:13 +08:00
committed by GitHub
parent 8236f2cabb
commit d48a5622df
6 changed files with 760 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
internal/agent/component/duckduckgo_component_test.go Normal file
View File

@@ -0,0 +1,132 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package component
import (
"context"
"encoding/json"
"strings"
"testing"
einotool "github.com/cloudwego/eino/components/tool"
)
type fakeDuckDuckGoInvoker struct {
args map[string]any
}
func (f *fakeDuckDuckGoInvoker) InvokableRun(_ context.Context, argsJSON string, _ ...einotool.Option) (string, error) {
if err := json.Unmarshal([]byte(argsJSON), &f.args); err != nil {
return "", err
}
return `{"results":[{"title":"RAGFlow","url":"https://ragflow.io","body":"Open source RAG engine"}]}`, nil
}
func TestDuckDuckGo_RegisteredFactory(t *testing.T) {
t.Parallel()
c, err := New("DuckDuckGo", nil)
if err != nil {
t.Fatalf("New(DuckDuckGo) errored: %v", err)
}
if got := c.Name(); got != "DuckDuckGo" {
t.Fatalf("Name() = %q, want DuckDuckGo", got)
}
formGetter, ok := c.(interface{ GetInputForm() map[string]any })
if !ok {
t.Fatal("DuckDuckGo component does not expose GetInputForm")
}
form := formGetter.GetInputForm()
query, ok := form["query"].(map[string]any)
if !ok {
t.Fatalf("GetInputForm()[query] has type %T, want map", form["query"])
}
if query["type"] != "line" {
t.Fatalf("GetInputForm()[query][type] = %v, want line", query["type"])
}
channel, ok := form["channel"].(map[string]any)
if !ok {
t.Fatalf("GetInputForm()[channel] has type %T, want map", form["channel"])
}
if channel["value"] != "general" {
t.Fatalf("GetInputForm()[channel][value] = %v, want general", channel["value"])
}
if _, ok := c.Outputs()["formalized_content"]; !ok {
t.Fatal("Outputs() missing formalized_content")
}
if _, ok := c.Outputs()["json"]; !ok {
t.Fatal("Outputs() missing json")
}
}
func TestDuckDuckGo_InvokeAdaptsCanvasInputsAndOutputs(t *testing.T) {
t.Parallel()
fake := &fakeDuckDuckGoInvoker{}
c := newDuckDuckGoComponentWithInvoker(fake)
out, err := c.Invoke(context.Background(), map[string]any{
"query": " privacy search ",
"channel": "news",
"top_n": float64(3),
})
if err != nil {
t.Fatalf("Invoke errored: %v", err)
}
if got := fake.args["query"]; got != "privacy search" {
t.Errorf("query arg = %v, want trimmed query", got)
}
if got := fake.args["channel"]; got != "news" {
t.Errorf("channel arg = %v, want news", got)
}
if got := fake.args["top_n"]; got != float64(3) {
t.Errorf("top_n arg = %v, want 3", got)
}
formalized, _ := out["formalized_content"].(string)
for _, want := range []string{"RAGFlow", "https://ragflow.io", "Open source RAG engine"} {
if !strings.Contains(formalized, want) {
t.Errorf("formalized_content missing %q: %s", want, formalized)
}
}
results, ok := out["json"].([]any)
if !ok {
t.Fatalf("json output has type %T, want []any", out["json"])
}
if len(results) != 1 {
t.Fatalf("json output length = %d, want 1", len(results))
}
}
func TestDuckDuckGo_InvokeEmptyQueryReturnsEmptyPayload(t *testing.T) {
t.Parallel()
c := newDuckDuckGoComponentWithInvoker(&fakeDuckDuckGoInvoker{})
out, err := c.Invoke(context.Background(), map[string]any{"query": " "})
if err != nil {
t.Fatalf("Invoke errored: %v", err)
}
if got := out["formalized_content"]; got != "" {
t.Errorf("formalized_content = %v, want empty string", got)
}
results, ok := out["json"].([]any)
if !ok || len(results) != 0 {
t.Fatalf("json output = %#v, want empty []any", out["json"])
}
}

1
internal/agent/component/fixture_stubs.go
View File

@@ -518,6 +518,7 @@ func init() {
Register(componentNameIteration, NewIterationStub)
Register(componentNameIterationItem, NewIterationItemStub)
Register("BGPT", newBGPTComponent)
Register("DuckDuckGo", newDuckDuckGoComponent)
Register("Google", newGoogleComponent)
Register("YahooFinance", newYahooFinanceComponent)
}

121
internal/agent/component/universe_a_wrappers.go
View File

@@ -313,6 +313,10 @@ type bgptInvoker interface {
InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error)
}
type duckDuckGoInvoker interface {
InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error)
}
// bgptComponent delegates to internal/agent/tool/BGPTTool and adapts
// the tool envelope to the BGPT canvas output contract.
type bgptComponent struct {
@@ -406,6 +410,121 @@ func (c *bgptComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[
return nil, nil
}
// duckDuckGoComponent delegates to internal/agent/tool/DuckDuckGoTool.
type duckDuckGoComponent struct {
inner duckDuckGoInvoker
}
func newDuckDuckGoComponent(_ map[string]any) (Component, error) {
return newDuckDuckGoComponentWithInvoker(agenttool.NewDuckDuckGoTool()), nil
}
func newDuckDuckGoComponentWithInvoker(inner duckDuckGoInvoker) Component {
return &duckDuckGoComponent{inner: inner}
}
func (c *duckDuckGoComponent) Name() string { return "DuckDuckGo" }
func (c *duckDuckGoComponent) Inputs() map[string]string {
return map[string]string{
"query": "Search query.",
"channel": "Search channel: general or news.",
"top_n": "Maximum number of results.",
}
}
func (c *duckDuckGoComponent) GetInputForm() map[string]any {
return map[string]any{
"query": map[string]any{
"name": "Query",
"type": "line",
},
"channel": map[string]any{
"name": "Channel",
"type": "options",
"value": "general",
"options": []string{"general", "news"},
},
}
}
func (c *duckDuckGoComponent) Outputs() map[string]string {
return map[string]string{
"formalized_content": "Rendered search results for downstream LLM prompts.",
"json": "Raw result list.",
}
}
func (c *duckDuckGoComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
query := strings.TrimSpace(stringParam(inputs["query"]))
if query == "" {
return map[string]any{"formalized_content": "", "json": []any{}}, nil
}
args := map[string]any{
"query": query,
}
if channel := strings.TrimSpace(stringParam(inputs["channel"])); channel != "" {
args["channel"] = channel
}
if topN := toIntParam(inputs["top_n"]); topN > 0 {
args["top_n"] = topN
}
argsJSON, _ := json.Marshal(args)
out, err := c.inner.InvokableRun(ctx, string(argsJSON))
decoded := parseToolEnvelope(out)
if err != nil {
if len(decoded) > 0 {
return map[string]any{
"formalized_content": "",
"json": []any{},
"_ERROR": decoded["_ERROR"],
}, nil
}
return nil, fmt.Errorf("canvas: DuckDuckGo: %w", err)
}
results := anySlice(decoded["results"])
return map[string]any{
"formalized_content": renderDuckDuckGoResults(results),
"json": results,
}, nil
}
func (c *duckDuckGoComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {
return nil, nil
}
func renderDuckDuckGoResults(results []any) string {
if len(results) == 0 {
return ""
}
blocks := make([]string, 0, len(results))
for _, item := range results {
m, ok := item.(map[string]any)
if !ok {
continue
}
field := func(key string) string {
v, ok := m[key]
if !ok || v == nil {
return "-"
}
text := strings.TrimSpace(fmt.Sprintf("%v", v))
if text == "" {
return "-"
}
return text
}
blocks = append(blocks, strings.Join([]string{
fmt.Sprintf("Title: %s", field("title")),
fmt.Sprintf("URL: %s", field("url")),
fmt.Sprintf("Body: %s", field("body")),
}, "\n"))
}
return strings.Join(blocks, "\n\n")
}
func stringParam(v any) string {
if s, ok := v.(string); ok {
return s
@@ -1309,6 +1428,7 @@ var (
_ Component = (*tavilySearchComponent)(nil)
_ Component = (*googleComponent)(nil)
_ Component = (*tavilyExtractComponent)(nil)
_ Component = (*duckDuckGoComponent)(nil)
_ Component = (*exesqlComponent)(nil)
_ Component = (*codeExecComponent)(nil)
_ Component = (*yahooFinanceComponent)(nil)
@@ -1319,4 +1439,5 @@ var (
var _ einotool.InvokableTool = (*agenttool.TavilyTool)(nil)
var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil)
var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil)
var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil)
var _ einotool.InvokableTool = (*agenttool.YahooFinanceTool)(nil)

4
internal/agent/component/verify_p1_test.go
View File

@@ -45,8 +45,8 @@ func TestVerifyRegistration_P1(t *testing.T) {
if len(missing) > 0 {
t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names)
}
if got := len(names); got < 12 || got > 34 {
t.Errorf("expected 12-34 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
if got := len(names); got < 12 || got > 35 {
t.Errorf("expected 12-35 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
}
// ExitLoop must NOT be in the registry (legacy compat lives at

410
internal/agent/tool/duckduckgo.go
View File

@@ -20,74 +20,67 @@ import (
"context"
"encoding/json"
"fmt"
"html"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"github.com/cloudwego/eino/components/tool"
"github.com/cloudwego/eino/schema"
xhtml "golang.org/x/net/html"
)
const duckduckgoToolName = "duckduckgo"
const duckduckgoToolDescription = "Search DuckDuckGo's Instant Answer API. Returns the abstract text and a list of related topics."
const duckduckgoToolDescription = "Search DuckDuckGo web or news results. Returns results[].{title, url, body}."
const duckduckgoChannelGeneral = "general"
const duckduckgoChannelNews = "news"
var duckduckgoSearchEndpoint = "https://duckduckgo.com/html/"
var duckduckgoNewsEndpoint = "https://duckduckgo.com/news.js"
var duckduckgoNewsBootstrapEndpoint = "https://duckduckgo.com/"
var duckduckgoVQDPattern = regexp.MustCompile(`vqd="([^"]+)"`)
// duckduckgoParams is the JSON shape the model sends into InvokableRun.
// max_results caps the number of RelatedTopics returned; the upstream
// endpoint itself is uncapped and uses heuristic ranking.
type duckduckgoParams struct {
Query string `json:"query"`
MaxResults int `json:"max_results"`
Query string `json:"query"`
Channel string `json:"channel"`
TopN int `json:"top_n"`
}
// duckduckgoTopic is one element of the upstream `RelatedTopics` array.
// DuckDuckGo returns a recursive tree: a topic may itself have
// `Topics`, and may be a "name"/"value" pair (no URL) or a regular
// hit with `FirstURL`. We flatten one level of nesting.
type duckduckgoTopic struct {
Text string `json:"text,omitempty"`
FirstURL string `json:"first_url,omitempty"`
Topics []duckduckgoTopic `json:"topics,omitempty"`
type duckduckgoResult struct {
Title string `json:"title"`
URL string `json:"url"`
Body string `json:"body"`
}
// duckduckgoResponse is the upstream Instant Answer envelope. We only
// model the fields we care about.
type duckduckgoResponse struct {
AbstractText string `json:"abstract_text"`
AbstractURL string `json:"abstract_url"`
Abstract string `json:"abstract"`
RelatedTopics []duckduckgoTopic `json:"related_topics"`
}
// duckduckgoTopicOut is the model-facing topic shape.
type duckduckgoTopicOut struct {
Text string `json:"text,omitempty"`
FirstURL string `json:"first_url,omitempty"`
}
// duckduckgoEnvelope is the JSON shape the model sees.
type duckduckgoEnvelope struct {
AbstractText string `json:"abstract_text,omitempty"`
AbstractURL string `json:"abstract_url,omitempty"`
RelatedTopics []duckduckgoTopicOut `json:"related_topics,omitempty"`
Error string `json:"_ERROR,omitempty"`
Results []duckduckgoResult `json:"results"`
Error string `json:"_ERROR,omitempty"`
}
type duckduckgoNewsResponse struct {
Results []duckduckgoNewsItem `json:"results"`
}
type duckduckgoNewsItem struct {
Title string `json:"title"`
URL string `json:"url"`
Excerpt string `json:"excerpt"`
}
// DuckDuckGoTool is the DuckDuckGo
// Instant Answer tool. It
// performs a GET against the public Instant Answer endpoint using the
// shared HTTPHelper and returns the abstract + a flat list of related
// topics.
type DuckDuckGoTool struct {
helper *HTTPHelper
}
// NewDuckDuckGoTool returns a DuckDuckGoTool using the default HTTPHelper.
func NewDuckDuckGoTool() *DuckDuckGoTool {
return NewDuckDuckGoToolWith(NewHTTPHelper())
}
// NewDuckDuckGoToolWith returns a DuckDuckGoTool that uses the provided
// HTTPHelper. Useful for tests.
func NewDuckDuckGoToolWith(h *HTTPHelper) *DuckDuckGoTool {
if h == nil {
h = NewHTTPHelper()
@@ -95,7 +88,6 @@ func NewDuckDuckGoToolWith(h *HTTPHelper) *DuckDuckGoTool {
return &DuckDuckGoTool{helper: h}
}
// Info returns the tool's metadata for the chat model.
func (d *DuckDuckGoTool) Info(_ context.Context) (*schema.ToolInfo, error) {
return &schema.ToolInfo{
Name: duckduckgoToolName,
@@ -103,61 +95,88 @@ func (d *DuckDuckGoTool) Info(_ context.Context) (*schema.ToolInfo, error) {
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
"query": {
Type: schema.String,
Desc: "Search query",
Desc: "Search query.",
Required: true,
},
"max_results": {
Type: schema.Integer,
Desc: "Maximum number of related topics to return. Defaults to 5.",
"channel": {
Type: schema.String,
Desc: "Search channel: general or news. Defaults to general.",
Required: false,
},
}),
}, nil
}
// buildDuckDuckGoURL constructs the Instant Answer API URL.
func buildDuckDuckGoURL(query string) string {
func buildDuckDuckGoSearchURL(query string, topN int) string {
if topN <= 0 {
topN = 10
}
q := url.Values{}
q.Set("q", query)
q.Set("format", "json")
q.Set("no_html", "1")
q.Set("skip_disambig", "1")
return "https://api.duckduckgo.com/?" + q.Encode()
q.Set("kl", "wt-wt")
q.Set("dc", strconv.Itoa(topN+1))
return duckduckgoSearchEndpoint + "?" + q.Encode()
}
// flattenDuckDuckGoTopics flattens the upstream topic tree (which can
// have arbitrary nesting) into a list, dropping entries without a URL.
func flattenDuckDuckGoTopics(in []duckduckgoTopic) []duckduckgoTopicOut {
var out []duckduckgoTopicOut
for _, t := range in {
if t.FirstURL != "" && t.Text != "" {
out = append(out, duckduckgoTopicOut{Text: t.Text, FirstURL: t.FirstURL})
}
// recurse one level; upstream nests categories here
if len(t.Topics) > 0 {
out = append(out, flattenDuckDuckGoTopics(t.Topics)...)
}
func buildDuckDuckGoNewsURL(query string, topN int) string {
return buildDuckDuckGoNewsURLWithVQD(query, topN, "")
}
func buildDuckDuckGoNewsURLWithVQD(query string, topN int, vqd string) string {
if topN <= 0 {
topN = 10
}
return out
q := url.Values{}
q.Set("q", query)
q.Set("l", "wt-wt")
q.Set("o", "json")
q.Set("p", "1")
q.Set("s", "0")
q.Set("dc", strconv.Itoa(topN))
if strings.TrimSpace(vqd) != "" {
q.Set("vqd", vqd)
q.Set("u", "bing")
}
return duckduckgoNewsEndpoint + "?" + q.Encode()
}
func buildDuckDuckGoNewsBootstrapURL(query string) string {
q := url.Values{}
q.Set("q", query)
q.Set("iar", "news")
q.Set("ia", "news")
q.Set("kl", "wt-wt")
return duckduckgoNewsBootstrapEndpoint + "?" + q.Encode()
}
// InvokableRun performs the DuckDuckGo Instant Answer query.
func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) {
var p duckduckgoParams
if err := json.Unmarshal([]byte(argsJSON), &p); err != nil {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse arguments: %w", err)),
fmt.Errorf("duckduckgo: parse arguments: %w", err)
}
if p.Query == "" {
if strings.TrimSpace(p.Query) == "" {
return duckduckgoErrJSON(fmt.Errorf("query is required")),
fmt.Errorf("duckduckgo: query is required")
}
if p.MaxResults <= 0 {
p.MaxResults = 5
channel := normalizeDuckDuckGoChannel(p.Channel)
topN := p.TopN
if topN <= 0 {
topN = 10
}
endpoint := buildDuckDuckGoURL(p.Query)
resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil)
if channel == duckduckgoChannelNews {
return d.runNewsSearch(ctx, p.Query, topN)
}
endpoint := buildDuckDuckGoSearchURL(p.Query, topN)
headers := map[string]string{
"User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)",
"Accept": "text/html,application/xhtml+xml",
}
resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)
if err != nil {
return duckduckgoErrJSON(err), err
}
@@ -168,29 +187,244 @@ func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ ..
fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)
}
var raw duckduckgoResponse
results, err := parseDuckDuckGoHTML(resp.Body, topN)
if err != nil {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse html: %w", err)),
fmt.Errorf("duckduckgo: parse html: %w", err)
}
return duckduckgoJSON(duckduckgoEnvelope{Results: results}), nil
}
func (d *DuckDuckGoTool) runNewsSearch(ctx context.Context, query string, topN int) (string, error) {
vqd, err := d.fetchDuckDuckGoNewsVQD(ctx, query)
if err != nil {
return duckduckgoErrJSON(err), err
}
endpoint := buildDuckDuckGoNewsURLWithVQD(query, topN, vqd)
headers := map[string]string{
"User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)",
"Accept": "application/json,text/javascript,*/*",
}
resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)
if err != nil {
return duckduckgoErrJSON(err), err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)),
fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)
}
var raw duckduckgoNewsResponse
if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode response: %w", err)),
fmt.Errorf("duckduckgo: decode response: %w", err)
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode news response: %w", err)),
fmt.Errorf("duckduckgo: decode news response: %w", err)
}
// Prefer AbstractText (rich) over Abstract (plain), as the Python
// tool did historically.
abstract := raw.AbstractText
if abstract == "" {
abstract = raw.Abstract
results := make([]duckduckgoResult, 0, min(topN, len(raw.Results)))
for _, item := range raw.Results {
if len(results) >= topN {
break
}
title := normalizeWhitespace(html.UnescapeString(item.Title))
resultURL := strings.TrimSpace(item.URL)
body := normalizeWhitespace(html.UnescapeString(item.Excerpt))
if title == "" || resultURL == "" {
continue
}
results = append(results, duckduckgoResult{
Title: title,
URL: resultURL,
Body: body,
})
}
topics := flattenDuckDuckGoTopics(raw.RelatedTopics)
if len(topics) > p.MaxResults {
topics = topics[:p.MaxResults]
return duckduckgoJSON(duckduckgoEnvelope{Results: results}), nil
}
func (d *DuckDuckGoTool) fetchDuckDuckGoNewsVQD(ctx context.Context, query string) (string, error) {
endpoint := buildDuckDuckGoNewsBootstrapURL(query)
headers := map[string]string{
"User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)",
"Accept": "text/html,application/xhtml+xml",
}
return duckduckgoJSON(duckduckgoEnvelope{
AbstractText: abstract,
AbstractURL: raw.AbstractURL,
RelatedTopics: topics,
}), nil
resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("duckduckgo: news bootstrap returned %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("duckduckgo: read news bootstrap: %w", err)
}
matches := duckduckgoVQDPattern.FindSubmatch(body)
if len(matches) != 2 {
return "", fmt.Errorf("duckduckgo: news bootstrap missing vqd")
}
return string(matches[1]), nil
}
func normalizeDuckDuckGoChannel(channel string) string {
value := strings.ToLower(strings.TrimSpace(channel))
switch value {
case "", "text", duckduckgoChannelGeneral:
return duckduckgoChannelGeneral
case duckduckgoChannelNews:
return duckduckgoChannelNews
default:
return duckduckgoChannelGeneral
}
}
func parseDuckDuckGoHTML(body interface {
Read(p []byte) (int, error)
}, topN int) ([]duckduckgoResult, error) {
if topN <= 0 {
topN = 10
}
doc, err := xhtml.Parse(body)
if err != nil {
return nil, err
}
return extractDuckDuckGoGeneralResults(doc, topN), nil
}
func extractDuckDuckGoGeneralResults(doc *xhtml.Node, topN int) []duckduckgoResult {
results := make([]duckduckgoResult, 0, topN)
var walk func(*xhtml.Node)
walk = func(n *xhtml.Node) {
if len(results) >= topN {
return
}
if n.Type == xhtml.ElementNode && hasClassToken(n, "result") {
if res, ok := extractDuckDuckGoGeneralResult(n); ok {
results = append(results, res)
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
return results
}
func extractDuckDuckGoGeneralResult(card *xhtml.Node) (duckduckgoResult, bool) {
var out duckduckgoResult
titleNode := findFirstNode(card, func(n *xhtml.Node) bool {
return n.Type == xhtml.ElementNode && n.Data == "a" && hasClassToken(n, "result__a")
})
if titleNode == nil {
return out, false
}
out.Title = normalizeWhitespace(collectText(titleNode))
out.URL = normalizeDuckDuckGoLink(attrValue(titleNode, "href"))
out.Body = normalizeWhitespace(textByClass(card, "result__snippet"))
if out.Title == "" || out.URL == "" {
return duckduckgoResult{}, false
}
return out, true
}
func findFirstNode(root *xhtml.Node, match func(*xhtml.Node) bool) *xhtml.Node {
if root == nil {
return nil
}
if match(root) {
return root
}
for c := root.FirstChild; c != nil; c = c.NextSibling {
if hit := findFirstNode(c, match); hit != nil {
return hit
}
}
return nil
}
func textByClass(root *xhtml.Node, className string) string {
node := findFirstNode(root, func(n *xhtml.Node) bool {
return n.Type == xhtml.ElementNode && hasClassToken(n, className)
})
if node == nil {
return ""
}
return collectText(node)
}
func hasClassToken(n *xhtml.Node, want string) bool {
if n == nil || n.Type != xhtml.ElementNode || want == "" {
return false
}
for _, a := range n.Attr {
if a.Key != "class" {
continue
}
for _, token := range strings.Fields(a.Val) {
if token == want {
return true
}
}
}
return false
}
func attrValue(n *xhtml.Node, key string) string {
if n == nil {
return ""
}
for _, a := range n.Attr {
if a.Key == key {
return strings.TrimSpace(a.Val)
}
}
return ""
}
func normalizeDuckDuckGoLink(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if strings.HasPrefix(raw, "//") {
raw = "https:" + raw
}
parsed, err := url.Parse(raw)
if err != nil {
return raw
}
if parsed.Scheme == "" || parsed.Host == "" {
return raw
}
if strings.Contains(parsed.Host, "duckduckgo.com") {
q := parsed.Query().Get("uddg")
if q != "" {
if decoded, err := url.QueryUnescape(q); err == nil && decoded != "" {
return decoded
}
return q
}
}
return raw
}
func normalizeWhitespace(s string) string {
return strings.Join(strings.Fields(strings.TrimSpace(s)), " ")
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
func duckduckgoJSON(env duckduckgoEnvelope) string {

282
internal/agent/tool/duckduckgo_test.go
View File

@@ -31,58 +31,90 @@ import (
"github.com/cloudwego/eino/schema"
)
func TestDuckDuckGo_BuildURL(t *testing.T) {
t.Parallel()
got := buildDuckDuckGoURL("rag flow")
func TestDuckDuckGo_BuildSearchURL(t *testing.T) {
got := buildDuckDuckGoSearchURL("rag flow", 7)
u, err := url.Parse(got)
if err != nil {
t.Fatalf("url.Parse(%q): %v", got, err)
}
if u.Host != "api.duckduckgo.com" {
t.Errorf("host = %q, want api.duckduckgo.com", u.Host)
if u.Host != "duckduckgo.com" {
t.Errorf("host = %q, want duckduckgo.com", u.Host)
}
if u.Path != "/html/" {
t.Errorf("path = %q, want /html/", u.Path)
}
q := u.Query()
if q.Get("q") != "rag flow" {
t.Errorf("q = %q, want 'rag flow' (no pre-encoding)", q.Get("q"))
t.Errorf("q = %q, want rag flow", q.Get("q"))
}
if q.Get("format") != "json" {
t.Errorf("format = %q, want json", q.Get("format"))
if q.Get("dc") != "8" {
t.Errorf("dc = %q, want 8", q.Get("dc"))
}
if q.Get("no_html") != "1" {
t.Errorf("no_html = %q, want 1", q.Get("no_html"))
if got := q.Get("vqd"); got != "" {
t.Errorf("vqd = %q, want omitted", got)
}
}
func TestDuckDuckGo_ParseTopics(t *testing.T) {
t.Parallel()
func TestDuckDuckGo_BuildNewsURL(t *testing.T) {
got := buildDuckDuckGoNewsURL("rag flow", 3)
u, err := url.Parse(got)
if err != nil {
t.Fatalf("url.Parse(%q): %v", got, err)
}
if u.Path != "/news.js" {
t.Errorf("path = %q, want /news.js", u.Path)
}
q := u.Query()
if q.Get("o") != "json" {
t.Fatalf("o = %q, want json", q.Get("o"))
}
if q.Get("l") != "wt-wt" {
t.Fatalf("l = %q, want wt-wt", q.Get("l"))
}
if q.Get("dc") != "3" {
t.Errorf("dc = %q, want 3", q.Get("dc"))
}
}
func TestDuckDuckGo_BuildNewsURLWithVQD(t *testing.T) {
got := buildDuckDuckGoNewsURLWithVQD("rag flow", 3, "vqd-1")
u, err := url.Parse(got)
if err != nil {
t.Fatalf("url.Parse(%q): %v", got, err)
}
q := u.Query()
if q.Get("vqd") != "vqd-1" {
t.Fatalf("vqd = %q, want vqd-1", q.Get("vqd"))
}
if q.Get("u") != "bing" {
t.Fatalf("u = %q, want bing", q.Get("u"))
}
}
func TestDuckDuckGo_ParseGeneralResults(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// upstream returns a recursive tree; include a category
// (`Topics` non-empty, no `FirstURL`) and a leaf hit.
_, _ = w.Write([]byte(`{
"abstract_text": "RAGFlow is an open-source RAG engine.",
"abstract_url": "https://ragflow.io",
"related_topics": [
{
"text": "Category: Technology",
"topics": [
{"text": "Open source", "first_url": "https://example.com/os"},
{"text": "Search engines", "first_url": "https://example.com/se"}
]
},
{"text": "GitHub", "first_url": "https://github.com/infiniflow/ragflow"}
]
}`))
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`<!doctype html><html><body>
<div class="results">
<div class="result">
<a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fragflow.io">RAGFlow</a>
<a class="result__snippet">Open source RAG engine</a>
</div>
<div class="result">
<a class="result__a" href="https://github.com/infiniflow/ragflow">GitHub</a>
<div class="result__snippet">Source code repository</div>
</div>
</div>
</body></html>`))
}))
defer srv.Close()
helper := NewHTTPHelper().WithClient(&http.Client{
Transport: rewriteHostTransport(srv.URL),
})
tool := NewDuckDuckGoToolWith(helper)
out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","max_results":10}`)
prevSearch := duckduckgoSearchEndpoint
duckduckgoSearchEndpoint = srv.URL
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
tool := NewDuckDuckGoTool()
out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","top_n":5}`)
if err != nil {
t.Fatalf("InvokableRun: %v", err)
}
@@ -94,53 +126,52 @@ func TestDuckDuckGo_ParseTopics(t *testing.T) {
if env.Error != "" {
t.Errorf("Error = %q, want empty", env.Error)
}
if env.AbstractText != "RAGFlow is an open-source RAG engine." {
t.Errorf("AbstractText = %q, want the upstream abstract", env.AbstractText)
if len(env.Results) != 2 {
t.Fatalf("Results len = %d, want 2", len(env.Results))
}
if env.AbstractURL != "https://ragflow.io" {
t.Errorf("AbstractURL = %q, want https://ragflow.io", env.AbstractURL)
if env.Results[0].Title != "RAGFlow" {
t.Errorf("Results[0].Title = %q, want RAGFlow", env.Results[0].Title)
}
if len(env.RelatedTopics) != 3 {
t.Fatalf("RelatedTopics len = %d, want 3 (category child leaves + direct hit)", len(env.RelatedTopics))
if env.Results[0].URL != "https://ragflow.io" {
t.Errorf("Results[0].URL = %q, want https://ragflow.io", env.Results[0].URL)
}
wantURLs := map[string]bool{
"https://example.com/os": false,
"https://example.com/se": false,
"https://github.com/infiniflow/ragflow": false,
}
for _, t2 := range env.RelatedTopics {
if _, ok := wantURLs[t2.FirstURL]; ok {
wantURLs[t2.FirstURL] = true
}
}
for u, seen := range wantURLs {
if !seen {
t.Errorf("missing topic URL %q in flattened result", u)
}
if env.Results[0].Body != "Open source RAG engine" {
t.Errorf("Results[0].Body = %q, want Open source RAG engine", env.Results[0].Body)
}
}
func TestDuckDuckGo_RespectsMaxResults(t *testing.T) {
t.Parallel()
func TestDuckDuckGo_ParseNewsResults(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"related_topics": [
{"text":"A","first_url":"https://a"},
{"text":"B","first_url":"https://b"},
{"text":"C","first_url":"https://c"},
{"text":"D","first_url":"https://d"}
]
}`))
switch r.URL.Path {
case "/bootstrap":
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`<!doctype html><html><head><script>var x=""; var vqd="test-vqd-1";</script></head><body></body></html>`))
case "/news.js":
if got := r.URL.Query().Get("vqd"); got != "test-vqd-1" {
t.Fatalf("vqd = %q, want test-vqd-1", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"results": [
{"title":"Story One","url":"https://news.example.com/story-1","excerpt":"Breaking update one"},
{"title":"Story Two","url":"https://news.example.com/story-2","excerpt":"Breaking &amp; update two"}
]
}`))
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer srv.Close()
helper := NewHTTPHelper().WithClient(&http.Client{
Transport: rewriteHostTransport(srv.URL),
})
tool := NewDuckDuckGoToolWith(helper)
out, err := tool.InvokableRun(context.Background(), `{"query":"x","max_results":2}`)
prevNews := duckduckgoNewsEndpoint
prevBootstrap := duckduckgoNewsBootstrapEndpoint
duckduckgoNewsEndpoint = srv.URL + "/news.js"
duckduckgoNewsBootstrapEndpoint = srv.URL + "/bootstrap"
t.Cleanup(func() { duckduckgoNewsEndpoint = prevNews })
t.Cleanup(func() { duckduckgoNewsBootstrapEndpoint = prevBootstrap })
tool := NewDuckDuckGoTool()
out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","channel":"news","top_n":1}`)
if err != nil {
t.Fatalf("InvokableRun: %v", err)
}
@@ -149,14 +180,47 @@ func TestDuckDuckGo_RespectsMaxResults(t *testing.T) {
if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {
t.Fatalf("output is not valid JSON: %v", jerr)
}
if len(env.RelatedTopics) != 2 {
t.Errorf("RelatedTopics len = %d, want 2 (capped by max_results)", len(env.RelatedTopics))
if len(env.Results) != 1 {
t.Fatalf("Results len = %d, want 1", len(env.Results))
}
if env.Results[0].Title != "Story One" {
t.Errorf("Results[0].Title = %q, want Story One", env.Results[0].Title)
}
if env.Results[0].URL != "https://news.example.com/story-1" {
t.Errorf("Results[0].URL = %q, want https://news.example.com/story-1", env.Results[0].URL)
}
if env.Results[0].Body != "Breaking update one" {
t.Errorf("Results[0].Body = %q, want Breaking update one", env.Results[0].Body)
}
}
func TestDuckDuckGo_DefaultChannelUsesGeneralSearch(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.URL.Path; got != "/" {
// keep old behavior impossible to hit if search endpoint override works incorrectly
}
if got := r.URL.Query().Get("o"); got != "" {
t.Fatalf("o = %q, want empty for general search html endpoint", got)
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`<!doctype html><html><body>
<div class="result"><a class="result__a" href="https://n.example/a">A</a></div>
</body></html>`))
}))
defer srv.Close()
prevSearch := duckduckgoSearchEndpoint
duckduckgoSearchEndpoint = srv.URL
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
tool := NewDuckDuckGoTool()
_, err := tool.InvokableRun(context.Background(), `{"query":"ragflow"}`)
if err != nil {
t.Fatalf("InvokableRun: %v", err)
}
}
func TestDuckDuckGo_Info(t *testing.T) {
t.Parallel()
tool := NewDuckDuckGoTool()
info, err := tool.Info(context.Background())
if err != nil {
@@ -168,38 +232,56 @@ func TestDuckDuckGo_Info(t *testing.T) {
if !strings.Contains(info.Desc, "DuckDuckGo") {
t.Errorf("Desc = %q, want to mention DuckDuckGo", info.Desc)
}
if info.ParamsOneOf == nil {
t.Fatal("ParamsOneOf = nil, want schema definition")
}
schema, err := info.ParamsOneOf.ToJSONSchema()
if err != nil {
t.Fatalf("ToJSONSchema: %v", err)
}
raw, err := json.Marshal(schema)
if err != nil {
t.Fatalf("marshal params schema: %v", err)
}
params := string(raw)
if !strings.Contains(params, `"channel"`) {
t.Fatalf("schema missing channel param: %s", params)
}
if strings.Contains(params, `"top_n"`) {
t.Fatalf("schema should not expose top_n param: %s", params)
}
}
// TestDuckDuckGo_RealReactAgent_ExecutesTool drives a real eino
// react.NewAgent with the real DuckDuckGoTool (httptest-stubbed
// upstream) and a scripted chat model. Proves the HTTP-based tool is
// actually invoked by eino, its JSON envelope is fed back as a
// ToolMessage, and the model can ground a final answer in the
// retrieved abstract. This is the HTTP-side counterpart to
// TestExeSQL_RealReactAgent_ExecutesTool — together they cover the
// two distinct wiring patterns (DB-backed vs HTTP-backed) the agent
// needs to handle.
func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) {
t.Parallel()
func TestDuckDuckGo_RequiresQuery(t *testing.T) {
tool := NewDuckDuckGoTool()
_, err := tool.InvokableRun(context.Background(), `{"query":""}`)
if err == nil {
t.Fatal("expected error for empty query")
}
if !strings.Contains(err.Error(), "query") {
t.Errorf("err = %v, want to mention query", err)
}
}
func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) {
var hitCount int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hitCount++
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"abstract_text": "RAGFlow is an open-source RAG engine.",
"abstract_url": "https://ragflow.io",
"related_topics": [
{"text": "GitHub", "first_url": "https://github.com/infiniflow/ragflow"}
]
}`))
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`<!doctype html><html><body>
<div class="result">
<a class="result__a" href="https://ragflow.io">RAGFlow</a>
<div class="result__snippet">RAGFlow is an open-source RAG engine.</div>
</div>
</body></html>`))
}))
defer srv.Close()
helper := NewHTTPHelper().WithClient(&http.Client{
Transport: rewriteHostTransport(srv.URL),
})
realTool := NewDuckDuckGoToolWith(helper)
prevSearch := duckduckgoSearchEndpoint
duckduckgoSearchEndpoint = srv.URL
t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch })
realTool := NewDuckDuckGoTool()
mdl := newReactScriptedModel(
"duckduckgo",
@@ -248,7 +330,7 @@ func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) {
}
}
if !sawToolResult {
t.Errorf("round 2 input did not contain a ToolMessage carrying the upstream abstract")
t.Errorf("round 2 input did not contain a ToolMessage carrying the upstream result")
}
if hitCount == 0 {
t.Error("test server was never hit; the tool did not actually call the upstream")