diff --git a/internal/agent/component/duckduckgo_component_test.go b/internal/agent/component/duckduckgo_component_test.go new file mode 100644 index 0000000000..2729a7d3b0 --- /dev/null +++ b/internal/agent/component/duckduckgo_component_test.go @@ -0,0 +1,132 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package component + +import ( + "context" + "encoding/json" + "strings" + "testing" + + einotool "github.com/cloudwego/eino/components/tool" +) + +type fakeDuckDuckGoInvoker struct { + args map[string]any +} + +func (f *fakeDuckDuckGoInvoker) InvokableRun(_ context.Context, argsJSON string, _ ...einotool.Option) (string, error) { + if err := json.Unmarshal([]byte(argsJSON), &f.args); err != nil { + return "", err + } + return `{"results":[{"title":"RAGFlow","url":"https://ragflow.io","body":"Open source RAG engine"}]}`, nil +} + +func TestDuckDuckGo_RegisteredFactory(t *testing.T) { + t.Parallel() + + c, err := New("DuckDuckGo", nil) + if err != nil { + t.Fatalf("New(DuckDuckGo) errored: %v", err) + } + if got := c.Name(); got != "DuckDuckGo" { + t.Fatalf("Name() = %q, want DuckDuckGo", got) + } + formGetter, ok := c.(interface{ GetInputForm() map[string]any }) + if !ok { + t.Fatal("DuckDuckGo component does not expose GetInputForm") + } + form := formGetter.GetInputForm() + query, ok := form["query"].(map[string]any) + if !ok { + t.Fatalf("GetInputForm()[query] has type %T, want map", form["query"]) + } + if query["type"] != "line" { + t.Fatalf("GetInputForm()[query][type] = %v, want line", query["type"]) + } + channel, ok := form["channel"].(map[string]any) + if !ok { + t.Fatalf("GetInputForm()[channel] has type %T, want map", form["channel"]) + } + if channel["value"] != "general" { + t.Fatalf("GetInputForm()[channel][value] = %v, want general", channel["value"]) + } + if _, ok := c.Outputs()["formalized_content"]; !ok { + t.Fatal("Outputs() missing formalized_content") + } + if _, ok := c.Outputs()["json"]; !ok { + t.Fatal("Outputs() missing json") + } +} + +func TestDuckDuckGo_InvokeAdaptsCanvasInputsAndOutputs(t *testing.T) { + t.Parallel() + + fake := &fakeDuckDuckGoInvoker{} + c := newDuckDuckGoComponentWithInvoker(fake) + + out, err := c.Invoke(context.Background(), map[string]any{ + "query": " privacy search ", + "channel": "news", + "top_n": float64(3), + }) + if err != nil { + t.Fatalf("Invoke errored: %v", err) + } + + if got := fake.args["query"]; got != "privacy search" { + t.Errorf("query arg = %v, want trimmed query", got) + } + if got := fake.args["channel"]; got != "news" { + t.Errorf("channel arg = %v, want news", got) + } + if got := fake.args["top_n"]; got != float64(3) { + t.Errorf("top_n arg = %v, want 3", got) + } + + formalized, _ := out["formalized_content"].(string) + for _, want := range []string{"RAGFlow", "https://ragflow.io", "Open source RAG engine"} { + if !strings.Contains(formalized, want) { + t.Errorf("formalized_content missing %q: %s", want, formalized) + } + } + + results, ok := out["json"].([]any) + if !ok { + t.Fatalf("json output has type %T, want []any", out["json"]) + } + if len(results) != 1 { + t.Fatalf("json output length = %d, want 1", len(results)) + } +} + +func TestDuckDuckGo_InvokeEmptyQueryReturnsEmptyPayload(t *testing.T) { + t.Parallel() + + c := newDuckDuckGoComponentWithInvoker(&fakeDuckDuckGoInvoker{}) + out, err := c.Invoke(context.Background(), map[string]any{"query": " "}) + if err != nil { + t.Fatalf("Invoke errored: %v", err) + } + if got := out["formalized_content"]; got != "" { + t.Errorf("formalized_content = %v, want empty string", got) + } + results, ok := out["json"].([]any) + if !ok || len(results) != 0 { + t.Fatalf("json output = %#v, want empty []any", out["json"]) + } +} diff --git a/internal/agent/component/fixture_stubs.go b/internal/agent/component/fixture_stubs.go index bc96244364..3f11863ce9 100644 --- a/internal/agent/component/fixture_stubs.go +++ b/internal/agent/component/fixture_stubs.go @@ -518,6 +518,7 @@ func init() { Register(componentNameIteration, NewIterationStub) Register(componentNameIterationItem, NewIterationItemStub) Register("BGPT", newBGPTComponent) + Register("DuckDuckGo", newDuckDuckGoComponent) Register("Google", newGoogleComponent) Register("YahooFinance", newYahooFinanceComponent) } diff --git a/internal/agent/component/universe_a_wrappers.go b/internal/agent/component/universe_a_wrappers.go index fee78a1213..bf8f47dd64 100644 --- a/internal/agent/component/universe_a_wrappers.go +++ b/internal/agent/component/universe_a_wrappers.go @@ -313,6 +313,10 @@ type bgptInvoker interface { InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error) } +type duckDuckGoInvoker interface { + InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error) +} + // bgptComponent delegates to internal/agent/tool/BGPTTool and adapts // the tool envelope to the BGPT canvas output contract. type bgptComponent struct { @@ -406,6 +410,121 @@ func (c *bgptComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[ return nil, nil } +// duckDuckGoComponent delegates to internal/agent/tool/DuckDuckGoTool. +type duckDuckGoComponent struct { + inner duckDuckGoInvoker +} + +func newDuckDuckGoComponent(_ map[string]any) (Component, error) { + return newDuckDuckGoComponentWithInvoker(agenttool.NewDuckDuckGoTool()), nil +} + +func newDuckDuckGoComponentWithInvoker(inner duckDuckGoInvoker) Component { + return &duckDuckGoComponent{inner: inner} +} + +func (c *duckDuckGoComponent) Name() string { return "DuckDuckGo" } + +func (c *duckDuckGoComponent) Inputs() map[string]string { + return map[string]string{ + "query": "Search query.", + "channel": "Search channel: general or news.", + "top_n": "Maximum number of results.", + } +} + +func (c *duckDuckGoComponent) GetInputForm() map[string]any { + return map[string]any{ + "query": map[string]any{ + "name": "Query", + "type": "line", + }, + "channel": map[string]any{ + "name": "Channel", + "type": "options", + "value": "general", + "options": []string{"general", "news"}, + }, + } +} + +func (c *duckDuckGoComponent) Outputs() map[string]string { + return map[string]string{ + "formalized_content": "Rendered search results for downstream LLM prompts.", + "json": "Raw result list.", + } +} + +func (c *duckDuckGoComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + query := strings.TrimSpace(stringParam(inputs["query"])) + if query == "" { + return map[string]any{"formalized_content": "", "json": []any{}}, nil + } + args := map[string]any{ + "query": query, + } + if channel := strings.TrimSpace(stringParam(inputs["channel"])); channel != "" { + args["channel"] = channel + } + if topN := toIntParam(inputs["top_n"]); topN > 0 { + args["top_n"] = topN + } + + argsJSON, _ := json.Marshal(args) + out, err := c.inner.InvokableRun(ctx, string(argsJSON)) + decoded := parseToolEnvelope(out) + if err != nil { + if len(decoded) > 0 { + return map[string]any{ + "formalized_content": "", + "json": []any{}, + "_ERROR": decoded["_ERROR"], + }, nil + } + return nil, fmt.Errorf("canvas: DuckDuckGo: %w", err) + } + + results := anySlice(decoded["results"]) + return map[string]any{ + "formalized_content": renderDuckDuckGoResults(results), + "json": results, + }, nil +} + +func (c *duckDuckGoComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) { + return nil, nil +} + +func renderDuckDuckGoResults(results []any) string { + if len(results) == 0 { + return "" + } + blocks := make([]string, 0, len(results)) + for _, item := range results { + m, ok := item.(map[string]any) + if !ok { + continue + } + field := func(key string) string { + v, ok := m[key] + if !ok || v == nil { + return "-" + } + text := strings.TrimSpace(fmt.Sprintf("%v", v)) + if text == "" { + return "-" + } + return text + } + blocks = append(blocks, strings.Join([]string{ + fmt.Sprintf("Title: %s", field("title")), + fmt.Sprintf("URL: %s", field("url")), + fmt.Sprintf("Body: %s", field("body")), + }, "\n")) + } + return strings.Join(blocks, "\n\n") +} + func stringParam(v any) string { if s, ok := v.(string); ok { return s @@ -1309,6 +1428,7 @@ var ( _ Component = (*tavilySearchComponent)(nil) _ Component = (*googleComponent)(nil) _ Component = (*tavilyExtractComponent)(nil) + _ Component = (*duckDuckGoComponent)(nil) _ Component = (*exesqlComponent)(nil) _ Component = (*codeExecComponent)(nil) _ Component = (*yahooFinanceComponent)(nil) @@ -1319,4 +1439,5 @@ var ( var _ einotool.InvokableTool = (*agenttool.TavilyTool)(nil) var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil) var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil) +var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil) var _ einotool.InvokableTool = (*agenttool.YahooFinanceTool)(nil) diff --git a/internal/agent/component/verify_p1_test.go b/internal/agent/component/verify_p1_test.go index 58827dad2b..446a134304 100644 --- a/internal/agent/component/verify_p1_test.go +++ b/internal/agent/component/verify_p1_test.go @@ -45,8 +45,8 @@ func TestVerifyRegistration_P1(t *testing.T) { if len(missing) > 0 { t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names) } - if got := len(names); got < 12 || got > 34 { - t.Errorf("expected 12-34 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names) + if got := len(names); got < 12 || got > 35 { + t.Errorf("expected 12-35 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names) } // ExitLoop must NOT be in the registry (legacy compat lives at diff --git a/internal/agent/tool/duckduckgo.go b/internal/agent/tool/duckduckgo.go index 91f203fc18..a6e3a56e64 100644 --- a/internal/agent/tool/duckduckgo.go +++ b/internal/agent/tool/duckduckgo.go @@ -20,74 +20,67 @@ import ( "context" "encoding/json" "fmt" + "html" + "io" "net/http" "net/url" + "regexp" + "strconv" + "strings" "github.com/cloudwego/eino/components/tool" "github.com/cloudwego/eino/schema" + xhtml "golang.org/x/net/html" ) const duckduckgoToolName = "duckduckgo" -const duckduckgoToolDescription = "Search DuckDuckGo's Instant Answer API. Returns the abstract text and a list of related topics." +const duckduckgoToolDescription = "Search DuckDuckGo web or news results. Returns results[].{title, url, body}." + +const duckduckgoChannelGeneral = "general" +const duckduckgoChannelNews = "news" + +var duckduckgoSearchEndpoint = "https://duckduckgo.com/html/" +var duckduckgoNewsEndpoint = "https://duckduckgo.com/news.js" +var duckduckgoNewsBootstrapEndpoint = "https://duckduckgo.com/" + +var duckduckgoVQDPattern = regexp.MustCompile(`vqd="([^"]+)"`) -// duckduckgoParams is the JSON shape the model sends into InvokableRun. -// max_results caps the number of RelatedTopics returned; the upstream -// endpoint itself is uncapped and uses heuristic ranking. type duckduckgoParams struct { - Query string `json:"query"` - MaxResults int `json:"max_results"` + Query string `json:"query"` + Channel string `json:"channel"` + TopN int `json:"top_n"` } -// duckduckgoTopic is one element of the upstream `RelatedTopics` array. -// DuckDuckGo returns a recursive tree: a topic may itself have -// `Topics`, and may be a "name"/"value" pair (no URL) or a regular -// hit with `FirstURL`. We flatten one level of nesting. -type duckduckgoTopic struct { - Text string `json:"text,omitempty"` - FirstURL string `json:"first_url,omitempty"` - Topics []duckduckgoTopic `json:"topics,omitempty"` +type duckduckgoResult struct { + Title string `json:"title"` + URL string `json:"url"` + Body string `json:"body"` } -// duckduckgoResponse is the upstream Instant Answer envelope. We only -// model the fields we care about. -type duckduckgoResponse struct { - AbstractText string `json:"abstract_text"` - AbstractURL string `json:"abstract_url"` - Abstract string `json:"abstract"` - RelatedTopics []duckduckgoTopic `json:"related_topics"` -} - -// duckduckgoTopicOut is the model-facing topic shape. -type duckduckgoTopicOut struct { - Text string `json:"text,omitempty"` - FirstURL string `json:"first_url,omitempty"` -} - -// duckduckgoEnvelope is the JSON shape the model sees. type duckduckgoEnvelope struct { - AbstractText string `json:"abstract_text,omitempty"` - AbstractURL string `json:"abstract_url,omitempty"` - RelatedTopics []duckduckgoTopicOut `json:"related_topics,omitempty"` - Error string `json:"_ERROR,omitempty"` + Results []duckduckgoResult `json:"results"` + Error string `json:"_ERROR,omitempty"` +} + +type duckduckgoNewsResponse struct { + Results []duckduckgoNewsItem `json:"results"` +} + +type duckduckgoNewsItem struct { + Title string `json:"title"` + URL string `json:"url"` + Excerpt string `json:"excerpt"` } -// DuckDuckGoTool is the DuckDuckGo -// Instant Answer tool. It -// performs a GET against the public Instant Answer endpoint using the -// shared HTTPHelper and returns the abstract + a flat list of related -// topics. type DuckDuckGoTool struct { helper *HTTPHelper } -// NewDuckDuckGoTool returns a DuckDuckGoTool using the default HTTPHelper. func NewDuckDuckGoTool() *DuckDuckGoTool { return NewDuckDuckGoToolWith(NewHTTPHelper()) } -// NewDuckDuckGoToolWith returns a DuckDuckGoTool that uses the provided -// HTTPHelper. Useful for tests. func NewDuckDuckGoToolWith(h *HTTPHelper) *DuckDuckGoTool { if h == nil { h = NewHTTPHelper() @@ -95,7 +88,6 @@ func NewDuckDuckGoToolWith(h *HTTPHelper) *DuckDuckGoTool { return &DuckDuckGoTool{helper: h} } -// Info returns the tool's metadata for the chat model. func (d *DuckDuckGoTool) Info(_ context.Context) (*schema.ToolInfo, error) { return &schema.ToolInfo{ Name: duckduckgoToolName, @@ -103,61 +95,88 @@ func (d *DuckDuckGoTool) Info(_ context.Context) (*schema.ToolInfo, error) { ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{ "query": { Type: schema.String, - Desc: "Search query", + Desc: "Search query.", Required: true, }, - "max_results": { - Type: schema.Integer, - Desc: "Maximum number of related topics to return. Defaults to 5.", + "channel": { + Type: schema.String, + Desc: "Search channel: general or news. Defaults to general.", Required: false, }, }), }, nil } -// buildDuckDuckGoURL constructs the Instant Answer API URL. -func buildDuckDuckGoURL(query string) string { +func buildDuckDuckGoSearchURL(query string, topN int) string { + if topN <= 0 { + topN = 10 + } q := url.Values{} q.Set("q", query) - q.Set("format", "json") - q.Set("no_html", "1") - q.Set("skip_disambig", "1") - return "https://api.duckduckgo.com/?" + q.Encode() + q.Set("kl", "wt-wt") + q.Set("dc", strconv.Itoa(topN+1)) + return duckduckgoSearchEndpoint + "?" + q.Encode() } -// flattenDuckDuckGoTopics flattens the upstream topic tree (which can -// have arbitrary nesting) into a list, dropping entries without a URL. -func flattenDuckDuckGoTopics(in []duckduckgoTopic) []duckduckgoTopicOut { - var out []duckduckgoTopicOut - for _, t := range in { - if t.FirstURL != "" && t.Text != "" { - out = append(out, duckduckgoTopicOut{Text: t.Text, FirstURL: t.FirstURL}) - } - // recurse one level; upstream nests categories here - if len(t.Topics) > 0 { - out = append(out, flattenDuckDuckGoTopics(t.Topics)...) - } +func buildDuckDuckGoNewsURL(query string, topN int) string { + return buildDuckDuckGoNewsURLWithVQD(query, topN, "") +} + +func buildDuckDuckGoNewsURLWithVQD(query string, topN int, vqd string) string { + if topN <= 0 { + topN = 10 } - return out + q := url.Values{} + q.Set("q", query) + q.Set("l", "wt-wt") + q.Set("o", "json") + q.Set("p", "1") + q.Set("s", "0") + q.Set("dc", strconv.Itoa(topN)) + if strings.TrimSpace(vqd) != "" { + q.Set("vqd", vqd) + q.Set("u", "bing") + } + return duckduckgoNewsEndpoint + "?" + q.Encode() +} + +func buildDuckDuckGoNewsBootstrapURL(query string) string { + q := url.Values{} + q.Set("q", query) + q.Set("iar", "news") + q.Set("ia", "news") + q.Set("kl", "wt-wt") + return duckduckgoNewsBootstrapEndpoint + "?" + q.Encode() } -// InvokableRun performs the DuckDuckGo Instant Answer query. func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) { var p duckduckgoParams if err := json.Unmarshal([]byte(argsJSON), &p); err != nil { return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse arguments: %w", err)), fmt.Errorf("duckduckgo: parse arguments: %w", err) } - if p.Query == "" { + if strings.TrimSpace(p.Query) == "" { return duckduckgoErrJSON(fmt.Errorf("query is required")), fmt.Errorf("duckduckgo: query is required") } - if p.MaxResults <= 0 { - p.MaxResults = 5 + + channel := normalizeDuckDuckGoChannel(p.Channel) + topN := p.TopN + if topN <= 0 { + topN = 10 } - endpoint := buildDuckDuckGoURL(p.Query) - resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil) + if channel == duckduckgoChannelNews { + return d.runNewsSearch(ctx, p.Query, topN) + } + + endpoint := buildDuckDuckGoSearchURL(p.Query, topN) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", + "Accept": "text/html,application/xhtml+xml", + } + + resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers) if err != nil { return duckduckgoErrJSON(err), err } @@ -168,29 +187,244 @@ func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ .. fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode) } - var raw duckduckgoResponse + results, err := parseDuckDuckGoHTML(resp.Body, topN) + if err != nil { + return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse html: %w", err)), + fmt.Errorf("duckduckgo: parse html: %w", err) + } + return duckduckgoJSON(duckduckgoEnvelope{Results: results}), nil +} + +func (d *DuckDuckGoTool) runNewsSearch(ctx context.Context, query string, topN int) (string, error) { + vqd, err := d.fetchDuckDuckGoNewsVQD(ctx, query) + if err != nil { + return duckduckgoErrJSON(err), err + } + + endpoint := buildDuckDuckGoNewsURLWithVQD(query, topN, vqd) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", + "Accept": "application/json,text/javascript,*/*", + } + + resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers) + if err != nil { + return duckduckgoErrJSON(err), err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return duckduckgoErrJSON(fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)), + fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode) + } + + var raw duckduckgoNewsResponse if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { - return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode response: %w", err)), - fmt.Errorf("duckduckgo: decode response: %w", err) + return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode news response: %w", err)), + fmt.Errorf("duckduckgo: decode news response: %w", err) } - // Prefer AbstractText (rich) over Abstract (plain), as the Python - // tool did historically. - abstract := raw.AbstractText - if abstract == "" { - abstract = raw.Abstract + results := make([]duckduckgoResult, 0, min(topN, len(raw.Results))) + for _, item := range raw.Results { + if len(results) >= topN { + break + } + title := normalizeWhitespace(html.UnescapeString(item.Title)) + resultURL := strings.TrimSpace(item.URL) + body := normalizeWhitespace(html.UnescapeString(item.Excerpt)) + if title == "" || resultURL == "" { + continue + } + results = append(results, duckduckgoResult{ + Title: title, + URL: resultURL, + Body: body, + }) } - topics := flattenDuckDuckGoTopics(raw.RelatedTopics) - if len(topics) > p.MaxResults { - topics = topics[:p.MaxResults] + return duckduckgoJSON(duckduckgoEnvelope{Results: results}), nil +} + +func (d *DuckDuckGoTool) fetchDuckDuckGoNewsVQD(ctx context.Context, query string) (string, error) { + endpoint := buildDuckDuckGoNewsBootstrapURL(query) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", + "Accept": "text/html,application/xhtml+xml", } - return duckduckgoJSON(duckduckgoEnvelope{ - AbstractText: abstract, - AbstractURL: raw.AbstractURL, - RelatedTopics: topics, - }), nil + resp, err := d.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("duckduckgo: news bootstrap returned %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("duckduckgo: read news bootstrap: %w", err) + } + matches := duckduckgoVQDPattern.FindSubmatch(body) + if len(matches) != 2 { + return "", fmt.Errorf("duckduckgo: news bootstrap missing vqd") + } + return string(matches[1]), nil +} + +func normalizeDuckDuckGoChannel(channel string) string { + value := strings.ToLower(strings.TrimSpace(channel)) + switch value { + case "", "text", duckduckgoChannelGeneral: + return duckduckgoChannelGeneral + case duckduckgoChannelNews: + return duckduckgoChannelNews + default: + return duckduckgoChannelGeneral + } +} + +func parseDuckDuckGoHTML(body interface { + Read(p []byte) (int, error) +}, topN int) ([]duckduckgoResult, error) { + if topN <= 0 { + topN = 10 + } + doc, err := xhtml.Parse(body) + if err != nil { + return nil, err + } + return extractDuckDuckGoGeneralResults(doc, topN), nil +} + +func extractDuckDuckGoGeneralResults(doc *xhtml.Node, topN int) []duckduckgoResult { + results := make([]duckduckgoResult, 0, topN) + var walk func(*xhtml.Node) + walk = func(n *xhtml.Node) { + if len(results) >= topN { + return + } + if n.Type == xhtml.ElementNode && hasClassToken(n, "result") { + if res, ok := extractDuckDuckGoGeneralResult(n); ok { + results = append(results, res) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return results +} + +func extractDuckDuckGoGeneralResult(card *xhtml.Node) (duckduckgoResult, bool) { + var out duckduckgoResult + titleNode := findFirstNode(card, func(n *xhtml.Node) bool { + return n.Type == xhtml.ElementNode && n.Data == "a" && hasClassToken(n, "result__a") + }) + if titleNode == nil { + return out, false + } + out.Title = normalizeWhitespace(collectText(titleNode)) + out.URL = normalizeDuckDuckGoLink(attrValue(titleNode, "href")) + out.Body = normalizeWhitespace(textByClass(card, "result__snippet")) + if out.Title == "" || out.URL == "" { + return duckduckgoResult{}, false + } + return out, true +} + +func findFirstNode(root *xhtml.Node, match func(*xhtml.Node) bool) *xhtml.Node { + if root == nil { + return nil + } + if match(root) { + return root + } + for c := root.FirstChild; c != nil; c = c.NextSibling { + if hit := findFirstNode(c, match); hit != nil { + return hit + } + } + return nil +} + +func textByClass(root *xhtml.Node, className string) string { + node := findFirstNode(root, func(n *xhtml.Node) bool { + return n.Type == xhtml.ElementNode && hasClassToken(n, className) + }) + if node == nil { + return "" + } + return collectText(node) +} + +func hasClassToken(n *xhtml.Node, want string) bool { + if n == nil || n.Type != xhtml.ElementNode || want == "" { + return false + } + for _, a := range n.Attr { + if a.Key != "class" { + continue + } + for _, token := range strings.Fields(a.Val) { + if token == want { + return true + } + } + } + return false +} + +func attrValue(n *xhtml.Node, key string) string { + if n == nil { + return "" + } + for _, a := range n.Attr { + if a.Key == key { + return strings.TrimSpace(a.Val) + } + } + return "" +} + +func normalizeDuckDuckGoLink(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + if strings.HasPrefix(raw, "//") { + raw = "https:" + raw + } + parsed, err := url.Parse(raw) + if err != nil { + return raw + } + if parsed.Scheme == "" || parsed.Host == "" { + return raw + } + if strings.Contains(parsed.Host, "duckduckgo.com") { + q := parsed.Query().Get("uddg") + if q != "" { + if decoded, err := url.QueryUnescape(q); err == nil && decoded != "" { + return decoded + } + return q + } + } + return raw +} + +func normalizeWhitespace(s string) string { + return strings.Join(strings.Fields(strings.TrimSpace(s)), " ") +} + +func min(a, b int) int { + if a < b { + return a + } + return b } func duckduckgoJSON(env duckduckgoEnvelope) string { diff --git a/internal/agent/tool/duckduckgo_test.go b/internal/agent/tool/duckduckgo_test.go index 09eb72372d..de6ead6dce 100644 --- a/internal/agent/tool/duckduckgo_test.go +++ b/internal/agent/tool/duckduckgo_test.go @@ -31,58 +31,90 @@ import ( "github.com/cloudwego/eino/schema" ) -func TestDuckDuckGo_BuildURL(t *testing.T) { - t.Parallel() - - got := buildDuckDuckGoURL("rag flow") +func TestDuckDuckGo_BuildSearchURL(t *testing.T) { + got := buildDuckDuckGoSearchURL("rag flow", 7) u, err := url.Parse(got) if err != nil { t.Fatalf("url.Parse(%q): %v", got, err) } - if u.Host != "api.duckduckgo.com" { - t.Errorf("host = %q, want api.duckduckgo.com", u.Host) + if u.Host != "duckduckgo.com" { + t.Errorf("host = %q, want duckduckgo.com", u.Host) + } + if u.Path != "/html/" { + t.Errorf("path = %q, want /html/", u.Path) } q := u.Query() if q.Get("q") != "rag flow" { - t.Errorf("q = %q, want 'rag flow' (no pre-encoding)", q.Get("q")) + t.Errorf("q = %q, want rag flow", q.Get("q")) } - if q.Get("format") != "json" { - t.Errorf("format = %q, want json", q.Get("format")) + if q.Get("dc") != "8" { + t.Errorf("dc = %q, want 8", q.Get("dc")) } - if q.Get("no_html") != "1" { - t.Errorf("no_html = %q, want 1", q.Get("no_html")) + if got := q.Get("vqd"); got != "" { + t.Errorf("vqd = %q, want omitted", got) } } -func TestDuckDuckGo_ParseTopics(t *testing.T) { - t.Parallel() +func TestDuckDuckGo_BuildNewsURL(t *testing.T) { + got := buildDuckDuckGoNewsURL("rag flow", 3) + u, err := url.Parse(got) + if err != nil { + t.Fatalf("url.Parse(%q): %v", got, err) + } + if u.Path != "/news.js" { + t.Errorf("path = %q, want /news.js", u.Path) + } + q := u.Query() + if q.Get("o") != "json" { + t.Fatalf("o = %q, want json", q.Get("o")) + } + if q.Get("l") != "wt-wt" { + t.Fatalf("l = %q, want wt-wt", q.Get("l")) + } + if q.Get("dc") != "3" { + t.Errorf("dc = %q, want 3", q.Get("dc")) + } +} +func TestDuckDuckGo_BuildNewsURLWithVQD(t *testing.T) { + got := buildDuckDuckGoNewsURLWithVQD("rag flow", 3, "vqd-1") + u, err := url.Parse(got) + if err != nil { + t.Fatalf("url.Parse(%q): %v", got, err) + } + q := u.Query() + if q.Get("vqd") != "vqd-1" { + t.Fatalf("vqd = %q, want vqd-1", q.Get("vqd")) + } + if q.Get("u") != "bing" { + t.Fatalf("u = %q, want bing", q.Get("u")) + } +} + +func TestDuckDuckGo_ParseGeneralResults(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - // upstream returns a recursive tree; include a category - // (`Topics` non-empty, no `FirstURL`) and a leaf hit. - _, _ = w.Write([]byte(`{ - "abstract_text": "RAGFlow is an open-source RAG engine.", - "abstract_url": "https://ragflow.io", - "related_topics": [ - { - "text": "Category: Technology", - "topics": [ - {"text": "Open source", "first_url": "https://example.com/os"}, - {"text": "Search engines", "first_url": "https://example.com/se"} - ] - }, - {"text": "GitHub", "first_url": "https://github.com/infiniflow/ragflow"} - ] - }`)) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` +
+
+ RAGFlow + Open source RAG engine +
+
+ GitHub +
Source code repository
+
+
+ `)) })) defer srv.Close() - helper := NewHTTPHelper().WithClient(&http.Client{ - Transport: rewriteHostTransport(srv.URL), - }) - tool := NewDuckDuckGoToolWith(helper) - out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","max_results":10}`) + prevSearch := duckduckgoSearchEndpoint + duckduckgoSearchEndpoint = srv.URL + t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch }) + + tool := NewDuckDuckGoTool() + out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","top_n":5}`) if err != nil { t.Fatalf("InvokableRun: %v", err) } @@ -94,53 +126,52 @@ func TestDuckDuckGo_ParseTopics(t *testing.T) { if env.Error != "" { t.Errorf("Error = %q, want empty", env.Error) } - if env.AbstractText != "RAGFlow is an open-source RAG engine." { - t.Errorf("AbstractText = %q, want the upstream abstract", env.AbstractText) + if len(env.Results) != 2 { + t.Fatalf("Results len = %d, want 2", len(env.Results)) } - if env.AbstractURL != "https://ragflow.io" { - t.Errorf("AbstractURL = %q, want https://ragflow.io", env.AbstractURL) + if env.Results[0].Title != "RAGFlow" { + t.Errorf("Results[0].Title = %q, want RAGFlow", env.Results[0].Title) } - if len(env.RelatedTopics) != 3 { - t.Fatalf("RelatedTopics len = %d, want 3 (category child leaves + direct hit)", len(env.RelatedTopics)) + if env.Results[0].URL != "https://ragflow.io" { + t.Errorf("Results[0].URL = %q, want https://ragflow.io", env.Results[0].URL) } - wantURLs := map[string]bool{ - "https://example.com/os": false, - "https://example.com/se": false, - "https://github.com/infiniflow/ragflow": false, - } - for _, t2 := range env.RelatedTopics { - if _, ok := wantURLs[t2.FirstURL]; ok { - wantURLs[t2.FirstURL] = true - } - } - for u, seen := range wantURLs { - if !seen { - t.Errorf("missing topic URL %q in flattened result", u) - } + if env.Results[0].Body != "Open source RAG engine" { + t.Errorf("Results[0].Body = %q, want Open source RAG engine", env.Results[0].Body) } } -func TestDuckDuckGo_RespectsMaxResults(t *testing.T) { - t.Parallel() - +func TestDuckDuckGo_ParseNewsResults(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "related_topics": [ - {"text":"A","first_url":"https://a"}, - {"text":"B","first_url":"https://b"}, - {"text":"C","first_url":"https://c"}, - {"text":"D","first_url":"https://d"} - ] - }`)) + switch r.URL.Path { + case "/bootstrap": + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(``)) + case "/news.js": + if got := r.URL.Query().Get("vqd"); got != "test-vqd-1" { + t.Fatalf("vqd = %q, want test-vqd-1", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "results": [ + {"title":"Story One","url":"https://news.example.com/story-1","excerpt":"Breaking update one"}, + {"title":"Story Two","url":"https://news.example.com/story-2","excerpt":"Breaking & update two"} + ] + }`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } })) defer srv.Close() - helper := NewHTTPHelper().WithClient(&http.Client{ - Transport: rewriteHostTransport(srv.URL), - }) - tool := NewDuckDuckGoToolWith(helper) - out, err := tool.InvokableRun(context.Background(), `{"query":"x","max_results":2}`) + prevNews := duckduckgoNewsEndpoint + prevBootstrap := duckduckgoNewsBootstrapEndpoint + duckduckgoNewsEndpoint = srv.URL + "/news.js" + duckduckgoNewsBootstrapEndpoint = srv.URL + "/bootstrap" + t.Cleanup(func() { duckduckgoNewsEndpoint = prevNews }) + t.Cleanup(func() { duckduckgoNewsBootstrapEndpoint = prevBootstrap }) + + tool := NewDuckDuckGoTool() + out, err := tool.InvokableRun(context.Background(), `{"query":"ragflow","channel":"news","top_n":1}`) if err != nil { t.Fatalf("InvokableRun: %v", err) } @@ -149,14 +180,47 @@ func TestDuckDuckGo_RespectsMaxResults(t *testing.T) { if jerr := json.Unmarshal([]byte(out), &env); jerr != nil { t.Fatalf("output is not valid JSON: %v", jerr) } - if len(env.RelatedTopics) != 2 { - t.Errorf("RelatedTopics len = %d, want 2 (capped by max_results)", len(env.RelatedTopics)) + if len(env.Results) != 1 { + t.Fatalf("Results len = %d, want 1", len(env.Results)) + } + if env.Results[0].Title != "Story One" { + t.Errorf("Results[0].Title = %q, want Story One", env.Results[0].Title) + } + if env.Results[0].URL != "https://news.example.com/story-1" { + t.Errorf("Results[0].URL = %q, want https://news.example.com/story-1", env.Results[0].URL) + } + if env.Results[0].Body != "Breaking update one" { + t.Errorf("Results[0].Body = %q, want Breaking update one", env.Results[0].Body) + } +} + +func TestDuckDuckGo_DefaultChannelUsesGeneralSearch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Path; got != "/" { + // keep old behavior impossible to hit if search endpoint override works incorrectly + } + if got := r.URL.Query().Get("o"); got != "" { + t.Fatalf("o = %q, want empty for general search html endpoint", got) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` +
A
+ `)) + })) + defer srv.Close() + + prevSearch := duckduckgoSearchEndpoint + duckduckgoSearchEndpoint = srv.URL + t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch }) + + tool := NewDuckDuckGoTool() + _, err := tool.InvokableRun(context.Background(), `{"query":"ragflow"}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) } } func TestDuckDuckGo_Info(t *testing.T) { - t.Parallel() - tool := NewDuckDuckGoTool() info, err := tool.Info(context.Background()) if err != nil { @@ -168,38 +232,56 @@ func TestDuckDuckGo_Info(t *testing.T) { if !strings.Contains(info.Desc, "DuckDuckGo") { t.Errorf("Desc = %q, want to mention DuckDuckGo", info.Desc) } + if info.ParamsOneOf == nil { + t.Fatal("ParamsOneOf = nil, want schema definition") + } + schema, err := info.ParamsOneOf.ToJSONSchema() + if err != nil { + t.Fatalf("ToJSONSchema: %v", err) + } + raw, err := json.Marshal(schema) + if err != nil { + t.Fatalf("marshal params schema: %v", err) + } + params := string(raw) + if !strings.Contains(params, `"channel"`) { + t.Fatalf("schema missing channel param: %s", params) + } + if strings.Contains(params, `"top_n"`) { + t.Fatalf("schema should not expose top_n param: %s", params) + } } -// TestDuckDuckGo_RealReactAgent_ExecutesTool drives a real eino -// react.NewAgent with the real DuckDuckGoTool (httptest-stubbed -// upstream) and a scripted chat model. Proves the HTTP-based tool is -// actually invoked by eino, its JSON envelope is fed back as a -// ToolMessage, and the model can ground a final answer in the -// retrieved abstract. This is the HTTP-side counterpart to -// TestExeSQL_RealReactAgent_ExecutesTool — together they cover the -// two distinct wiring patterns (DB-backed vs HTTP-backed) the agent -// needs to handle. -func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) { - t.Parallel() +func TestDuckDuckGo_RequiresQuery(t *testing.T) { + tool := NewDuckDuckGoTool() + _, err := tool.InvokableRun(context.Background(), `{"query":""}`) + if err == nil { + t.Fatal("expected error for empty query") + } + if !strings.Contains(err.Error(), "query") { + t.Errorf("err = %v, want to mention query", err) + } +} +func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) { var hitCount int srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hitCount++ - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "abstract_text": "RAGFlow is an open-source RAG engine.", - "abstract_url": "https://ragflow.io", - "related_topics": [ - {"text": "GitHub", "first_url": "https://github.com/infiniflow/ragflow"} - ] - }`)) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` +
+ RAGFlow +
RAGFlow is an open-source RAG engine.
+
+ `)) })) defer srv.Close() - helper := NewHTTPHelper().WithClient(&http.Client{ - Transport: rewriteHostTransport(srv.URL), - }) - realTool := NewDuckDuckGoToolWith(helper) + prevSearch := duckduckgoSearchEndpoint + duckduckgoSearchEndpoint = srv.URL + t.Cleanup(func() { duckduckgoSearchEndpoint = prevSearch }) + + realTool := NewDuckDuckGoTool() mdl := newReactScriptedModel( "duckduckgo", @@ -248,7 +330,7 @@ func TestDuckDuckGo_RealReactAgent_ExecutesTool(t *testing.T) { } } if !sawToolResult { - t.Errorf("round 2 input did not contain a ToolMessage carrying the upstream abstract") + t.Errorf("round 2 input did not contain a ToolMessage carrying the upstream result") } if hitCount == 0 { t.Error("test server was never hit; the tool did not actually call the upstream")