From 347a45967ec5912247dac2336335c6464fa10027 Mon Sep 17 00:00:00 2001 From: Hz_ Date: Mon, 13 Jul 2026 10:19:50 +0800 Subject: [PATCH] fix(go-agent): ArXiv component registration and request parity (#16808) ## Summary - register the Go `ArXiv` canvas component and add its input form - align the Go ArXiv request/schema with Python by keeping only `query` in runtime args and moving `top_n`/`sort_by` to node params - keep ArXiv results consistent for canvas output and tool response handling ## Test - `bash build.sh --test ./internal/agent/tool ./internal/agent/component` image --- .../agent/component/arxiv_component_test.go | 151 ++++++++++++++++++ internal/agent/component/fixture_stubs.go | 1 + .../agent/component/universe_a_wrappers.go | 107 +++++++++++++ internal/agent/component/verify_p1_test.go | 9 +- internal/agent/tool/arxiv.go | 81 ++++++++-- internal/agent/tool/arxiv_test.go | 14 +- internal/agent/tool/registry.go | 27 +++- 7 files changed, 366 insertions(+), 24 deletions(-) create mode 100644 internal/agent/component/arxiv_component_test.go diff --git a/internal/agent/component/arxiv_component_test.go b/internal/agent/component/arxiv_component_test.go new file mode 100644 index 0000000000..6baa4a4bd5 --- /dev/null +++ b/internal/agent/component/arxiv_component_test.go @@ -0,0 +1,151 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package component + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + agenttool "ragflow/internal/agent/tool" +) + +type arxivRoundTripper func(*http.Request) (*http.Response, error) + +func (fn arxivRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return fn(request) +} + +func TestArXiv_RegisteredFactoryAndInputForm(t *testing.T) { + t.Parallel() + + c, err := New("ArXiv", map[string]any{ + "top_n": float64(7), + "sort_by": "relevance", + }) + if err != nil { + t.Fatalf("New(ArXiv): %v", err) + } + if got := c.Name(); got != "ArXiv" { + t.Fatalf("Name() = %q, want ArXiv", got) + } + formGetter, ok := c.(interface{ GetInputForm() map[string]any }) + if !ok { + t.Fatal("ArXiv component does not expose GetInputForm") + } + query, ok := formGetter.GetInputForm()["query"].(map[string]any) + if !ok || query["type"] != "line" { + t.Fatalf("GetInputForm()[query] = %#v, want Query line input", query) + } + if _, ok := c.Outputs()["formalized_content"]; !ok { + t.Fatal("Outputs() missing formalized_content") + } + if _, ok := c.Outputs()["json"]; !ok { + t.Fatal("Outputs() missing json") + } +} + +func TestArXiv_InvalidNodeParams(t *testing.T) { + t.Parallel() + + for _, params := range []map[string]any{ + {"top_n": 0}, + {"sort_by": "newest"}, + } { + if _, err := New("ArXiv", params); err == nil { + t.Fatalf("New(ArXiv, %#v) succeeded, want validation error", params) + } + } +} + +func TestArXiv_InvokeSendsOnlyQueryAndFormatsPythonFields(t *testing.T) { + t.Parallel() + + const atom = ` + + + http://arxiv.org/abs/2501.12345v1 + Test Paper + Paper summary. + Author One + + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if got := query.Get("search_query"); got != "all:retrieval augmented generation" { + t.Errorf("search_query = %q", got) + } + if got := query.Get("max_results"); got != "7" { + t.Errorf("max_results = %q, want 7", got) + } + if got := query.Get("sortBy"); got != "relevance" { + t.Errorf("sortBy = %q, want relevance", got) + } + _, _ = w.Write([]byte(atom)) + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(server.URL): %v", err) + } + component := &arxivComponent{inner: agenttool.NewArxivToolWithParams(agenttool.NewHTTPHelper().WithClient(&http.Client{ + Transport: arxivRoundTripper(func(request *http.Request) (*http.Response, error) { + request.URL.Scheme = serverURL.Scheme + request.URL.Host = serverURL.Host + return http.DefaultTransport.RoundTrip(request) + }), + }), 7, "relevance")} + + out, err := component.Invoke(context.Background(), map[string]any{"query": " retrieval augmented generation "}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + formalized, _ := out["formalized_content"].(string) + for _, want := range []string{"Test Paper", "http://arxiv.org/pdf/2501.12345v1", "Paper summary."} { + if !strings.Contains(formalized, want) { + t.Errorf("formalized_content missing %q: %s", want, formalized) + } + } + if results, ok := out["json"].([]any); !ok || len(results) != 1 { + t.Fatalf("json output = %#v, want one paper", out["json"]) + } +} + +func TestArXiv_InvokeEmptyQueryReturnsEmptyPayload(t *testing.T) { + t.Parallel() + + c, err := newArxivComponent(nil) + if err != nil { + t.Fatalf("newArxivComponent: %v", err) + } + out, err := c.Invoke(context.Background(), map[string]any{"query": " "}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if got := out["formalized_content"]; got != "" { + t.Errorf("formalized_content = %v, want empty string", got) + } + if results, ok := out["json"].([]any); !ok || len(results) != 0 { + t.Fatalf("json output = %#v, want empty []any", out["json"]) + } +} diff --git a/internal/agent/component/fixture_stubs.go b/internal/agent/component/fixture_stubs.go index ee1c0d8735..1b8be85f77 100644 --- a/internal/agent/component/fixture_stubs.go +++ b/internal/agent/component/fixture_stubs.go @@ -522,6 +522,7 @@ func init() { Register("DuckDuckGo", newDuckDuckGoComponent) Register("Google", newGoogleComponent) Register("GoogleScholar", newGoogleScholarComponent) + Register("ArXiv", newArxivComponent) Register("PubMed", newPubMedComponent) Register("YahooFinance", newYahooFinanceComponent) } diff --git a/internal/agent/component/universe_a_wrappers.go b/internal/agent/component/universe_a_wrappers.go index f7414618f1..cf98059dcb 100644 --- a/internal/agent/component/universe_a_wrappers.go +++ b/internal/agent/component/universe_a_wrappers.go @@ -1501,6 +1501,112 @@ func (c *yahooFinanceComponent) Stream(_ context.Context, _ map[string]any) (<-c return nil, nil } +// arxivComponent delegates to internal/agent/tool/ArxivTool. Query is the +// runtime input; top_n and sort_by are validated node parameters. +type arxivComponent struct { + inner *agenttool.ArxivTool +} + +func newArxivComponent(params map[string]any) (Component, error) { + topN := 12 + if v, ok := params["top_n"]; ok { + topN = toIntParam(v) + } + if topN <= 0 { + return nil, fmt.Errorf("canvas: ArXiv: top_n must be a positive integer") + } + sortBy := "submittedDate" + if v, ok := params["sort_by"].(string); ok && strings.TrimSpace(v) != "" { + sortBy = strings.TrimSpace(v) + } + if !agenttool.ArxivSortBySupported(sortBy) { + return nil, fmt.Errorf("canvas: ArXiv: unsupported sort_by %q", sortBy) + } + return &arxivComponent{inner: agenttool.NewArxivToolWithParams(nil, topN, sortBy)}, nil +} + +func (c *arxivComponent) Name() string { return "ArXiv" } + +func (c *arxivComponent) Inputs() map[string]string { + return map[string]string{ + "query": "Search query.", + } +} + +func (c *arxivComponent) Outputs() map[string]string { + return map[string]string{ + "formalized_content": "Rendered arXiv papers for downstream LLM prompts.", + "json": "Raw arXiv paper list.", + } +} + +func (c *arxivComponent) GetInputForm() map[string]any { + return map[string]any{ + "query": map[string]any{ + "name": "Query", + "type": "line", + }, + } +} + +func (c *arxivComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + query := strings.TrimSpace(stringParam(inputs["query"])) + if query == "" { + return map[string]any{"formalized_content": "", "json": []any{}}, nil + } + argsJSON, _ := json.Marshal(map[string]any{"query": query}) + out, err := c.inner.InvokableRun(ctx, string(argsJSON)) + decoded := parseToolEnvelope(out) + if err != nil { + if len(decoded) > 0 { + return map[string]any{ + "formalized_content": "", + "json": []any{}, + "_ERROR": decoded["_ERROR"], + }, nil + } + return nil, fmt.Errorf("canvas: ArXiv: %w", err) + } + results := anySlice(decoded["results"]) + return map[string]any{ + "formalized_content": renderArxivResults(results), + "json": results, + }, nil +} + +func (c *arxivComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) { + return nil, nil +} + +func renderArxivResults(results []any) string { + if len(results) == 0 { + return "" + } + blocks := make([]string, 0, len(results)) + for _, item := range results { + paper, ok := item.(map[string]any) + if !ok { + continue + } + summary := strings.TrimSpace(stringParam(paper["summary"])) + if summary == "" { + continue + } + title := strings.TrimSpace(stringParam(paper["title"])) + url := strings.TrimSpace(stringParam(paper["pdf_url"])) + lines := make([]string, 0, 3) + if title != "" { + lines = append(lines, fmt.Sprintf("Title: %s", title)) + } + if url != "" { + lines = append(lines, fmt.Sprintf("URL: %s", url)) + } + lines = append(lines, summary) + blocks = append(blocks, strings.Join(lines, "\n")) + } + return strings.Join(blocks, "\n\n") +} + // googleScholarComponent delegates to internal/agent/tool/GoogleScholarTool. type googleScholarComponent struct { inner googleScholarInvoker @@ -1781,6 +1887,7 @@ var ( _ Component = (*duckDuckGoComponent)(nil) _ Component = (*exesqlComponent)(nil) _ Component = (*codeExecComponent)(nil) + _ Component = (*arxivComponent)(nil) _ Component = (*wikipediaComponent)(nil) _ Component = (*googleScholarComponent)(nil) _ Component = (*pubMedComponent)(nil) diff --git a/internal/agent/component/verify_p1_test.go b/internal/agent/component/verify_p1_test.go index f8ea31b7f3..f5f54968da 100644 --- a/internal/agent/component/verify_p1_test.go +++ b/internal/agent/component/verify_p1_test.go @@ -11,10 +11,11 @@ import ( // read from plan §2.11.10 — P0 (8) + P1 (5) + P2 (4) + P3 (2) + P4 (3) = 22 // at plan completion, plus v1 fixture wrappers/stubs (including Retrieval, // TavilySearch, TavilyExtract, ExeSQL, CodeExec, Google, BGPT, YahooFinance, -// Wikipedia, GoogleScholar, PubMed, DuckDuckGo, Generate, Answer, Iteration, -// and IterationItem) registered by fixture_stubs.go to keep the dsl-examples -// and canvas tool surface compiling. The test allows counts between 12 (P0+P1 -// minus the removed ExitLoop) and 55 to roll forward as subsequent batches land. +// Wikipedia, GoogleScholar, ArXiv, PubMed, DuckDuckGo, Generate, Answer, +// Iteration, and IterationItem) registered by fixture_stubs.go to keep the +// dsl-examples and canvas tool surface compiling. The test allows counts +// between 12 (P0+P1 minus the removed ExitLoop) and 55 to roll forward as +// subsequent batches land. // // Note: ExitLoop is intentionally NOT in the registry anymore. The // canvas engine (internal/agent/canvas/canvas.go's legacyNoOpNames) diff --git a/internal/agent/tool/arxiv.go b/internal/agent/tool/arxiv.go index 39cd3f9965..6071c6d1bf 100644 --- a/internal/agent/tool/arxiv.go +++ b/internal/agent/tool/arxiv.go @@ -35,10 +35,16 @@ const arxivToolName = "arxiv" const arxivToolDescription = "Search arXiv and return matching preprints as {title, authors, summary, pdf_url, entry_id}." -// arxivParams is the JSON shape the model sends into InvokableRun. +const defaultArxivTopN = 12 + +const defaultArxivSortBy = "submittedDate" + +// arxivParams carries the query and ArXiv search settings. Info exposes only +// query to match Python's tool meta; top_n and sort_by come from node params. type arxivParams struct { - Query string `json:"query"` - MaxResults int `json:"max_results"` + Query string `json:"query"` + TopN int `json:"top_n"` + SortBy string `json:"sort_by"` } // arxivResult is one entry in the model-facing result list. @@ -88,7 +94,8 @@ type arxivLink struct { // against the public ArXiv API and parses the Atom XML response // using the stdlib encoding/xml package. type ArxivTool struct { - helper *HTTPHelper + helper *HTTPHelper + defaults arxivParams } // NewArxivTool returns an ArxivTool using the default HTTPHelper. @@ -99,10 +106,22 @@ func NewArxivTool() *ArxivTool { // NewArxivToolWith returns an ArxivTool that uses the provided // HTTPHelper. Useful for tests. func NewArxivToolWith(h *HTTPHelper) *ArxivTool { + return NewArxivToolWithParams(h, defaultArxivTopN, defaultArxivSortBy) +} + +// NewArxivToolWithParams returns an ArxivTool with node-level search +// settings. Query remains the only model-provided argument. +func NewArxivToolWithParams(h *HTTPHelper, topN int, sortBy string) *ArxivTool { if h == nil { h = NewHTTPHelper() } - return &ArxivTool{helper: h} + if topN <= 0 { + topN = defaultArxivTopN + } + if sortBy == "" { + sortBy = defaultArxivSortBy + } + return &ArxivTool{helper: h, defaults: arxivParams{TopN: topN, SortBy: sortBy}} } // Info returns the tool's metadata for the chat model. @@ -113,26 +132,25 @@ func (a *ArxivTool) Info(_ context.Context) (*schema.ToolInfo, error) { ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{ "query": { Type: schema.String, - Desc: "Search query (matches the arXiv `all:` field).", + Desc: "The search keywords to execute with arXiv. The keywords should be the most important words/terms(includes synonyms) from the original request.", Required: true, }, - "max_results": { - Type: schema.Integer, - Desc: "Maximum number of results to return. Defaults to 5.", - Required: false, - }, }), }, nil } // buildArxivURL constructs the ArXiv /api/query URL. -func buildArxivURL(query string, maxResults int) string { - if maxResults <= 0 { - maxResults = 5 +func buildArxivURL(query string, topN int, sortBy string) string { + if topN <= 0 { + topN = defaultArxivTopN + } + if sortBy == "" { + sortBy = defaultArxivSortBy } q := url.Values{} q.Set("search_query", "all:"+query) - q.Set("max_results", fmt.Sprintf("%d", maxResults)) + q.Set("max_results", fmt.Sprintf("%d", topN)) + q.Set("sortBy", sortBy) return "http://export.arxiv.org/api/query?" + q.Encode() } @@ -209,12 +227,21 @@ func (a *ArxivTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool return arxivErrJSON(fmt.Errorf("arxiv: parse arguments: %w", err)), fmt.Errorf("arxiv: parse arguments: %w", err) } - if p.Query == "" { + p = mergeArxivDefaults(a.defaults, p) + if strings.TrimSpace(p.Query) == "" { return arxivErrJSON(fmt.Errorf("query is required")), fmt.Errorf("arxiv: query is required") } + if p.TopN <= 0 { + return arxivErrJSON(fmt.Errorf("top_n must be a positive integer")), + fmt.Errorf("arxiv: top_n must be a positive integer") + } + if !ArxivSortBySupported(p.SortBy) { + return arxivErrJSON(fmt.Errorf("unsupported sort_by %q", p.SortBy)), + fmt.Errorf("arxiv: unsupported sort_by %q", p.SortBy) + } - endpoint := buildArxivURL(p.Query, p.MaxResults) + endpoint := buildArxivURL(strings.TrimSpace(p.Query), p.TopN, p.SortBy) resp, err := a.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil) if err != nil { return arxivErrJSON(err), err @@ -239,6 +266,26 @@ func (a *ArxivTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool return arxivJSON(arxivEnvelope{Results: results}), nil } +func mergeArxivDefaults(defaults, p arxivParams) arxivParams { + if p.TopN == 0 { + p.TopN = defaults.TopN + } + if p.SortBy == "" { + p.SortBy = defaults.SortBy + } + return p +} + +// ArxivSortBySupported reports whether sortBy is accepted by the ArXiv API. +func ArxivSortBySupported(sortBy string) bool { + switch sortBy { + case "submittedDate", "lastUpdatedDate", "relevance": + return true + default: + return false + } +} + func arxivJSON(env arxivEnvelope) string { b, err := json.Marshal(env) if err != nil { diff --git a/internal/agent/tool/arxiv_test.go b/internal/agent/tool/arxiv_test.go index 1fe090503d..67f19acc44 100644 --- a/internal/agent/tool/arxiv_test.go +++ b/internal/agent/tool/arxiv_test.go @@ -29,7 +29,7 @@ import ( func TestArxiv_BuildURL(t *testing.T) { t.Parallel() - got := buildArxivURL("transformer", 3) + got := buildArxivURL("transformer", 3, "lastUpdatedDate") u, err := url.Parse(got) if err != nil { t.Fatalf("url.Parse(%q): %v", got, err) @@ -47,6 +47,9 @@ func TestArxiv_BuildURL(t *testing.T) { if q.Get("max_results") != "3" { t.Errorf("max_results = %q, want 3", q.Get("max_results")) } + if q.Get("sortBy") != "lastUpdatedDate" { + t.Errorf("sortBy = %q, want lastUpdatedDate", q.Get("sortBy")) + } } func TestArxiv_ParseAtomEntry(t *testing.T) { @@ -134,6 +137,13 @@ func TestArxiv_Info(t *testing.T) { if !strings.Contains(info.Desc, "arXiv") { t.Errorf("Desc = %q, want to mention arXiv", info.Desc) } + params, err := json.Marshal(info.ParamsOneOf) + if err != nil { + t.Fatalf("marshal params: %v", err) + } + if strings.Contains(string(params), "top_n") || strings.Contains(string(params), "sort_by") || strings.Contains(string(params), "max_results") { + t.Errorf("schema must only expose query: %s", params) + } } func TestArxiv_RequiresQuery(t *testing.T) { @@ -176,7 +186,7 @@ func TestArxiv_FullRoundtrip(t *testing.T) { Transport: rewriteHostTransport(srv.URL), }) tool := NewArxivToolWith(helper) - out, err := tool.InvokableRun(context.Background(), `{"query":"rag","max_results":5}`) + out, err := tool.InvokableRun(context.Background(), `{"query":"rag"}`) if err != nil { t.Fatalf("InvokableRun: %v", err) } diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.go index 89636bdc84..1fcac9500b 100644 --- a/internal/agent/tool/registry.go +++ b/internal/agent/tool/registry.go @@ -30,7 +30,7 @@ type Factory func(params map[string]any) (einotool.BaseTool, error) var registry = map[string]Factory{ "akshare": buildAkShareTool, - "arxiv": noConfig("arxiv", func() einotool.BaseTool { return NewArxivTool() }), + "arxiv": buildArxivTool, "bgpt": noConfig("bgpt", func() einotool.BaseTool { return NewBGPTTool() }), "code_exec": noConfig("code_exec", func() einotool.BaseTool { return NewCodeExecTool() }), "crawler": noConfig("crawler", func() einotool.BaseTool { return NewCrawlerTool() }), @@ -128,6 +128,31 @@ func buildAkShareTool(params map[string]any) (einotool.BaseTool, error) { return NewAkShareToolWithTopN(nil, topN), nil } +func buildArxivTool(params map[string]any) (einotool.BaseTool, error) { + topN := defaultArxivTopN + sortBy := defaultArxivSortBy + for key := range params { + switch key { + case "top_n", "sort_by": + default: + return nil, fmt.Errorf("agent tool: tool %q does not accept node-level param %s", "arxiv", key) + } + } + if v, ok := intParam(params, "top_n"); ok { + topN = v + } + if topN <= 0 { + return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "arxiv") + } + if v, ok := stringParam(params, "sort_by"); ok { + sortBy = v + } + if !ArxivSortBySupported(sortBy) { + return nil, fmt.Errorf("agent tool: tool %q has unsupported sort_by %q", "arxiv", sortBy) + } + return NewArxivToolWithParams(nil, topN, sortBy), nil +} + func buildExeSQLTool(params map[string]any) (einotool.BaseTool, error) { conn, err := decodeExeSQLConnParams(params) if err != nil {