diff --git a/internal/agent/component/fixture_stubs.go b/internal/agent/component/fixture_stubs.go index a6a6ccaa4c..d20ce08e89 100644 --- a/internal/agent/component/fixture_stubs.go +++ b/internal/agent/component/fixture_stubs.go @@ -518,6 +518,7 @@ func init() { Register(componentNameIteration, NewIterationStub) Register(componentNameIterationItem, NewIterationItemStub) Register("BGPT", newBGPTComponent) + Register("Wikipedia", newWikipediaComponent) Register("DuckDuckGo", newDuckDuckGoComponent) Register("Google", newGoogleComponent) Register("GoogleScholar", newGoogleScholarComponent) diff --git a/internal/agent/component/universe_a_wrappers.go b/internal/agent/component/universe_a_wrappers.go index 514d174514..8a3f907132 100644 --- a/internal/agent/component/universe_a_wrappers.go +++ b/internal/agent/component/universe_a_wrappers.go @@ -410,6 +410,85 @@ func (c *bgptComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[ return nil, nil } +// wikipediaComponent delegates to internal/agent/tool/WikipediaTool. +// Python's canvas component is named "Wikipedia" and stores top_n/language +// on the node params while accepting query at runtime. +type wikipediaComponent struct { + inner *agenttool.WikipediaTool +} + +func newWikipediaComponent(params map[string]any) (Component, error) { + topN := 10 + if v, ok := params["top_n"]; ok { + topN = toIntParam(v) + } + if topN <= 0 { + return nil, fmt.Errorf("canvas: Wikipedia: top_n must be a positive integer") + } + language := "en" + if v, ok := params["language"].(string); ok && strings.TrimSpace(v) != "" { + language = strings.TrimSpace(v) + } + if !agenttool.WikipediaLanguageSupported(language) { + return nil, fmt.Errorf("canvas: Wikipedia: unsupported language %q", language) + } + return &wikipediaComponent{inner: agenttool.NewWikipediaToolWithParams(nil, topN, language)}, nil +} + +func (c *wikipediaComponent) Name() string { return "Wikipedia" } + +func (c *wikipediaComponent) Inputs() map[string]string { + return map[string]string{ + "query": "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.", + } +} + +func (c *wikipediaComponent) GetInputForm() map[string]any { + return map[string]any{ + "query": map[string]any{ + "name": "Query", + "type": "line", + }, + } +} + +func (c *wikipediaComponent) Outputs() map[string]string { + return map[string]string{ + "formalized_content": "Rendered Wikipedia article summaries for downstream LLM prompts.", + "json": "Raw Wikipedia result list.", + } +} + +func (c *wikipediaComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + query := strings.TrimSpace(stringParam(inputs["query"])) + if query == "" { + return map[string]any{"formalized_content": "", "json": []any{}}, nil + } + argsJSON, _ := json.Marshal(map[string]any{"query": query}) + out, err := c.inner.InvokableRun(ctx, string(argsJSON)) + decoded := parseToolEnvelope(out) + if results, ok := decoded["results"]; ok { + decoded["json"] = results + } + if err != nil { + if len(decoded) > 0 { + if _, ok := decoded["formalized_content"]; !ok { + decoded["formalized_content"] = "" + } + if _, ok := decoded["json"]; !ok { + decoded["json"] = []any{} + } + return decoded, nil + } + return nil, fmt.Errorf("canvas: Wikipedia: %w", err) + } + return decoded, nil +} + +func (c *wikipediaComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) { + return nil, nil +} + // duckDuckGoComponent delegates to internal/agent/tool/DuckDuckGoTool. type duckDuckGoComponent struct { inner duckDuckGoInvoker @@ -1580,6 +1659,7 @@ var ( _ Component = (*duckDuckGoComponent)(nil) _ Component = (*exesqlComponent)(nil) _ Component = (*codeExecComponent)(nil) + _ Component = (*wikipediaComponent)(nil) _ Component = (*googleScholarComponent)(nil) _ Component = (*yahooFinanceComponent)(nil) ) @@ -1587,6 +1667,7 @@ var ( // Compile-time check that the eino InvokableTool methods we call // are reachable (catches a future refactor that renames them). var _ einotool.InvokableTool = (*agenttool.TavilyTool)(nil) +var _ einotool.InvokableTool = (*agenttool.WikipediaTool)(nil) var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil) var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil) var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil) diff --git a/internal/agent/component/verify_p1_test.go b/internal/agent/component/verify_p1_test.go index d3b37d38fe..a0fabb24a5 100644 --- a/internal/agent/component/verify_p1_test.go +++ b/internal/agent/component/verify_p1_test.go @@ -10,12 +10,11 @@ import ( // case-insensitive, and returned in sorted order. The expected count is // read from plan §2.11.10 — P0 (8) + P1 (5) + P2 (4) + P3 (2) + P4 (3) = 22 // at plan completion, plus v1 fixture wrappers/stubs (including Retrieval, -// TavilySearch, TavilyExtract, ExeSQL, Google, BGPT, YahooFinance, GoogleScholar, -// Generate, Answer, Iteration, and IterationItem) registered by fixture_stubs.go -// to keep the dsl-examples and canvas tool surface compiling. The test allows -// counts between 12 (P0+P1 minus the removed ExitLoop) and 36 (the 22 plan -// components plus the wrappers/stubs currently registered by fixture_stubs.go) -// to roll forward as subsequent batches land. +// TavilySearch, TavilyExtract, ExeSQL, CodeExec, Google, BGPT, YahooFinance, +// Wikipedia, GoogleScholar, DuckDuckGo, Generate, Answer, Iteration, and IterationItem) +// registered by fixture_stubs.go to keep the dsl-examples and canvas tool +// surface compiling. The test allows counts between 12 (P0+P1 minus the +// removed ExitLoop) and 55 to roll forward as subsequent batches land. // // Note: ExitLoop is intentionally NOT in the registry anymore. The // canvas engine (internal/agent/canvas/canvas.go's legacyNoOpNames) @@ -45,8 +44,8 @@ func TestVerifyRegistration_P1(t *testing.T) { if len(missing) > 0 { t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names) } - if got := len(names); got < 12 || got > 36 { - t.Errorf("expected 12-36 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names) + if got := len(names); got < 12 || got > 55 { + t.Errorf("expected 12-55 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names) } // ExitLoop must NOT be in the registry (legacy compat lives at diff --git a/internal/agent/component/wikipedia_component_test.go b/internal/agent/component/wikipedia_component_test.go new file mode 100644 index 0000000000..eb6f0df9a8 --- /dev/null +++ b/internal/agent/component/wikipedia_component_test.go @@ -0,0 +1,132 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package component + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + agenttool "ragflow/internal/agent/tool" +) + +func TestWikipedia_RegisteredFactory(t *testing.T) { + c, err := New("Wikipedia", map[string]any{"top_n": float64(3), "language": "en"}) + if err != nil { + t.Fatalf("New(Wikipedia) errored: %v", err) + } + if got := c.Name(); got != "Wikipedia" { + t.Fatalf("Name() = %q, want Wikipedia", got) + } + formGetter, ok := c.(interface{ GetInputForm() map[string]any }) + if !ok { + t.Fatal("Wikipedia component does not expose GetInputForm") + } + query, ok := formGetter.GetInputForm()["query"].(map[string]any) + if !ok { + t.Fatalf("GetInputForm()[query] has type %T, want map", formGetter.GetInputForm()["query"]) + } + if query["type"] != "line" { + t.Fatalf("GetInputForm()[query][type] = %v, want line", query["type"]) + } + if _, ok := c.Outputs()["formalized_content"]; !ok { + t.Fatal("Outputs() missing formalized_content") + } + if _, ok := c.Outputs()["json"]; !ok { + t.Fatal("Outputs() missing json") + } +} + +func TestWikipedia_InvokeEmptyQueryMatchesPython(t *testing.T) { + c, err := New("Wikipedia", nil) + if err != nil { + t.Fatalf("New(Wikipedia) errored: %v", err) + } + out, err := c.Invoke(context.Background(), map[string]any{"query": " "}) + if err != nil { + t.Fatalf("Invoke errored: %v", err) + } + if got, _ := out["formalized_content"].(string); got != "" { + t.Fatalf("formalized_content = %q, want empty", got) + } + if _, ok := out["json"].([]any); !ok { + t.Fatalf("json output has type %T, want []any", out["json"]) + } +} + +func TestWikipedia_InvokeParamsBakedAtConstruct(t *testing.T) { + t.Parallel() + + var gotLimit string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotLimit = r.URL.Query().Get("gsrlimit") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query": { + "pages": { + "10": {"index":1,"title":"RAG","extract":"RAG is an acronym."} + } + } + }`)) + })) + defer srv.Close() + + helper := agenttool.NewHTTPHelper().WithClient(&http.Client{ + Transport: componentRewriteHostTransport(srv.URL), + }) + inner := agenttool.NewWikipediaToolWithParams(helper, 2, "en") + c := &wikipediaComponent{inner: inner} + out, err := c.Invoke(context.Background(), map[string]any{"query": "rag"}) + if err != nil { + t.Fatalf("Invoke errored: %v", err) + } + if gotLimit != "2" { + t.Fatalf("gsrlimit = %q, want 2", gotLimit) + } + if got, _ := out["formalized_content"].(string); !strings.Contains(got, "RAG is an acronym.") { + t.Fatalf("formalized_content = %q, want rendered result", got) + } + results := anySlice(out["json"]) + if len(results) != 1 { + t.Fatalf("json len = %d, want 1", len(results)) + } +} + +func componentRewriteHostTransport(srvURL string) http.RoundTripper { + u, err := url.Parse(srvURL) + if err != nil { + panic("componentRewriteHostTransport: bad srvURL: " + err.Error()) + } + return &componentHostSwapRT{inner: http.DefaultTransport, host: u.Host, scheme: u.Scheme} +} + +type componentHostSwapRT struct { + inner http.RoundTripper + host string + scheme string +} + +func (t *componentHostSwapRT) RoundTrip(req *http.Request) (*http.Response, error) { + r2 := req.Clone(req.Context()) + r2.URL.Scheme = t.scheme + r2.URL.Host = t.host + r2.Host = t.host + return t.inner.RoundTrip(r2) +} diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.go index ac78ad88cc..4386cc4438 100644 --- a/internal/agent/tool/registry.go +++ b/internal/agent/tool/registry.go @@ -56,7 +56,8 @@ var registry = map[string]Factory{ "tushare": noConfig("tushare", func() einotool.BaseTool { return NewTushareTool() }), "wencai": noConfig("wencai", func() einotool.BaseTool { return NewWencaiTool() }), "web_crawler": noConfig("web_crawler", func() einotool.BaseTool { return NewCrawlerTool() }), - "wikipedia": noConfig("wikipedia", func() einotool.BaseTool { return NewWikipediaTool() }), + "wikipedia": buildWikipediaTool, + "wikipedia_search": buildWikipediaTool, "yahoo_finance": noConfig("yahoo_finance", func() einotool.BaseTool { return NewYahooFinanceTool() }), } @@ -217,6 +218,32 @@ func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) { return NewKeenableToolWithAPIKey(nil, apiKey), nil } +func buildWikipediaTool(params map[string]any) (einotool.BaseTool, error) { + topN := defaultWikipediaTopN + language := defaultWikipediaLanguage + for key := range params { + if key != "top_n" && key != "language" { + return nil, fmt.Errorf("agent tool: tool %q only accepts node-level params top_n/language", "wikipedia") + } + } + if v, ok := intParam(params, "top_n"); ok { + topN = v + } + if topN <= 0 { + return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "wikipedia") + } + if v, ok := stringParam(params, "language"); ok { + language = strings.TrimSpace(v) + } + if language == "" { + return nil, fmt.Errorf("agent tool: tool %q requires non-empty string node-level param language", "wikipedia") + } + if !WikipediaLanguageSupported(language) { + return nil, fmt.Errorf("agent tool: tool %q unsupported node-level param language %q", "wikipedia", language) + } + return NewWikipediaToolWithParams(nil, topN, language), nil +} + func decodeExeSQLConnParams(params map[string]any) (exesqlConnParams, error) { if len(params) == 0 { return exesqlConnParams{}, fmt.Errorf( diff --git a/internal/agent/tool/registry_test.go b/internal/agent/tool/registry_test.go index 51c7ae8237..dba1f761ca 100644 --- a/internal/agent/tool/registry_test.go +++ b/internal/agent/tool/registry_test.go @@ -25,8 +25,8 @@ func TestBuildAll_KnownTools(t *testing.T) { if err != nil { t.Fatalf("tools[1].Info: %v", err) } - if info1.Name != "wikipedia" { - t.Errorf("tools[1].Info().Name = %q, want wikipedia", info1.Name) + if info1.Name != "wikipedia_search" { + t.Errorf("tools[1].Info().Name = %q, want wikipedia_search", info1.Name) } } @@ -47,7 +47,7 @@ func TestBuildAll_AllRegisteredTools(t *testing.T) { "duckduckgo", "email", "exesql", "execute_sql", "github", "google", "google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather", "retrieval", "search_my_dataset", "search_my_dateset", "searxng", - "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", + "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search", "yahoo_finance", } params := map[string]map[string]any{ @@ -122,7 +122,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) { "duckduckgo", "email", "execute_sql", "exesql", "github", "google", "google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather", "retrieval", "search_my_dataset", "search_my_dateset", "searxng", - "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", + "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search", "yahoo_finance", } params := map[string]map[string]any{ @@ -188,6 +188,8 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) { "search_my_dateset": "search_my_dateset", "crawler": "web_crawler", "web_crawler": "web_crawler", + "wikipedia": "wikipedia_search", + "wikipedia_search": "wikipedia_search", } for _, name := range names { canonical, ok := canonicalByAlias[name] diff --git a/internal/agent/tool/wikipedia.go b/internal/agent/tool/wikipedia.go index 4eeda90773..05705a44ba 100644 --- a/internal/agent/tool/wikipedia.go +++ b/internal/agent/tool/wikipedia.go @@ -22,43 +22,77 @@ import ( "fmt" "net/http" "net/url" + "sort" + "strings" "github.com/cloudwego/eino/components/tool" "github.com/cloudwego/eino/schema" ) -const wikipediaToolName = "wikipedia" +const wikipediaToolName = "wikipedia_search" -const wikipediaToolDescription = "Search Wikipedia and return matching articles as {title, snippet, url}." +const wikipediaToolDescription = "A wide range of how-to and information pages are made available in wikipedia. Since 2001, it has grown rapidly to become the world's largest reference website. From Wikipedia, the free encyclopedia." -// wikipediaParams is the JSON shape the model sends into InvokableRun. -// lang is the language subdomain (e.g. "en", "zh", "de"); max_results -// defaults to 5 when unset or non-positive. -type wikipediaParams struct { - Query string `json:"query"` - Lang string `json:"lang"` - MaxResults int `json:"max_results"` +const wikipediaUserAgent = "Mozilla/5.0 (compatible; ragflow/1.0; +https://github.com/infiniflow/ragflow)" + +const ( + defaultWikipediaTopN = 10 + defaultWikipediaLanguage = "en" +) + +var wikipediaLanguages = map[string]struct{}{ + "af": {}, "pl": {}, "ar": {}, "ast": {}, "az": {}, "bg": {}, "nan": {}, "bn": {}, "be": {}, "ca": {}, + "cs": {}, "cy": {}, "da": {}, "de": {}, "et": {}, "el": {}, "en": {}, "es": {}, "eo": {}, "eu": {}, + "fa": {}, "fr": {}, "gl": {}, "ko": {}, "hy": {}, "hi": {}, "hr": {}, "id": {}, "it": {}, "he": {}, + "ka": {}, "lld": {}, "la": {}, "lv": {}, "lt": {}, "hu": {}, "mk": {}, "arz": {}, "ms": {}, "min": {}, + "my": {}, "nl": {}, "ja": {}, "nb": {}, "nn": {}, "ce": {}, "uz": {}, "pt": {}, "kk": {}, "ro": {}, + "ru": {}, "ceb": {}, "sk": {}, "sl": {}, "sr": {}, "sh": {}, "fi": {}, "sv": {}, "ta": {}, "tt": {}, + "th": {}, "tg": {}, "azb": {}, "tr": {}, "uk": {}, "ur": {}, "vi": {}, "war": {}, "zh": {}, "yue": {}, } -// wikipediaResult is one row in the upstream `query.search` array. +// WikipediaLanguageSupported mirrors Python WikipediaParam.check's accepted +// language list. +func WikipediaLanguageSupported(language string) bool { + _, ok := wikipediaLanguages[strings.TrimSpace(language)] + return ok +} + +// wikipediaParams is the JSON shape the model sends into InvokableRun. +// LLM only sees query via Info(); lang and max_results are accepted for +// non-agent callers and fall back to the tool instance's node-level +// defaults (WikipediaTool.lang / WikipediaTool.topN) when absent. +type wikipediaParams struct { + Query string `json:"query"` + Lang string `json:"lang,omitempty"` + MaxResults int `json:"max_results,omitempty"` +} + +// wikipediaResult mirrors the fields Python passes to _retrieve_chunks: +// title, url, and summary content. type wikipediaResult struct { Title string `json:"title"` - Snippet string `json:"snippet"` + Content string `json:"content"` URL string `json:"url"` } -// wikipediaResponse is the upstream MediaWiki API envelope. +// wikipediaResponse is the upstream MediaWiki generator=search envelope. type wikipediaResponse struct { Query struct { - Search []wikipediaResult `json:"search"` + Pages map[string]struct { + Index int `json:"index"` + Title string `json:"title"` + Extract string `json:"extract"` + FullURL string `json:"fullurl"` + } `json:"pages"` } `json:"query"` } -// wikipediaEnvelope is what the model sees. It mirrors the Python tool's -// output: a flat list of {title, snippet, url} entries. +// wikipediaEnvelope is what the model sees. The formalized_content field +// is the canvas-facing equivalent of Python ToolBase._retrieve_chunks(). type wikipediaEnvelope struct { - Results []wikipediaResult `json:"results"` - Error string `json:"_ERROR,omitempty"` + FormalizedContent string `json:"formalized_content,omitempty"` + Results []wikipediaResult `json:"results,omitempty"` + Error string `json:"_ERROR,omitempty"` } // WikipediaTool is the Wikipedia @@ -67,6 +101,8 @@ type wikipediaEnvelope struct { // top N matches for the query. type WikipediaTool struct { helper *HTTPHelper + topN int + lang string } // NewWikipediaTool returns a WikipediaTool using the default HTTPHelper. @@ -77,10 +113,22 @@ func NewWikipediaTool() *WikipediaTool { // NewWikipediaToolWith returns a WikipediaTool that uses the provided // HTTPHelper. Useful for tests that want to inject a custom transport. func NewWikipediaToolWith(h *HTTPHelper) *WikipediaTool { + return NewWikipediaToolWithParams(h, defaultWikipediaTopN, defaultWikipediaLanguage) +} + +// NewWikipediaToolWithParams returns a WikipediaTool with node-level +// parameters matching Python's WikipediaParam.top_n and language fields. +func NewWikipediaToolWithParams(h *HTTPHelper, topN int, language string) *WikipediaTool { if h == nil { h = NewHTTPHelper() } - return &WikipediaTool{helper: h} + if topN <= 0 { + topN = defaultWikipediaTopN + } + if strings.TrimSpace(language) == "" { + language = defaultWikipediaLanguage + } + return &WikipediaTool{helper: h, topN: topN, lang: strings.TrimSpace(language)} } // Info returns the tool's metadata for the chat model. @@ -91,36 +139,27 @@ func (w *WikipediaTool) Info(_ context.Context) (*schema.ToolInfo, error) { ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{ "query": { Type: schema.String, - Desc: "Search query", + Desc: "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.", Required: true, }, - "lang": { - Type: schema.String, - Desc: `Language subdomain. Defaults to "en".`, - Required: false, - }, - "max_results": { - Type: schema.Integer, - Desc: "Maximum number of results to return. Defaults to 5.", - Required: false, - }, }), }, nil } -// buildWikipediaURL constructs the MediaWiki action=query URL. Centralized -// so the test suite can verify URL encoding without spinning up a server. -func buildWikipediaURL(lang, query string, maxResults int) string { +// buildWikipediaURL constructs a MediaWiki generator=search URL that returns +// the same page fields Python reads from wikipedia.page(): title, url, +// and summary-like introductory extract. +func buildWikipediaURL(lang, query string, topN int) string { if lang == "" { - lang = "en" + lang = defaultWikipediaLanguage } - if maxResults <= 0 { - maxResults = 5 + if topN <= 0 { + topN = defaultWikipediaTopN } return fmt.Sprintf( - "https://%s.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=%d&srsearch=%s", + "https://%s.wikipedia.org/w/api.php?action=query&generator=search&format=json&gsrlimit=%d&gsrsearch=%s&prop=extracts%%7Cinfo&exintro=1&explaintext=1&inprop=url", lang, - maxResults, + topN, url.QueryEscape(query), ) } @@ -137,8 +176,22 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ... fmt.Errorf("wikipedia: query is required") } - endpoint := buildWikipediaURL(p.Lang, p.Query, p.MaxResults) - resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil) + lang := p.Lang + if lang == "" { + lang = w.lang + } + maxResults := p.MaxResults + if maxResults <= 0 { + maxResults = w.topN + } + if !WikipediaLanguageSupported(lang) { + return wikipediaErrJSON(fmt.Errorf("wikipedia: unsupported language %q", lang)), + fmt.Errorf("wikipedia: unsupported language %q", lang) + } + endpoint := buildWikipediaURL(lang, p.Query, maxResults) + resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", map[string]string{ + "User-Agent": wikipediaUserAgent, + }) if err != nil { return wikipediaErrJSON(err), err } @@ -155,22 +208,61 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ... fmt.Errorf("wikipedia: decode response: %w", err) } - // Compose canonical Wikipedia URLs for the snippets. The MediaWiki - // search endpoint doesn't include a URL; the conventional one is - // https://.wikipedia.org/wiki/. - lang := p.Lang - if lang == "" { - lang = "en" - } - results := make([]wikipediaResult, 0, len(raw.Query.Search)) - for _, r := range raw.Query.Search { - results = append(results, wikipediaResult{ - Title: r.Title, - Snippet: r.Snippet, - URL: fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(r.Title)), + pages := make([]struct { + Index int + Title string + Extract string + FullURL string + }, 0, len(raw.Query.Pages)) + for _, p := range raw.Query.Pages { + pages = append(pages, struct { + Index int + Title string + Extract string + FullURL string + }{ + Index: p.Index, + Title: p.Title, + Extract: p.Extract, + FullURL: p.FullURL, }) } - return wikipediaJSON(wikipediaEnvelope{Results: results}), nil + sort.SliceStable(pages, func(i, j int) bool { return pages[i].Index < pages[j].Index }) + + results := make([]wikipediaResult, 0, len(pages)) + for _, p := range pages { + content := strings.TrimSpace(p.Extract) + if content == "" { + continue + } + if len(content) > 10000 { + content = content[:10000] + } + fullURL := p.FullURL + if fullURL == "" { + fullURL = fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(p.Title)) + } + results = append(results, wikipediaResult{ + Title: p.Title, + Content: content, + URL: fullURL, + }) + } + return wikipediaJSON(wikipediaEnvelope{ + FormalizedContent: renderWikipediaResults(results), + Results: results, + }), nil +} + +func renderWikipediaResults(results []wikipediaResult) string { + if len(results) == 0 { + return "" + } + blocks := make([]string, 0, len(results)) + for _, r := range results { + blocks = append(blocks, fmt.Sprintf("Title: %s\nURL: %s\nContent: %s", r.Title, r.URL, r.Content)) + } + return strings.Join(blocks, "\n\n") } func wikipediaJSON(env wikipediaEnvelope) string { diff --git a/internal/agent/tool/wikipedia_test.go b/internal/agent/tool/wikipedia_test.go index 8d32159b1e..862fb571e5 100644 --- a/internal/agent/tool/wikipedia_test.go +++ b/internal/agent/tool/wikipedia_test.go @@ -80,14 +80,14 @@ func TestWikipedia_BuildURL(t *testing.T) { if q.Get("action") != "query" { t.Errorf("action = %q, want query", q.Get("action")) } - if q.Get("list") != "search" { - t.Errorf("list = %q, want search", q.Get("list")) + if q.Get("generator") != "search" { + t.Errorf("generator = %q, want search", q.Get("generator")) } if q.Get("format") != "json" { t.Errorf("format = %q, want json", q.Get("format")) } - if q.Get("srsearch") != tc.query { - t.Errorf("srsearch = %q, want %q (raw query, not pre-encoded)", q.Get("srsearch"), tc.query) + if q.Get("gsrsearch") != tc.query { + t.Errorf("gsrsearch = %q, want %q (raw query, not pre-encoded)", q.Get("gsrsearch"), tc.query) } }) } @@ -96,14 +96,16 @@ func TestWikipedia_BuildURL(t *testing.T) { func TestWikipedia_ParseResults(t *testing.T) { t.Parallel() + var gotUA string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUA = r.Header.Get("User-Agent") w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ "query": { - "search": [ - {"title":"RAG","snippet":"<span>rag</span> is ..."}, - {"title":"Retrieval-augmented generation","snippet":"<b>RAG</b> is ..."} - ] + "pages": { + "11": {"index":2,"title":"Retrieval-augmented generation","extract":"RAG is a generation technique.","fullurl":"https://en.wikipedia.org/wiki/Retrieval-augmented_generation"}, + "10": {"index":1,"title":"RAG","extract":"RAG is an acronym.","fullurl":"https://en.wikipedia.org/wiki/RAG"} + } } }`)) })) @@ -119,6 +121,9 @@ func TestWikipedia_ParseResults(t *testing.T) { if err != nil { t.Fatalf("InvokableRun: %v", err) } + if !strings.Contains(gotUA, "ragflow") { + t.Errorf("User-Agent = %q, want to contain ragflow", gotUA) + } var env wikipediaEnvelope if jerr := json.Unmarshal([]byte(out), &env); jerr != nil { @@ -133,6 +138,12 @@ func TestWikipedia_ParseResults(t *testing.T) { if env.Results[0].Title != "RAG" { t.Errorf("Results[0].Title = %q, want RAG", env.Results[0].Title) } + if env.Results[0].Content != "RAG is an acronym." { + t.Errorf("Results[0].Content = %q, want summary extract", env.Results[0].Content) + } + if !strings.Contains(env.FormalizedContent, "RAG is an acronym.") { + t.Errorf("FormalizedContent = %q, want rendered summary", env.FormalizedContent) + } if !strings.HasPrefix(env.Results[0].URL, "https://en.wikipedia.org/wiki/") { t.Errorf("Results[0].URL = %q, want to start with https://en.wikipedia.org/wiki/", env.Results[0].URL) } @@ -172,8 +183,8 @@ func TestWikipedia_Info(t *testing.T) { if err != nil { t.Fatalf("Info: %v", err) } - if info.Name != "wikipedia" { - t.Errorf("Name = %q, want wikipedia", info.Name) + if info.Name != "wikipedia_search" { + t.Errorf("Name = %q, want wikipedia_search", info.Name) } if !strings.Contains(info.Desc, "Wikipedia") { t.Errorf("Desc = %q, want to mention Wikipedia", info.Desc)