mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 14:15:40 +08:00
fix(go-agent): unknown component "Wikipedia" canvas error. (#16784)
## Summary - Register Wikipedia component + tool alias `wikipedia`/`wikipedia_search` - Use `generator=search` to get title/summary/url in one request (was N+1) - Node params `top_n`/`language` with validation - Return `formalized_content` for downstream - tests pass <img width="1817" height="972" alt="image" src="https://github.com/user-attachments/assets/f6d79599-6d1f-4ea6-84f7-ac06d0de13b0" />
This commit is contained in:
@@ -518,6 +518,7 @@ func init() {
|
||||
Register(componentNameIteration, NewIterationStub)
|
||||
Register(componentNameIterationItem, NewIterationItemStub)
|
||||
Register("BGPT", newBGPTComponent)
|
||||
Register("Wikipedia", newWikipediaComponent)
|
||||
Register("DuckDuckGo", newDuckDuckGoComponent)
|
||||
Register("Google", newGoogleComponent)
|
||||
Register("GoogleScholar", newGoogleScholarComponent)
|
||||
|
||||
@@ -410,6 +410,85 @@ func (c *bgptComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// wikipediaComponent delegates to internal/agent/tool/WikipediaTool.
|
||||
// Python's canvas component is named "Wikipedia" and stores top_n/language
|
||||
// on the node params while accepting query at runtime.
|
||||
type wikipediaComponent struct {
|
||||
inner *agenttool.WikipediaTool
|
||||
}
|
||||
|
||||
func newWikipediaComponent(params map[string]any) (Component, error) {
|
||||
topN := 10
|
||||
if v, ok := params["top_n"]; ok {
|
||||
topN = toIntParam(v)
|
||||
}
|
||||
if topN <= 0 {
|
||||
return nil, fmt.Errorf("canvas: Wikipedia: top_n must be a positive integer")
|
||||
}
|
||||
language := "en"
|
||||
if v, ok := params["language"].(string); ok && strings.TrimSpace(v) != "" {
|
||||
language = strings.TrimSpace(v)
|
||||
}
|
||||
if !agenttool.WikipediaLanguageSupported(language) {
|
||||
return nil, fmt.Errorf("canvas: Wikipedia: unsupported language %q", language)
|
||||
}
|
||||
return &wikipediaComponent{inner: agenttool.NewWikipediaToolWithParams(nil, topN, language)}, nil
|
||||
}
|
||||
|
||||
func (c *wikipediaComponent) Name() string { return "Wikipedia" }
|
||||
|
||||
func (c *wikipediaComponent) Inputs() map[string]string {
|
||||
return map[string]string{
|
||||
"query": "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.",
|
||||
}
|
||||
}
|
||||
|
||||
func (c *wikipediaComponent) GetInputForm() map[string]any {
|
||||
return map[string]any{
|
||||
"query": map[string]any{
|
||||
"name": "Query",
|
||||
"type": "line",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *wikipediaComponent) Outputs() map[string]string {
|
||||
return map[string]string{
|
||||
"formalized_content": "Rendered Wikipedia article summaries for downstream LLM prompts.",
|
||||
"json": "Raw Wikipedia result list.",
|
||||
}
|
||||
}
|
||||
|
||||
func (c *wikipediaComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
|
||||
query := strings.TrimSpace(stringParam(inputs["query"]))
|
||||
if query == "" {
|
||||
return map[string]any{"formalized_content": "", "json": []any{}}, nil
|
||||
}
|
||||
argsJSON, _ := json.Marshal(map[string]any{"query": query})
|
||||
out, err := c.inner.InvokableRun(ctx, string(argsJSON))
|
||||
decoded := parseToolEnvelope(out)
|
||||
if results, ok := decoded["results"]; ok {
|
||||
decoded["json"] = results
|
||||
}
|
||||
if err != nil {
|
||||
if len(decoded) > 0 {
|
||||
if _, ok := decoded["formalized_content"]; !ok {
|
||||
decoded["formalized_content"] = ""
|
||||
}
|
||||
if _, ok := decoded["json"]; !ok {
|
||||
decoded["json"] = []any{}
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
return nil, fmt.Errorf("canvas: Wikipedia: %w", err)
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
func (c *wikipediaComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// duckDuckGoComponent delegates to internal/agent/tool/DuckDuckGoTool.
|
||||
type duckDuckGoComponent struct {
|
||||
inner duckDuckGoInvoker
|
||||
@@ -1580,6 +1659,7 @@ var (
|
||||
_ Component = (*duckDuckGoComponent)(nil)
|
||||
_ Component = (*exesqlComponent)(nil)
|
||||
_ Component = (*codeExecComponent)(nil)
|
||||
_ Component = (*wikipediaComponent)(nil)
|
||||
_ Component = (*googleScholarComponent)(nil)
|
||||
_ Component = (*yahooFinanceComponent)(nil)
|
||||
)
|
||||
@@ -1587,6 +1667,7 @@ var (
|
||||
// Compile-time check that the eino InvokableTool methods we call
|
||||
// are reachable (catches a future refactor that renames them).
|
||||
var _ einotool.InvokableTool = (*agenttool.TavilyTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.WikipediaTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil)
|
||||
|
||||
@@ -10,12 +10,11 @@ import (
|
||||
// case-insensitive, and returned in sorted order. The expected count is
|
||||
// read from plan §2.11.10 — P0 (8) + P1 (5) + P2 (4) + P3 (2) + P4 (3) = 22
|
||||
// at plan completion, plus v1 fixture wrappers/stubs (including Retrieval,
|
||||
// TavilySearch, TavilyExtract, ExeSQL, Google, BGPT, YahooFinance, GoogleScholar,
|
||||
// Generate, Answer, Iteration, and IterationItem) registered by fixture_stubs.go
|
||||
// to keep the dsl-examples and canvas tool surface compiling. The test allows
|
||||
// counts between 12 (P0+P1 minus the removed ExitLoop) and 36 (the 22 plan
|
||||
// components plus the wrappers/stubs currently registered by fixture_stubs.go)
|
||||
// to roll forward as subsequent batches land.
|
||||
// TavilySearch, TavilyExtract, ExeSQL, CodeExec, Google, BGPT, YahooFinance,
|
||||
// Wikipedia, GoogleScholar, DuckDuckGo, Generate, Answer, Iteration, and IterationItem)
|
||||
// registered by fixture_stubs.go to keep the dsl-examples and canvas tool
|
||||
// surface compiling. The test allows counts between 12 (P0+P1 minus the
|
||||
// removed ExitLoop) and 55 to roll forward as subsequent batches land.
|
||||
//
|
||||
// Note: ExitLoop is intentionally NOT in the registry anymore. The
|
||||
// canvas engine (internal/agent/canvas/canvas.go's legacyNoOpNames)
|
||||
@@ -45,8 +44,8 @@ func TestVerifyRegistration_P1(t *testing.T) {
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names)
|
||||
}
|
||||
if got := len(names); got < 12 || got > 36 {
|
||||
t.Errorf("expected 12-36 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
|
||||
if got := len(names); got < 12 || got > 55 {
|
||||
t.Errorf("expected 12-55 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
|
||||
}
|
||||
|
||||
// ExitLoop must NOT be in the registry (legacy compat lives at
|
||||
|
||||
132
internal/agent/component/wikipedia_component_test.go
Normal file
132
internal/agent/component/wikipedia_component_test.go
Normal file
@@ -0,0 +1,132 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package component
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
agenttool "ragflow/internal/agent/tool"
|
||||
)
|
||||
|
||||
func TestWikipedia_RegisteredFactory(t *testing.T) {
|
||||
c, err := New("Wikipedia", map[string]any{"top_n": float64(3), "language": "en"})
|
||||
if err != nil {
|
||||
t.Fatalf("New(Wikipedia) errored: %v", err)
|
||||
}
|
||||
if got := c.Name(); got != "Wikipedia" {
|
||||
t.Fatalf("Name() = %q, want Wikipedia", got)
|
||||
}
|
||||
formGetter, ok := c.(interface{ GetInputForm() map[string]any })
|
||||
if !ok {
|
||||
t.Fatal("Wikipedia component does not expose GetInputForm")
|
||||
}
|
||||
query, ok := formGetter.GetInputForm()["query"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("GetInputForm()[query] has type %T, want map", formGetter.GetInputForm()["query"])
|
||||
}
|
||||
if query["type"] != "line" {
|
||||
t.Fatalf("GetInputForm()[query][type] = %v, want line", query["type"])
|
||||
}
|
||||
if _, ok := c.Outputs()["formalized_content"]; !ok {
|
||||
t.Fatal("Outputs() missing formalized_content")
|
||||
}
|
||||
if _, ok := c.Outputs()["json"]; !ok {
|
||||
t.Fatal("Outputs() missing json")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikipedia_InvokeEmptyQueryMatchesPython(t *testing.T) {
|
||||
c, err := New("Wikipedia", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("New(Wikipedia) errored: %v", err)
|
||||
}
|
||||
out, err := c.Invoke(context.Background(), map[string]any{"query": " "})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke errored: %v", err)
|
||||
}
|
||||
if got, _ := out["formalized_content"].(string); got != "" {
|
||||
t.Fatalf("formalized_content = %q, want empty", got)
|
||||
}
|
||||
if _, ok := out["json"].([]any); !ok {
|
||||
t.Fatalf("json output has type %T, want []any", out["json"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikipedia_InvokeParamsBakedAtConstruct(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var gotLimit string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotLimit = r.URL.Query().Get("gsrlimit")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"query": {
|
||||
"pages": {
|
||||
"10": {"index":1,"title":"RAG","extract":"RAG is an acronym."}
|
||||
}
|
||||
}
|
||||
}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
helper := agenttool.NewHTTPHelper().WithClient(&http.Client{
|
||||
Transport: componentRewriteHostTransport(srv.URL),
|
||||
})
|
||||
inner := agenttool.NewWikipediaToolWithParams(helper, 2, "en")
|
||||
c := &wikipediaComponent{inner: inner}
|
||||
out, err := c.Invoke(context.Background(), map[string]any{"query": "rag"})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke errored: %v", err)
|
||||
}
|
||||
if gotLimit != "2" {
|
||||
t.Fatalf("gsrlimit = %q, want 2", gotLimit)
|
||||
}
|
||||
if got, _ := out["formalized_content"].(string); !strings.Contains(got, "RAG is an acronym.") {
|
||||
t.Fatalf("formalized_content = %q, want rendered result", got)
|
||||
}
|
||||
results := anySlice(out["json"])
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("json len = %d, want 1", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func componentRewriteHostTransport(srvURL string) http.RoundTripper {
|
||||
u, err := url.Parse(srvURL)
|
||||
if err != nil {
|
||||
panic("componentRewriteHostTransport: bad srvURL: " + err.Error())
|
||||
}
|
||||
return &componentHostSwapRT{inner: http.DefaultTransport, host: u.Host, scheme: u.Scheme}
|
||||
}
|
||||
|
||||
type componentHostSwapRT struct {
|
||||
inner http.RoundTripper
|
||||
host string
|
||||
scheme string
|
||||
}
|
||||
|
||||
func (t *componentHostSwapRT) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
r2 := req.Clone(req.Context())
|
||||
r2.URL.Scheme = t.scheme
|
||||
r2.URL.Host = t.host
|
||||
r2.Host = t.host
|
||||
return t.inner.RoundTrip(r2)
|
||||
}
|
||||
@@ -56,7 +56,8 @@ var registry = map[string]Factory{
|
||||
"tushare": noConfig("tushare", func() einotool.BaseTool { return NewTushareTool() }),
|
||||
"wencai": noConfig("wencai", func() einotool.BaseTool { return NewWencaiTool() }),
|
||||
"web_crawler": noConfig("web_crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
|
||||
"wikipedia": noConfig("wikipedia", func() einotool.BaseTool { return NewWikipediaTool() }),
|
||||
"wikipedia": buildWikipediaTool,
|
||||
"wikipedia_search": buildWikipediaTool,
|
||||
"yahoo_finance": noConfig("yahoo_finance", func() einotool.BaseTool { return NewYahooFinanceTool() }),
|
||||
}
|
||||
|
||||
@@ -217,6 +218,32 @@ func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
return NewKeenableToolWithAPIKey(nil, apiKey), nil
|
||||
}
|
||||
|
||||
func buildWikipediaTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
topN := defaultWikipediaTopN
|
||||
language := defaultWikipediaLanguage
|
||||
for key := range params {
|
||||
if key != "top_n" && key != "language" {
|
||||
return nil, fmt.Errorf("agent tool: tool %q only accepts node-level params top_n/language", "wikipedia")
|
||||
}
|
||||
}
|
||||
if v, ok := intParam(params, "top_n"); ok {
|
||||
topN = v
|
||||
}
|
||||
if topN <= 0 {
|
||||
return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "wikipedia")
|
||||
}
|
||||
if v, ok := stringParam(params, "language"); ok {
|
||||
language = strings.TrimSpace(v)
|
||||
}
|
||||
if language == "" {
|
||||
return nil, fmt.Errorf("agent tool: tool %q requires non-empty string node-level param language", "wikipedia")
|
||||
}
|
||||
if !WikipediaLanguageSupported(language) {
|
||||
return nil, fmt.Errorf("agent tool: tool %q unsupported node-level param language %q", "wikipedia", language)
|
||||
}
|
||||
return NewWikipediaToolWithParams(nil, topN, language), nil
|
||||
}
|
||||
|
||||
func decodeExeSQLConnParams(params map[string]any) (exesqlConnParams, error) {
|
||||
if len(params) == 0 {
|
||||
return exesqlConnParams{}, fmt.Errorf(
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestBuildAll_KnownTools(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("tools[1].Info: %v", err)
|
||||
}
|
||||
if info1.Name != "wikipedia" {
|
||||
t.Errorf("tools[1].Info().Name = %q, want wikipedia", info1.Name)
|
||||
if info1.Name != "wikipedia_search" {
|
||||
t.Errorf("tools[1].Info().Name = %q, want wikipedia_search", info1.Name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestBuildAll_AllRegisteredTools(t *testing.T) {
|
||||
"duckduckgo", "email", "exesql", "execute_sql", "github", "google",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
|
||||
"yahoo_finance",
|
||||
}
|
||||
params := map[string]map[string]any{
|
||||
@@ -122,7 +122,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
"duckduckgo", "email", "execute_sql", "exesql", "github", "google",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
|
||||
"yahoo_finance",
|
||||
}
|
||||
params := map[string]map[string]any{
|
||||
@@ -188,6 +188,8 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
"search_my_dateset": "search_my_dateset",
|
||||
"crawler": "web_crawler",
|
||||
"web_crawler": "web_crawler",
|
||||
"wikipedia": "wikipedia_search",
|
||||
"wikipedia_search": "wikipedia_search",
|
||||
}
|
||||
for _, name := range names {
|
||||
canonical, ok := canonicalByAlias[name]
|
||||
|
||||
@@ -22,43 +22,77 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/eino/components/tool"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
const wikipediaToolName = "wikipedia"
|
||||
const wikipediaToolName = "wikipedia_search"
|
||||
|
||||
const wikipediaToolDescription = "Search Wikipedia and return matching articles as {title, snippet, url}."
|
||||
const wikipediaToolDescription = "A wide range of how-to and information pages are made available in wikipedia. Since 2001, it has grown rapidly to become the world's largest reference website. From Wikipedia, the free encyclopedia."
|
||||
|
||||
// wikipediaParams is the JSON shape the model sends into InvokableRun.
|
||||
// lang is the language subdomain (e.g. "en", "zh", "de"); max_results
|
||||
// defaults to 5 when unset or non-positive.
|
||||
type wikipediaParams struct {
|
||||
Query string `json:"query"`
|
||||
Lang string `json:"lang"`
|
||||
MaxResults int `json:"max_results"`
|
||||
const wikipediaUserAgent = "Mozilla/5.0 (compatible; ragflow/1.0; +https://github.com/infiniflow/ragflow)"
|
||||
|
||||
const (
|
||||
defaultWikipediaTopN = 10
|
||||
defaultWikipediaLanguage = "en"
|
||||
)
|
||||
|
||||
var wikipediaLanguages = map[string]struct{}{
|
||||
"af": {}, "pl": {}, "ar": {}, "ast": {}, "az": {}, "bg": {}, "nan": {}, "bn": {}, "be": {}, "ca": {},
|
||||
"cs": {}, "cy": {}, "da": {}, "de": {}, "et": {}, "el": {}, "en": {}, "es": {}, "eo": {}, "eu": {},
|
||||
"fa": {}, "fr": {}, "gl": {}, "ko": {}, "hy": {}, "hi": {}, "hr": {}, "id": {}, "it": {}, "he": {},
|
||||
"ka": {}, "lld": {}, "la": {}, "lv": {}, "lt": {}, "hu": {}, "mk": {}, "arz": {}, "ms": {}, "min": {},
|
||||
"my": {}, "nl": {}, "ja": {}, "nb": {}, "nn": {}, "ce": {}, "uz": {}, "pt": {}, "kk": {}, "ro": {},
|
||||
"ru": {}, "ceb": {}, "sk": {}, "sl": {}, "sr": {}, "sh": {}, "fi": {}, "sv": {}, "ta": {}, "tt": {},
|
||||
"th": {}, "tg": {}, "azb": {}, "tr": {}, "uk": {}, "ur": {}, "vi": {}, "war": {}, "zh": {}, "yue": {},
|
||||
}
|
||||
|
||||
// wikipediaResult is one row in the upstream `query.search` array.
|
||||
// WikipediaLanguageSupported mirrors Python WikipediaParam.check's accepted
|
||||
// language list.
|
||||
func WikipediaLanguageSupported(language string) bool {
|
||||
_, ok := wikipediaLanguages[strings.TrimSpace(language)]
|
||||
return ok
|
||||
}
|
||||
|
||||
// wikipediaParams is the JSON shape the model sends into InvokableRun.
|
||||
// LLM only sees query via Info(); lang and max_results are accepted for
|
||||
// non-agent callers and fall back to the tool instance's node-level
|
||||
// defaults (WikipediaTool.lang / WikipediaTool.topN) when absent.
|
||||
type wikipediaParams struct {
|
||||
Query string `json:"query"`
|
||||
Lang string `json:"lang,omitempty"`
|
||||
MaxResults int `json:"max_results,omitempty"`
|
||||
}
|
||||
|
||||
// wikipediaResult mirrors the fields Python passes to _retrieve_chunks:
|
||||
// title, url, and summary content.
|
||||
type wikipediaResult struct {
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Content string `json:"content"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// wikipediaResponse is the upstream MediaWiki API envelope.
|
||||
// wikipediaResponse is the upstream MediaWiki generator=search envelope.
|
||||
type wikipediaResponse struct {
|
||||
Query struct {
|
||||
Search []wikipediaResult `json:"search"`
|
||||
Pages map[string]struct {
|
||||
Index int `json:"index"`
|
||||
Title string `json:"title"`
|
||||
Extract string `json:"extract"`
|
||||
FullURL string `json:"fullurl"`
|
||||
} `json:"pages"`
|
||||
} `json:"query"`
|
||||
}
|
||||
|
||||
// wikipediaEnvelope is what the model sees. It mirrors the Python tool's
|
||||
// output: a flat list of {title, snippet, url} entries.
|
||||
// wikipediaEnvelope is what the model sees. The formalized_content field
|
||||
// is the canvas-facing equivalent of Python ToolBase._retrieve_chunks().
|
||||
type wikipediaEnvelope struct {
|
||||
Results []wikipediaResult `json:"results"`
|
||||
Error string `json:"_ERROR,omitempty"`
|
||||
FormalizedContent string `json:"formalized_content,omitempty"`
|
||||
Results []wikipediaResult `json:"results,omitempty"`
|
||||
Error string `json:"_ERROR,omitempty"`
|
||||
}
|
||||
|
||||
// WikipediaTool is the Wikipedia
|
||||
@@ -67,6 +101,8 @@ type wikipediaEnvelope struct {
|
||||
// top N matches for the query.
|
||||
type WikipediaTool struct {
|
||||
helper *HTTPHelper
|
||||
topN int
|
||||
lang string
|
||||
}
|
||||
|
||||
// NewWikipediaTool returns a WikipediaTool using the default HTTPHelper.
|
||||
@@ -77,10 +113,22 @@ func NewWikipediaTool() *WikipediaTool {
|
||||
// NewWikipediaToolWith returns a WikipediaTool that uses the provided
|
||||
// HTTPHelper. Useful for tests that want to inject a custom transport.
|
||||
func NewWikipediaToolWith(h *HTTPHelper) *WikipediaTool {
|
||||
return NewWikipediaToolWithParams(h, defaultWikipediaTopN, defaultWikipediaLanguage)
|
||||
}
|
||||
|
||||
// NewWikipediaToolWithParams returns a WikipediaTool with node-level
|
||||
// parameters matching Python's WikipediaParam.top_n and language fields.
|
||||
func NewWikipediaToolWithParams(h *HTTPHelper, topN int, language string) *WikipediaTool {
|
||||
if h == nil {
|
||||
h = NewHTTPHelper()
|
||||
}
|
||||
return &WikipediaTool{helper: h}
|
||||
if topN <= 0 {
|
||||
topN = defaultWikipediaTopN
|
||||
}
|
||||
if strings.TrimSpace(language) == "" {
|
||||
language = defaultWikipediaLanguage
|
||||
}
|
||||
return &WikipediaTool{helper: h, topN: topN, lang: strings.TrimSpace(language)}
|
||||
}
|
||||
|
||||
// Info returns the tool's metadata for the chat model.
|
||||
@@ -91,36 +139,27 @@ func (w *WikipediaTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
|
||||
"query": {
|
||||
Type: schema.String,
|
||||
Desc: "Search query",
|
||||
Desc: "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.",
|
||||
Required: true,
|
||||
},
|
||||
"lang": {
|
||||
Type: schema.String,
|
||||
Desc: `Language subdomain. Defaults to "en".`,
|
||||
Required: false,
|
||||
},
|
||||
"max_results": {
|
||||
Type: schema.Integer,
|
||||
Desc: "Maximum number of results to return. Defaults to 5.",
|
||||
Required: false,
|
||||
},
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildWikipediaURL constructs the MediaWiki action=query URL. Centralized
|
||||
// so the test suite can verify URL encoding without spinning up a server.
|
||||
func buildWikipediaURL(lang, query string, maxResults int) string {
|
||||
// buildWikipediaURL constructs a MediaWiki generator=search URL that returns
|
||||
// the same page fields Python reads from wikipedia.page(): title, url,
|
||||
// and summary-like introductory extract.
|
||||
func buildWikipediaURL(lang, query string, topN int) string {
|
||||
if lang == "" {
|
||||
lang = "en"
|
||||
lang = defaultWikipediaLanguage
|
||||
}
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
if topN <= 0 {
|
||||
topN = defaultWikipediaTopN
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"https://%s.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=%d&srsearch=%s",
|
||||
"https://%s.wikipedia.org/w/api.php?action=query&generator=search&format=json&gsrlimit=%d&gsrsearch=%s&prop=extracts%%7Cinfo&exintro=1&explaintext=1&inprop=url",
|
||||
lang,
|
||||
maxResults,
|
||||
topN,
|
||||
url.QueryEscape(query),
|
||||
)
|
||||
}
|
||||
@@ -137,8 +176,22 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ...
|
||||
fmt.Errorf("wikipedia: query is required")
|
||||
}
|
||||
|
||||
endpoint := buildWikipediaURL(p.Lang, p.Query, p.MaxResults)
|
||||
resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil)
|
||||
lang := p.Lang
|
||||
if lang == "" {
|
||||
lang = w.lang
|
||||
}
|
||||
maxResults := p.MaxResults
|
||||
if maxResults <= 0 {
|
||||
maxResults = w.topN
|
||||
}
|
||||
if !WikipediaLanguageSupported(lang) {
|
||||
return wikipediaErrJSON(fmt.Errorf("wikipedia: unsupported language %q", lang)),
|
||||
fmt.Errorf("wikipedia: unsupported language %q", lang)
|
||||
}
|
||||
endpoint := buildWikipediaURL(lang, p.Query, maxResults)
|
||||
resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", map[string]string{
|
||||
"User-Agent": wikipediaUserAgent,
|
||||
})
|
||||
if err != nil {
|
||||
return wikipediaErrJSON(err), err
|
||||
}
|
||||
@@ -155,22 +208,61 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ...
|
||||
fmt.Errorf("wikipedia: decode response: %w", err)
|
||||
}
|
||||
|
||||
// Compose canonical Wikipedia URLs for the snippets. The MediaWiki
|
||||
// search endpoint doesn't include a URL; the conventional one is
|
||||
// https://<lang>.wikipedia.org/wiki/<Title>.
|
||||
lang := p.Lang
|
||||
if lang == "" {
|
||||
lang = "en"
|
||||
}
|
||||
results := make([]wikipediaResult, 0, len(raw.Query.Search))
|
||||
for _, r := range raw.Query.Search {
|
||||
results = append(results, wikipediaResult{
|
||||
Title: r.Title,
|
||||
Snippet: r.Snippet,
|
||||
URL: fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(r.Title)),
|
||||
pages := make([]struct {
|
||||
Index int
|
||||
Title string
|
||||
Extract string
|
||||
FullURL string
|
||||
}, 0, len(raw.Query.Pages))
|
||||
for _, p := range raw.Query.Pages {
|
||||
pages = append(pages, struct {
|
||||
Index int
|
||||
Title string
|
||||
Extract string
|
||||
FullURL string
|
||||
}{
|
||||
Index: p.Index,
|
||||
Title: p.Title,
|
||||
Extract: p.Extract,
|
||||
FullURL: p.FullURL,
|
||||
})
|
||||
}
|
||||
return wikipediaJSON(wikipediaEnvelope{Results: results}), nil
|
||||
sort.SliceStable(pages, func(i, j int) bool { return pages[i].Index < pages[j].Index })
|
||||
|
||||
results := make([]wikipediaResult, 0, len(pages))
|
||||
for _, p := range pages {
|
||||
content := strings.TrimSpace(p.Extract)
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
if len(content) > 10000 {
|
||||
content = content[:10000]
|
||||
}
|
||||
fullURL := p.FullURL
|
||||
if fullURL == "" {
|
||||
fullURL = fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(p.Title))
|
||||
}
|
||||
results = append(results, wikipediaResult{
|
||||
Title: p.Title,
|
||||
Content: content,
|
||||
URL: fullURL,
|
||||
})
|
||||
}
|
||||
return wikipediaJSON(wikipediaEnvelope{
|
||||
FormalizedContent: renderWikipediaResults(results),
|
||||
Results: results,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func renderWikipediaResults(results []wikipediaResult) string {
|
||||
if len(results) == 0 {
|
||||
return ""
|
||||
}
|
||||
blocks := make([]string, 0, len(results))
|
||||
for _, r := range results {
|
||||
blocks = append(blocks, fmt.Sprintf("Title: %s\nURL: %s\nContent: %s", r.Title, r.URL, r.Content))
|
||||
}
|
||||
return strings.Join(blocks, "\n\n")
|
||||
}
|
||||
|
||||
func wikipediaJSON(env wikipediaEnvelope) string {
|
||||
|
||||
@@ -80,14 +80,14 @@ func TestWikipedia_BuildURL(t *testing.T) {
|
||||
if q.Get("action") != "query" {
|
||||
t.Errorf("action = %q, want query", q.Get("action"))
|
||||
}
|
||||
if q.Get("list") != "search" {
|
||||
t.Errorf("list = %q, want search", q.Get("list"))
|
||||
if q.Get("generator") != "search" {
|
||||
t.Errorf("generator = %q, want search", q.Get("generator"))
|
||||
}
|
||||
if q.Get("format") != "json" {
|
||||
t.Errorf("format = %q, want json", q.Get("format"))
|
||||
}
|
||||
if q.Get("srsearch") != tc.query {
|
||||
t.Errorf("srsearch = %q, want %q (raw query, not pre-encoded)", q.Get("srsearch"), tc.query)
|
||||
if q.Get("gsrsearch") != tc.query {
|
||||
t.Errorf("gsrsearch = %q, want %q (raw query, not pre-encoded)", q.Get("gsrsearch"), tc.query)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -96,14 +96,16 @@ func TestWikipedia_BuildURL(t *testing.T) {
|
||||
func TestWikipedia_ParseResults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var gotUA string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotUA = r.Header.Get("User-Agent")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"query": {
|
||||
"search": [
|
||||
{"title":"RAG","snippet":"<span>rag</span> is ..."},
|
||||
{"title":"Retrieval-augmented generation","snippet":"<b>RAG</b> is ..."}
|
||||
]
|
||||
"pages": {
|
||||
"11": {"index":2,"title":"Retrieval-augmented generation","extract":"RAG is a generation technique.","fullurl":"https://en.wikipedia.org/wiki/Retrieval-augmented_generation"},
|
||||
"10": {"index":1,"title":"RAG","extract":"RAG is an acronym.","fullurl":"https://en.wikipedia.org/wiki/RAG"}
|
||||
}
|
||||
}
|
||||
}`))
|
||||
}))
|
||||
@@ -119,6 +121,9 @@ func TestWikipedia_ParseResults(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("InvokableRun: %v", err)
|
||||
}
|
||||
if !strings.Contains(gotUA, "ragflow") {
|
||||
t.Errorf("User-Agent = %q, want to contain ragflow", gotUA)
|
||||
}
|
||||
|
||||
var env wikipediaEnvelope
|
||||
if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {
|
||||
@@ -133,6 +138,12 @@ func TestWikipedia_ParseResults(t *testing.T) {
|
||||
if env.Results[0].Title != "RAG" {
|
||||
t.Errorf("Results[0].Title = %q, want RAG", env.Results[0].Title)
|
||||
}
|
||||
if env.Results[0].Content != "RAG is an acronym." {
|
||||
t.Errorf("Results[0].Content = %q, want summary extract", env.Results[0].Content)
|
||||
}
|
||||
if !strings.Contains(env.FormalizedContent, "RAG is an acronym.") {
|
||||
t.Errorf("FormalizedContent = %q, want rendered summary", env.FormalizedContent)
|
||||
}
|
||||
if !strings.HasPrefix(env.Results[0].URL, "https://en.wikipedia.org/wiki/") {
|
||||
t.Errorf("Results[0].URL = %q, want to start with https://en.wikipedia.org/wiki/", env.Results[0].URL)
|
||||
}
|
||||
@@ -172,8 +183,8 @@ func TestWikipedia_Info(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Info: %v", err)
|
||||
}
|
||||
if info.Name != "wikipedia" {
|
||||
t.Errorf("Name = %q, want wikipedia", info.Name)
|
||||
if info.Name != "wikipedia_search" {
|
||||
t.Errorf("Name = %q, want wikipedia_search", info.Name)
|
||||
}
|
||||
if !strings.Contains(info.Desc, "Wikipedia") {
|
||||
t.Errorf("Desc = %q, want to mention Wikipedia", info.Desc)
|
||||
|
||||
Reference in New Issue
Block a user