fix(go-agent): unknown component "Wikipedia" canvas error. (#16784)

## Summary

- Register Wikipedia component + tool alias
`wikipedia`/`wikipedia_search`
- Use `generator=search` to get title/summary/url in one request (was
N+1)
- Node params `top_n`/`language` with validation
- Return `formalized_content` for downstream
- tests pass

<img width="1817" height="972" alt="image"
src="https://github.com/user-attachments/assets/f6d79599-6d1f-4ea6-84f7-ac06d0de13b0"
/>
This commit is contained in:
Hz_
2026-07-10 16:12:09 +08:00
committed by GitHub
parent fb42e5531d
commit 07a3523b09
8 changed files with 421 additions and 76 deletions

View File

@@ -518,6 +518,7 @@ func init() {
Register(componentNameIteration, NewIterationStub)
Register(componentNameIterationItem, NewIterationItemStub)
Register("BGPT", newBGPTComponent)
Register("Wikipedia", newWikipediaComponent)
Register("DuckDuckGo", newDuckDuckGoComponent)
Register("Google", newGoogleComponent)
Register("GoogleScholar", newGoogleScholarComponent)

View File

@@ -410,6 +410,85 @@ func (c *bgptComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[
return nil, nil
}
// wikipediaComponent delegates to internal/agent/tool/WikipediaTool.
// Python's canvas component is named "Wikipedia" and stores top_n/language
// on the node params while accepting query at runtime.
type wikipediaComponent struct {
inner *agenttool.WikipediaTool
}
func newWikipediaComponent(params map[string]any) (Component, error) {
topN := 10
if v, ok := params["top_n"]; ok {
topN = toIntParam(v)
}
if topN <= 0 {
return nil, fmt.Errorf("canvas: Wikipedia: top_n must be a positive integer")
}
language := "en"
if v, ok := params["language"].(string); ok && strings.TrimSpace(v) != "" {
language = strings.TrimSpace(v)
}
if !agenttool.WikipediaLanguageSupported(language) {
return nil, fmt.Errorf("canvas: Wikipedia: unsupported language %q", language)
}
return &wikipediaComponent{inner: agenttool.NewWikipediaToolWithParams(nil, topN, language)}, nil
}
func (c *wikipediaComponent) Name() string { return "Wikipedia" }
func (c *wikipediaComponent) Inputs() map[string]string {
return map[string]string{
"query": "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.",
}
}
func (c *wikipediaComponent) GetInputForm() map[string]any {
return map[string]any{
"query": map[string]any{
"name": "Query",
"type": "line",
},
}
}
func (c *wikipediaComponent) Outputs() map[string]string {
return map[string]string{
"formalized_content": "Rendered Wikipedia article summaries for downstream LLM prompts.",
"json": "Raw Wikipedia result list.",
}
}
func (c *wikipediaComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
query := strings.TrimSpace(stringParam(inputs["query"]))
if query == "" {
return map[string]any{"formalized_content": "", "json": []any{}}, nil
}
argsJSON, _ := json.Marshal(map[string]any{"query": query})
out, err := c.inner.InvokableRun(ctx, string(argsJSON))
decoded := parseToolEnvelope(out)
if results, ok := decoded["results"]; ok {
decoded["json"] = results
}
if err != nil {
if len(decoded) > 0 {
if _, ok := decoded["formalized_content"]; !ok {
decoded["formalized_content"] = ""
}
if _, ok := decoded["json"]; !ok {
decoded["json"] = []any{}
}
return decoded, nil
}
return nil, fmt.Errorf("canvas: Wikipedia: %w", err)
}
return decoded, nil
}
func (c *wikipediaComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {
return nil, nil
}
// duckDuckGoComponent delegates to internal/agent/tool/DuckDuckGoTool.
type duckDuckGoComponent struct {
inner duckDuckGoInvoker
@@ -1580,6 +1659,7 @@ var (
_ Component = (*duckDuckGoComponent)(nil)
_ Component = (*exesqlComponent)(nil)
_ Component = (*codeExecComponent)(nil)
_ Component = (*wikipediaComponent)(nil)
_ Component = (*googleScholarComponent)(nil)
_ Component = (*yahooFinanceComponent)(nil)
)
@@ -1587,6 +1667,7 @@ var (
// Compile-time check that the eino InvokableTool methods we call
// are reachable (catches a future refactor that renames them).
var _ einotool.InvokableTool = (*agenttool.TavilyTool)(nil)
var _ einotool.InvokableTool = (*agenttool.WikipediaTool)(nil)
var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil)
var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil)
var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil)

View File

@@ -10,12 +10,11 @@ import (
// case-insensitive, and returned in sorted order. The expected count is
// read from plan §2.11.10 — P0 (8) + P1 (5) + P2 (4) + P3 (2) + P4 (3) = 22
// at plan completion, plus v1 fixture wrappers/stubs (including Retrieval,
// TavilySearch, TavilyExtract, ExeSQL, Google, BGPT, YahooFinance, GoogleScholar,
// Generate, Answer, Iteration, and IterationItem) registered by fixture_stubs.go
// to keep the dsl-examples and canvas tool surface compiling. The test allows
// counts between 12 (P0+P1 minus the removed ExitLoop) and 36 (the 22 plan
// components plus the wrappers/stubs currently registered by fixture_stubs.go)
// to roll forward as subsequent batches land.
// TavilySearch, TavilyExtract, ExeSQL, CodeExec, Google, BGPT, YahooFinance,
// Wikipedia, GoogleScholar, DuckDuckGo, Generate, Answer, Iteration, and IterationItem)
// registered by fixture_stubs.go to keep the dsl-examples and canvas tool
// surface compiling. The test allows counts between 12 (P0+P1 minus the
// removed ExitLoop) and 55 to roll forward as subsequent batches land.
//
// Note: ExitLoop is intentionally NOT in the registry anymore. The
// canvas engine (internal/agent/canvas/canvas.go's legacyNoOpNames)
@@ -45,8 +44,8 @@ func TestVerifyRegistration_P1(t *testing.T) {
if len(missing) > 0 {
t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names)
}
if got := len(names); got < 12 || got > 36 {
t.Errorf("expected 12-36 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
if got := len(names); got < 12 || got > 55 {
t.Errorf("expected 12-55 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
}
// ExitLoop must NOT be in the registry (legacy compat lives at

View File

@@ -0,0 +1,132 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package component
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
agenttool "ragflow/internal/agent/tool"
)
func TestWikipedia_RegisteredFactory(t *testing.T) {
c, err := New("Wikipedia", map[string]any{"top_n": float64(3), "language": "en"})
if err != nil {
t.Fatalf("New(Wikipedia) errored: %v", err)
}
if got := c.Name(); got != "Wikipedia" {
t.Fatalf("Name() = %q, want Wikipedia", got)
}
formGetter, ok := c.(interface{ GetInputForm() map[string]any })
if !ok {
t.Fatal("Wikipedia component does not expose GetInputForm")
}
query, ok := formGetter.GetInputForm()["query"].(map[string]any)
if !ok {
t.Fatalf("GetInputForm()[query] has type %T, want map", formGetter.GetInputForm()["query"])
}
if query["type"] != "line" {
t.Fatalf("GetInputForm()[query][type] = %v, want line", query["type"])
}
if _, ok := c.Outputs()["formalized_content"]; !ok {
t.Fatal("Outputs() missing formalized_content")
}
if _, ok := c.Outputs()["json"]; !ok {
t.Fatal("Outputs() missing json")
}
}
func TestWikipedia_InvokeEmptyQueryMatchesPython(t *testing.T) {
c, err := New("Wikipedia", nil)
if err != nil {
t.Fatalf("New(Wikipedia) errored: %v", err)
}
out, err := c.Invoke(context.Background(), map[string]any{"query": " "})
if err != nil {
t.Fatalf("Invoke errored: %v", err)
}
if got, _ := out["formalized_content"].(string); got != "" {
t.Fatalf("formalized_content = %q, want empty", got)
}
if _, ok := out["json"].([]any); !ok {
t.Fatalf("json output has type %T, want []any", out["json"])
}
}
func TestWikipedia_InvokeParamsBakedAtConstruct(t *testing.T) {
t.Parallel()
var gotLimit string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotLimit = r.URL.Query().Get("gsrlimit")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query": {
"pages": {
"10": {"index":1,"title":"RAG","extract":"RAG is an acronym."}
}
}
}`))
}))
defer srv.Close()
helper := agenttool.NewHTTPHelper().WithClient(&http.Client{
Transport: componentRewriteHostTransport(srv.URL),
})
inner := agenttool.NewWikipediaToolWithParams(helper, 2, "en")
c := &wikipediaComponent{inner: inner}
out, err := c.Invoke(context.Background(), map[string]any{"query": "rag"})
if err != nil {
t.Fatalf("Invoke errored: %v", err)
}
if gotLimit != "2" {
t.Fatalf("gsrlimit = %q, want 2", gotLimit)
}
if got, _ := out["formalized_content"].(string); !strings.Contains(got, "RAG is an acronym.") {
t.Fatalf("formalized_content = %q, want rendered result", got)
}
results := anySlice(out["json"])
if len(results) != 1 {
t.Fatalf("json len = %d, want 1", len(results))
}
}
func componentRewriteHostTransport(srvURL string) http.RoundTripper {
u, err := url.Parse(srvURL)
if err != nil {
panic("componentRewriteHostTransport: bad srvURL: " + err.Error())
}
return &componentHostSwapRT{inner: http.DefaultTransport, host: u.Host, scheme: u.Scheme}
}
type componentHostSwapRT struct {
inner http.RoundTripper
host string
scheme string
}
func (t *componentHostSwapRT) RoundTrip(req *http.Request) (*http.Response, error) {
r2 := req.Clone(req.Context())
r2.URL.Scheme = t.scheme
r2.URL.Host = t.host
r2.Host = t.host
return t.inner.RoundTrip(r2)
}

View File

@@ -56,7 +56,8 @@ var registry = map[string]Factory{
"tushare": noConfig("tushare", func() einotool.BaseTool { return NewTushareTool() }),
"wencai": noConfig("wencai", func() einotool.BaseTool { return NewWencaiTool() }),
"web_crawler": noConfig("web_crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
"wikipedia": noConfig("wikipedia", func() einotool.BaseTool { return NewWikipediaTool() }),
"wikipedia": buildWikipediaTool,
"wikipedia_search": buildWikipediaTool,
"yahoo_finance": noConfig("yahoo_finance", func() einotool.BaseTool { return NewYahooFinanceTool() }),
}
@@ -217,6 +218,32 @@ func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {
return NewKeenableToolWithAPIKey(nil, apiKey), nil
}
func buildWikipediaTool(params map[string]any) (einotool.BaseTool, error) {
topN := defaultWikipediaTopN
language := defaultWikipediaLanguage
for key := range params {
if key != "top_n" && key != "language" {
return nil, fmt.Errorf("agent tool: tool %q only accepts node-level params top_n/language", "wikipedia")
}
}
if v, ok := intParam(params, "top_n"); ok {
topN = v
}
if topN <= 0 {
return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "wikipedia")
}
if v, ok := stringParam(params, "language"); ok {
language = strings.TrimSpace(v)
}
if language == "" {
return nil, fmt.Errorf("agent tool: tool %q requires non-empty string node-level param language", "wikipedia")
}
if !WikipediaLanguageSupported(language) {
return nil, fmt.Errorf("agent tool: tool %q unsupported node-level param language %q", "wikipedia", language)
}
return NewWikipediaToolWithParams(nil, topN, language), nil
}
func decodeExeSQLConnParams(params map[string]any) (exesqlConnParams, error) {
if len(params) == 0 {
return exesqlConnParams{}, fmt.Errorf(

View File

@@ -25,8 +25,8 @@ func TestBuildAll_KnownTools(t *testing.T) {
if err != nil {
t.Fatalf("tools[1].Info: %v", err)
}
if info1.Name != "wikipedia" {
t.Errorf("tools[1].Info().Name = %q, want wikipedia", info1.Name)
if info1.Name != "wikipedia_search" {
t.Errorf("tools[1].Info().Name = %q, want wikipedia_search", info1.Name)
}
}
@@ -47,7 +47,7 @@ func TestBuildAll_AllRegisteredTools(t *testing.T) {
"duckduckgo", "email", "exesql", "execute_sql", "github", "google",
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
"yahoo_finance",
}
params := map[string]map[string]any{
@@ -122,7 +122,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
"duckduckgo", "email", "execute_sql", "exesql", "github", "google",
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
"yahoo_finance",
}
params := map[string]map[string]any{
@@ -188,6 +188,8 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
"search_my_dateset": "search_my_dateset",
"crawler": "web_crawler",
"web_crawler": "web_crawler",
"wikipedia": "wikipedia_search",
"wikipedia_search": "wikipedia_search",
}
for _, name := range names {
canonical, ok := canonicalByAlias[name]

View File

@@ -22,43 +22,77 @@ import (
"fmt"
"net/http"
"net/url"
"sort"
"strings"
"github.com/cloudwego/eino/components/tool"
"github.com/cloudwego/eino/schema"
)
const wikipediaToolName = "wikipedia"
const wikipediaToolName = "wikipedia_search"
const wikipediaToolDescription = "Search Wikipedia and return matching articles as {title, snippet, url}."
const wikipediaToolDescription = "A wide range of how-to and information pages are made available in wikipedia. Since 2001, it has grown rapidly to become the world's largest reference website. From Wikipedia, the free encyclopedia."
// wikipediaParams is the JSON shape the model sends into InvokableRun.
// lang is the language subdomain (e.g. "en", "zh", "de"); max_results
// defaults to 5 when unset or non-positive.
type wikipediaParams struct {
Query string `json:"query"`
Lang string `json:"lang"`
MaxResults int `json:"max_results"`
const wikipediaUserAgent = "Mozilla/5.0 (compatible; ragflow/1.0; +https://github.com/infiniflow/ragflow)"
const (
defaultWikipediaTopN = 10
defaultWikipediaLanguage = "en"
)
var wikipediaLanguages = map[string]struct{}{
"af": {}, "pl": {}, "ar": {}, "ast": {}, "az": {}, "bg": {}, "nan": {}, "bn": {}, "be": {}, "ca": {},
"cs": {}, "cy": {}, "da": {}, "de": {}, "et": {}, "el": {}, "en": {}, "es": {}, "eo": {}, "eu": {},
"fa": {}, "fr": {}, "gl": {}, "ko": {}, "hy": {}, "hi": {}, "hr": {}, "id": {}, "it": {}, "he": {},
"ka": {}, "lld": {}, "la": {}, "lv": {}, "lt": {}, "hu": {}, "mk": {}, "arz": {}, "ms": {}, "min": {},
"my": {}, "nl": {}, "ja": {}, "nb": {}, "nn": {}, "ce": {}, "uz": {}, "pt": {}, "kk": {}, "ro": {},
"ru": {}, "ceb": {}, "sk": {}, "sl": {}, "sr": {}, "sh": {}, "fi": {}, "sv": {}, "ta": {}, "tt": {},
"th": {}, "tg": {}, "azb": {}, "tr": {}, "uk": {}, "ur": {}, "vi": {}, "war": {}, "zh": {}, "yue": {},
}
// wikipediaResult is one row in the upstream `query.search` array.
// WikipediaLanguageSupported mirrors Python WikipediaParam.check's accepted
// language list.
func WikipediaLanguageSupported(language string) bool {
_, ok := wikipediaLanguages[strings.TrimSpace(language)]
return ok
}
// wikipediaParams is the JSON shape the model sends into InvokableRun.
// LLM only sees query via Info(); lang and max_results are accepted for
// non-agent callers and fall back to the tool instance's node-level
// defaults (WikipediaTool.lang / WikipediaTool.topN) when absent.
type wikipediaParams struct {
Query string `json:"query"`
Lang string `json:"lang,omitempty"`
MaxResults int `json:"max_results,omitempty"`
}
// wikipediaResult mirrors the fields Python passes to _retrieve_chunks:
// title, url, and summary content.
type wikipediaResult struct {
Title string `json:"title"`
Snippet string `json:"snippet"`
Content string `json:"content"`
URL string `json:"url"`
}
// wikipediaResponse is the upstream MediaWiki API envelope.
// wikipediaResponse is the upstream MediaWiki generator=search envelope.
type wikipediaResponse struct {
Query struct {
Search []wikipediaResult `json:"search"`
Pages map[string]struct {
Index int `json:"index"`
Title string `json:"title"`
Extract string `json:"extract"`
FullURL string `json:"fullurl"`
} `json:"pages"`
} `json:"query"`
}
// wikipediaEnvelope is what the model sees. It mirrors the Python tool's
// output: a flat list of {title, snippet, url} entries.
// wikipediaEnvelope is what the model sees. The formalized_content field
// is the canvas-facing equivalent of Python ToolBase._retrieve_chunks().
type wikipediaEnvelope struct {
Results []wikipediaResult `json:"results"`
Error string `json:"_ERROR,omitempty"`
FormalizedContent string `json:"formalized_content,omitempty"`
Results []wikipediaResult `json:"results,omitempty"`
Error string `json:"_ERROR,omitempty"`
}
// WikipediaTool is the Wikipedia
@@ -67,6 +101,8 @@ type wikipediaEnvelope struct {
// top N matches for the query.
type WikipediaTool struct {
helper *HTTPHelper
topN int
lang string
}
// NewWikipediaTool returns a WikipediaTool using the default HTTPHelper.
@@ -77,10 +113,22 @@ func NewWikipediaTool() *WikipediaTool {
// NewWikipediaToolWith returns a WikipediaTool that uses the provided
// HTTPHelper. Useful for tests that want to inject a custom transport.
func NewWikipediaToolWith(h *HTTPHelper) *WikipediaTool {
return NewWikipediaToolWithParams(h, defaultWikipediaTopN, defaultWikipediaLanguage)
}
// NewWikipediaToolWithParams returns a WikipediaTool with node-level
// parameters matching Python's WikipediaParam.top_n and language fields.
func NewWikipediaToolWithParams(h *HTTPHelper, topN int, language string) *WikipediaTool {
if h == nil {
h = NewHTTPHelper()
}
return &WikipediaTool{helper: h}
if topN <= 0 {
topN = defaultWikipediaTopN
}
if strings.TrimSpace(language) == "" {
language = defaultWikipediaLanguage
}
return &WikipediaTool{helper: h, topN: topN, lang: strings.TrimSpace(language)}
}
// Info returns the tool's metadata for the chat model.
@@ -91,36 +139,27 @@ func (w *WikipediaTool) Info(_ context.Context) (*schema.ToolInfo, error) {
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
"query": {
Type: schema.String,
Desc: "Search query",
Desc: "The search keyword to execute with wikipedia. The keyword MUST be a specific subject that can match the title.",
Required: true,
},
"lang": {
Type: schema.String,
Desc: `Language subdomain. Defaults to "en".`,
Required: false,
},
"max_results": {
Type: schema.Integer,
Desc: "Maximum number of results to return. Defaults to 5.",
Required: false,
},
}),
}, nil
}
// buildWikipediaURL constructs the MediaWiki action=query URL. Centralized
// so the test suite can verify URL encoding without spinning up a server.
func buildWikipediaURL(lang, query string, maxResults int) string {
// buildWikipediaURL constructs a MediaWiki generator=search URL that returns
// the same page fields Python reads from wikipedia.page(): title, url,
// and summary-like introductory extract.
func buildWikipediaURL(lang, query string, topN int) string {
if lang == "" {
lang = "en"
lang = defaultWikipediaLanguage
}
if maxResults <= 0 {
maxResults = 5
if topN <= 0 {
topN = defaultWikipediaTopN
}
return fmt.Sprintf(
"https://%s.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=%d&srsearch=%s",
"https://%s.wikipedia.org/w/api.php?action=query&generator=search&format=json&gsrlimit=%d&gsrsearch=%s&prop=extracts%%7Cinfo&exintro=1&explaintext=1&inprop=url",
lang,
maxResults,
topN,
url.QueryEscape(query),
)
}
@@ -137,8 +176,22 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ...
fmt.Errorf("wikipedia: query is required")
}
endpoint := buildWikipediaURL(p.Lang, p.Query, p.MaxResults)
resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", nil)
lang := p.Lang
if lang == "" {
lang = w.lang
}
maxResults := p.MaxResults
if maxResults <= 0 {
maxResults = w.topN
}
if !WikipediaLanguageSupported(lang) {
return wikipediaErrJSON(fmt.Errorf("wikipedia: unsupported language %q", lang)),
fmt.Errorf("wikipedia: unsupported language %q", lang)
}
endpoint := buildWikipediaURL(lang, p.Query, maxResults)
resp, err := w.helper.Do(ctx, http.MethodGet, endpoint, "", "", map[string]string{
"User-Agent": wikipediaUserAgent,
})
if err != nil {
return wikipediaErrJSON(err), err
}
@@ -155,22 +208,61 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ...
fmt.Errorf("wikipedia: decode response: %w", err)
}
// Compose canonical Wikipedia URLs for the snippets. The MediaWiki
// search endpoint doesn't include a URL; the conventional one is
// https://<lang>.wikipedia.org/wiki/<Title>.
lang := p.Lang
if lang == "" {
lang = "en"
}
results := make([]wikipediaResult, 0, len(raw.Query.Search))
for _, r := range raw.Query.Search {
results = append(results, wikipediaResult{
Title: r.Title,
Snippet: r.Snippet,
URL: fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(r.Title)),
pages := make([]struct {
Index int
Title string
Extract string
FullURL string
}, 0, len(raw.Query.Pages))
for _, p := range raw.Query.Pages {
pages = append(pages, struct {
Index int
Title string
Extract string
FullURL string
}{
Index: p.Index,
Title: p.Title,
Extract: p.Extract,
FullURL: p.FullURL,
})
}
return wikipediaJSON(wikipediaEnvelope{Results: results}), nil
sort.SliceStable(pages, func(i, j int) bool { return pages[i].Index < pages[j].Index })
results := make([]wikipediaResult, 0, len(pages))
for _, p := range pages {
content := strings.TrimSpace(p.Extract)
if content == "" {
continue
}
if len(content) > 10000 {
content = content[:10000]
}
fullURL := p.FullURL
if fullURL == "" {
fullURL = fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(p.Title))
}
results = append(results, wikipediaResult{
Title: p.Title,
Content: content,
URL: fullURL,
})
}
return wikipediaJSON(wikipediaEnvelope{
FormalizedContent: renderWikipediaResults(results),
Results: results,
}), nil
}
func renderWikipediaResults(results []wikipediaResult) string {
if len(results) == 0 {
return ""
}
blocks := make([]string, 0, len(results))
for _, r := range results {
blocks = append(blocks, fmt.Sprintf("Title: %s\nURL: %s\nContent: %s", r.Title, r.URL, r.Content))
}
return strings.Join(blocks, "\n\n")
}
func wikipediaJSON(env wikipediaEnvelope) string {

View File

@@ -80,14 +80,14 @@ func TestWikipedia_BuildURL(t *testing.T) {
if q.Get("action") != "query" {
t.Errorf("action = %q, want query", q.Get("action"))
}
if q.Get("list") != "search" {
t.Errorf("list = %q, want search", q.Get("list"))
if q.Get("generator") != "search" {
t.Errorf("generator = %q, want search", q.Get("generator"))
}
if q.Get("format") != "json" {
t.Errorf("format = %q, want json", q.Get("format"))
}
if q.Get("srsearch") != tc.query {
t.Errorf("srsearch = %q, want %q (raw query, not pre-encoded)", q.Get("srsearch"), tc.query)
if q.Get("gsrsearch") != tc.query {
t.Errorf("gsrsearch = %q, want %q (raw query, not pre-encoded)", q.Get("gsrsearch"), tc.query)
}
})
}
@@ -96,14 +96,16 @@ func TestWikipedia_BuildURL(t *testing.T) {
func TestWikipedia_ParseResults(t *testing.T) {
t.Parallel()
var gotUA string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotUA = r.Header.Get("User-Agent")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query": {
"search": [
{"title":"RAG","snippet":"<span>rag</span> is ..."},
{"title":"Retrieval-augmented generation","snippet":"<b>RAG</b> is ..."}
]
"pages": {
"11": {"index":2,"title":"Retrieval-augmented generation","extract":"RAG is a generation technique.","fullurl":"https://en.wikipedia.org/wiki/Retrieval-augmented_generation"},
"10": {"index":1,"title":"RAG","extract":"RAG is an acronym.","fullurl":"https://en.wikipedia.org/wiki/RAG"}
}
}
}`))
}))
@@ -119,6 +121,9 @@ func TestWikipedia_ParseResults(t *testing.T) {
if err != nil {
t.Fatalf("InvokableRun: %v", err)
}
if !strings.Contains(gotUA, "ragflow") {
t.Errorf("User-Agent = %q, want to contain ragflow", gotUA)
}
var env wikipediaEnvelope
if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {
@@ -133,6 +138,12 @@ func TestWikipedia_ParseResults(t *testing.T) {
if env.Results[0].Title != "RAG" {
t.Errorf("Results[0].Title = %q, want RAG", env.Results[0].Title)
}
if env.Results[0].Content != "RAG is an acronym." {
t.Errorf("Results[0].Content = %q, want summary extract", env.Results[0].Content)
}
if !strings.Contains(env.FormalizedContent, "RAG is an acronym.") {
t.Errorf("FormalizedContent = %q, want rendered summary", env.FormalizedContent)
}
if !strings.HasPrefix(env.Results[0].URL, "https://en.wikipedia.org/wiki/") {
t.Errorf("Results[0].URL = %q, want to start with https://en.wikipedia.org/wiki/", env.Results[0].URL)
}
@@ -172,8 +183,8 @@ func TestWikipedia_Info(t *testing.T) {
if err != nil {
t.Fatalf("Info: %v", err)
}
if info.Name != "wikipedia" {
t.Errorf("Name = %q, want wikipedia", info.Name)
if info.Name != "wikipedia_search" {
t.Errorf("Name = %q, want wikipedia_search", info.Name)
}
if !strings.Contains(info.Desc, "Wikipedia") {
t.Errorf("Desc = %q, want to mention Wikipedia", info.Desc)