mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 14:15:40 +08:00
feat(go-agent): merge Google Scholar node params with runtime inputs (#16802)
## Summary - align the Go Google Scholar component with the Python-side config pattern - merge node-level params with runtime inputs so canvas defaults are preserved and per-run inputs can override them - add tests covering node param fallback and runtime override behavior ## Verification - `bash build.sh --test ./internal/agent/component/... -run TestGoogleScholar` <img width="1873" height="1165" alt="image" src="https://github.com/user-attachments/assets/67198c6f-6a0e-43bf-a500-8e88d82b8751" />
This commit is contained in:
@@ -520,5 +520,6 @@ func init() {
|
||||
Register("BGPT", newBGPTComponent)
|
||||
Register("DuckDuckGo", newDuckDuckGoComponent)
|
||||
Register("Google", newGoogleComponent)
|
||||
Register("GoogleScholar", newGoogleScholarComponent)
|
||||
Register("YahooFinance", newYahooFinanceComponent)
|
||||
}
|
||||
|
||||
131
internal/agent/component/google_scholar_component_test.go
Normal file
131
internal/agent/component/google_scholar_component_test.go
Normal file
@@ -0,0 +1,131 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package component
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
einotool "github.com/cloudwego/eino/components/tool"
|
||||
)
|
||||
|
||||
type fakeGoogleScholarInvoker struct {
|
||||
args map[string]any
|
||||
}
|
||||
|
||||
func (f *fakeGoogleScholarInvoker) InvokableRun(_ context.Context, argsJSON string, _ ...einotool.Option) (string, error) {
|
||||
if err := json.Unmarshal([]byte(argsJSON), &f.args); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return `{"results":[{"title":"Paper","link":"https://example.com","authors":"A Author","year":"2024","snippet":"Abstract"}]}`, nil
|
||||
}
|
||||
|
||||
func TestGoogleScholar_InvokePassesCanvasParams(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := &fakeGoogleScholarInvoker{}
|
||||
c := newGoogleScholarComponentWithInvoker(fake, nil)
|
||||
|
||||
_, err := c.Invoke(context.Background(), map[string]any{
|
||||
"query": " retrieval augmented generation ",
|
||||
"top_n": float64(7),
|
||||
"sort_by": "date",
|
||||
"year_low": float64(2020),
|
||||
"year_high": float64(2024),
|
||||
"patents": false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke errored: %v", err)
|
||||
}
|
||||
|
||||
if got := fake.args["query"]; got != "retrieval augmented generation" {
|
||||
t.Errorf("query arg = %v, want trimmed query", got)
|
||||
}
|
||||
if got := fake.args["top_n"]; got != float64(7) {
|
||||
t.Errorf("top_n arg = %v, want 7", got)
|
||||
}
|
||||
if _, ok := fake.args["max_results"]; ok {
|
||||
t.Fatalf("max_results should not be sent for GoogleScholar args: %#v", fake.args)
|
||||
}
|
||||
if got := fake.args["sort_by"]; got != "date" {
|
||||
t.Errorf("sort_by arg = %v, want date", got)
|
||||
}
|
||||
if got := fake.args["year_low"]; got != float64(2020) {
|
||||
t.Errorf("year_low arg = %v, want 2020", got)
|
||||
}
|
||||
if got := fake.args["year_high"]; got != float64(2024) {
|
||||
t.Errorf("year_high arg = %v, want 2024", got)
|
||||
}
|
||||
if got := fake.args["patents"]; got != false {
|
||||
t.Errorf("patents arg = %v, want false", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleScholar_InvokeMergesNodeParams(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := &fakeGoogleScholarInvoker{}
|
||||
c := newGoogleScholarComponentWithInvoker(fake, map[string]any{
|
||||
"top_n": float64(20),
|
||||
"sort_by": "date",
|
||||
"patents": false,
|
||||
})
|
||||
|
||||
_, err := c.Invoke(context.Background(), map[string]any{
|
||||
"query": "machine learning",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke errored: %v", err)
|
||||
}
|
||||
|
||||
if got := fake.args["top_n"]; got != float64(20) {
|
||||
t.Errorf("top_n arg = %v, want 20 (from node params)", got)
|
||||
}
|
||||
if got := fake.args["sort_by"]; got != "date" {
|
||||
t.Errorf("sort_by arg = %v, want date (from node params)", got)
|
||||
}
|
||||
if got := fake.args["patents"]; got != false {
|
||||
t.Errorf("patents arg = %v, want false (from node params)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleScholar_InvokeInputsOverrideNodeParams(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := &fakeGoogleScholarInvoker{}
|
||||
c := newGoogleScholarComponentWithInvoker(fake, map[string]any{
|
||||
"top_n": float64(20),
|
||||
"sort_by": "relevance",
|
||||
})
|
||||
|
||||
_, err := c.Invoke(context.Background(), map[string]any{
|
||||
"query": "deep learning",
|
||||
"top_n": float64(5),
|
||||
"sort_by": "date",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke errored: %v", err)
|
||||
}
|
||||
|
||||
if got := fake.args["top_n"]; got != float64(5) {
|
||||
t.Errorf("top_n arg = %v, want 5 (inputs override node params)", got)
|
||||
}
|
||||
if got := fake.args["sort_by"]; got != "date" {
|
||||
t.Errorf("sort_by arg = %v, want date (inputs override node params)", got)
|
||||
}
|
||||
}
|
||||
@@ -1422,6 +1422,155 @@ func (c *yahooFinanceComponent) Stream(_ context.Context, _ map[string]any) (<-c
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// googleScholarComponent delegates to internal/agent/tool/GoogleScholarTool.
|
||||
type googleScholarComponent struct {
|
||||
inner googleScholarInvoker
|
||||
params map[string]any
|
||||
}
|
||||
|
||||
type googleScholarInvoker interface {
|
||||
InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error)
|
||||
}
|
||||
|
||||
func newGoogleScholarComponent(params map[string]any) (Component, error) {
|
||||
cloned := make(map[string]any, len(params))
|
||||
for k, v := range params {
|
||||
cloned[k] = v
|
||||
}
|
||||
return &googleScholarComponent{
|
||||
inner: agenttool.NewGoogleScholarTool(),
|
||||
params: cloned,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newGoogleScholarComponentWithInvoker(inner googleScholarInvoker, params map[string]any) Component {
|
||||
cloned := make(map[string]any, len(params))
|
||||
for k, v := range params {
|
||||
cloned[k] = v
|
||||
}
|
||||
return &googleScholarComponent{inner: inner, params: cloned}
|
||||
}
|
||||
|
||||
func (c *googleScholarComponent) Name() string { return "GoogleScholar" }
|
||||
|
||||
func (c *googleScholarComponent) Inputs() map[string]string {
|
||||
return map[string]string{
|
||||
"query": "Search query.",
|
||||
"top_n": "Maximum number of results (default 12).",
|
||||
"sort_by": "Sort order: relevance or date.",
|
||||
"year_low": "Earliest publication year to include.",
|
||||
"year_high": "Latest publication year to include.",
|
||||
"patents": "Whether to include patents, defaults to true.",
|
||||
}
|
||||
}
|
||||
|
||||
func (c *googleScholarComponent) Outputs() map[string]string {
|
||||
return map[string]string{
|
||||
"formalized_content": "Rendered search results for downstream LLM prompts.",
|
||||
"json": "Raw result list.",
|
||||
}
|
||||
}
|
||||
|
||||
func (c *googleScholarComponent) GetInputForm() map[string]any {
|
||||
return map[string]any{
|
||||
"query": map[string]any{
|
||||
"name": "Query",
|
||||
"type": "line",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *googleScholarComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) {
|
||||
merged := make(map[string]any, len(c.params)+len(inputs))
|
||||
for k, v := range c.params {
|
||||
merged[k] = v
|
||||
}
|
||||
for k, v := range inputs {
|
||||
merged[k] = v
|
||||
}
|
||||
|
||||
query := strings.TrimSpace(stringParam(merged["query"]))
|
||||
if query == "" {
|
||||
return map[string]any{"formalized_content": "", "json": []any{}}, nil
|
||||
}
|
||||
args := map[string]any{
|
||||
"query": query,
|
||||
}
|
||||
if topN := toIntParam(merged["top_n"]); topN > 0 {
|
||||
args["top_n"] = topN
|
||||
}
|
||||
if sortBy := strings.TrimSpace(stringParam(merged["sort_by"])); sortBy != "" {
|
||||
args["sort_by"] = sortBy
|
||||
}
|
||||
if yearLow := toIntParam(merged["year_low"]); yearLow > 0 {
|
||||
args["year_low"] = yearLow
|
||||
}
|
||||
if yearHigh := toIntParam(merged["year_high"]); yearHigh > 0 {
|
||||
args["year_high"] = yearHigh
|
||||
}
|
||||
if patents, ok := merged["patents"].(bool); ok {
|
||||
args["patents"] = patents
|
||||
} else {
|
||||
args["patents"] = true
|
||||
}
|
||||
|
||||
argsJSON, _ := json.Marshal(args)
|
||||
out, err := c.inner.InvokableRun(ctx, string(argsJSON))
|
||||
decoded := parseToolEnvelope(out)
|
||||
if err != nil {
|
||||
if len(decoded) > 0 {
|
||||
return map[string]any{
|
||||
"formalized_content": "",
|
||||
"json": []any{},
|
||||
"_ERROR": decoded["_ERROR"],
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("canvas: GoogleScholar: %w", err)
|
||||
}
|
||||
|
||||
results := anySlice(decoded["results"])
|
||||
return map[string]any{
|
||||
"formalized_content": renderGoogleScholarResults(results),
|
||||
"json": results,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *googleScholarComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func renderGoogleScholarResults(results []any) string {
|
||||
if len(results) == 0 {
|
||||
return ""
|
||||
}
|
||||
blocks := make([]string, 0, len(results))
|
||||
for _, item := range results {
|
||||
m, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
field := func(key string) string {
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return "-"
|
||||
}
|
||||
text := strings.TrimSpace(fmt.Sprintf("%v", v))
|
||||
if text == "" {
|
||||
return "-"
|
||||
}
|
||||
return text
|
||||
}
|
||||
blocks = append(blocks, strings.Join([]string{
|
||||
fmt.Sprintf("Title: %s", field("title")),
|
||||
fmt.Sprintf("URL: %s", field("link")),
|
||||
fmt.Sprintf("Authors: %s", field("authors")),
|
||||
fmt.Sprintf("Year: %s", field("year")),
|
||||
fmt.Sprintf("Snippet: %s", field("snippet")),
|
||||
}, "\n"))
|
||||
}
|
||||
return strings.Join(blocks, "\n\n")
|
||||
}
|
||||
|
||||
// Compile-time interface checks.
|
||||
var (
|
||||
_ Component = (*retrievalComponent)(nil)
|
||||
@@ -1431,6 +1580,7 @@ var (
|
||||
_ Component = (*duckDuckGoComponent)(nil)
|
||||
_ Component = (*exesqlComponent)(nil)
|
||||
_ Component = (*codeExecComponent)(nil)
|
||||
_ Component = (*googleScholarComponent)(nil)
|
||||
_ Component = (*yahooFinanceComponent)(nil)
|
||||
)
|
||||
|
||||
@@ -1441,3 +1591,4 @@ var _ einotool.InvokableTool = (*agenttool.GoogleTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.TavilyExtractTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.DuckDuckGoTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.YahooFinanceTool)(nil)
|
||||
var _ einotool.InvokableTool = (*agenttool.GoogleScholarTool)(nil)
|
||||
|
||||
@@ -10,12 +10,12 @@ import (
|
||||
// case-insensitive, and returned in sorted order. The expected count is
|
||||
// read from plan §2.11.10 — P0 (8) + P1 (5) + P2 (4) + P3 (2) + P4 (3) = 22
|
||||
// at plan completion, plus v1 fixture wrappers/stubs (including Retrieval,
|
||||
// TavilySearch, TavilyExtract, ExeSQL, Google, BGPT, YahooFinance, Generate,
|
||||
// Answer, Iteration, and IterationItem) registered by fixture_stubs.go to keep
|
||||
// the dsl-examples and canvas tool surface compiling. The test allows counts
|
||||
// between 12 (P0+P1 minus the removed ExitLoop) and 34 (the 22 plan components
|
||||
// plus the wrappers/stubs currently registered by fixture_stubs.go) to roll
|
||||
// forward as subsequent batches land.
|
||||
// TavilySearch, TavilyExtract, ExeSQL, Google, BGPT, YahooFinance, GoogleScholar,
|
||||
// Generate, Answer, Iteration, and IterationItem) registered by fixture_stubs.go
|
||||
// to keep the dsl-examples and canvas tool surface compiling. The test allows
|
||||
// counts between 12 (P0+P1 minus the removed ExitLoop) and 36 (the 22 plan
|
||||
// components plus the wrappers/stubs currently registered by fixture_stubs.go)
|
||||
// to roll forward as subsequent batches land.
|
||||
//
|
||||
// Note: ExitLoop is intentionally NOT in the registry anymore. The
|
||||
// canvas engine (internal/agent/canvas/canvas.go's legacyNoOpNames)
|
||||
@@ -45,8 +45,8 @@ func TestVerifyRegistration_P1(t *testing.T) {
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("missing P0/P1 components: %v (have %d: %v)", missing, len(names), names)
|
||||
}
|
||||
if got := len(names); got < 12 || got > 35 {
|
||||
t.Errorf("expected 12-35 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
|
||||
if got := len(names); got < 12 || got > 36 {
|
||||
t.Errorf("expected 12-36 registered (current plan scope + v1 wrappers/stubs), got %d: %v", got, names)
|
||||
}
|
||||
|
||||
// ExitLoop must NOT be in the registry (legacy compat lives at
|
||||
|
||||
@@ -32,12 +32,22 @@ import (
|
||||
|
||||
const googleScholarToolName = "google_scholar"
|
||||
|
||||
const googleScholarToolDescription = "Search Google Scholar for academic articles. Returns {title, link, snippet, authors, year}."
|
||||
const googleScholarToolDescription = "Google Scholar provides a simple way to broadly search for scholarly literature. From one place, you can search across many disciplines and sources: articles, theses, books, abstracts and court opinions, from academic publishers, professional societies, online repositories, universities and other web sites. Google Scholar helps you find relevant work across the world of scholarly research."
|
||||
|
||||
const defaultGoogleScholarTopN = 12
|
||||
|
||||
// googleScholarParams is the JSON shape the model sends into InvokableRun.
|
||||
// All fields come from the canvas node form; only query is exposed to
|
||||
// the LLM via Info() (matching Python's meta — the LLM only sees query).
|
||||
// Patents uses *bool so the zero value (nil) means "not set" → default
|
||||
// true (include patents). False means explicitly exclude patents.
|
||||
type googleScholarParams struct {
|
||||
Query string `json:"query"`
|
||||
MaxResults int `json:"max_results"`
|
||||
Query string `json:"query"`
|
||||
TopN int `json:"top_n"`
|
||||
SortBy string `json:"sort_by"`
|
||||
YearLow int `json:"year_low"`
|
||||
YearHigh int `json:"year_high"`
|
||||
Patents *bool `json:"patents"`
|
||||
}
|
||||
|
||||
// googleScholarResult is one row in the parsed result list.
|
||||
@@ -64,7 +74,8 @@ var googleScholarEndpoint = "https://scholar.google.com/scholar"
|
||||
// There is no public Scholar API, so we fetch the search-results
|
||||
// HTML and parse it with golang.org/x/net/html.
|
||||
type GoogleScholarTool struct {
|
||||
helper *HTTPHelper
|
||||
helper *HTTPHelper
|
||||
defaults googleScholarParams
|
||||
}
|
||||
|
||||
// NewGoogleScholarTool returns a GoogleScholarTool using the default
|
||||
@@ -82,7 +93,19 @@ func NewGoogleScholarToolWith(h *HTTPHelper) *GoogleScholarTool {
|
||||
return &GoogleScholarTool{helper: h}
|
||||
}
|
||||
|
||||
// NewGoogleScholarToolWithDefaults returns a GoogleScholarTool that
|
||||
// uses the provided HTTPHelper and node-level default params.
|
||||
func NewGoogleScholarToolWithDefaults(h *HTTPHelper, defaults googleScholarParams) *GoogleScholarTool {
|
||||
if h == nil {
|
||||
h = NewHTTPHelper()
|
||||
}
|
||||
return &GoogleScholarTool{helper: h, defaults: defaults}
|
||||
}
|
||||
|
||||
// Info returns the tool's metadata for the chat model.
|
||||
// Only query is exposed — matching Python's meta which does not
|
||||
// include top_n / sort_by / year_low / year_high / patents.
|
||||
// Those are canvas-form-only parameters passed by the component wrapper.
|
||||
func (g *GoogleScholarTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
return &schema.ToolInfo{
|
||||
Name: googleScholarToolName,
|
||||
@@ -90,31 +113,39 @@ func (g *GoogleScholarTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
|
||||
"query": {
|
||||
Type: schema.String,
|
||||
Desc: "Search query.",
|
||||
Desc: "The search keyword to execute with Google Scholar. The keywords should be the most important words/terms(includes synonyms) from the original request.",
|
||||
Required: true,
|
||||
},
|
||||
"max_results": {
|
||||
Type: schema.Integer,
|
||||
Desc: "Maximum number of results to return. Defaults to 5 (max 20 per page).",
|
||||
Required: false,
|
||||
},
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildGoogleScholarURL composes the Scholar query URL. Centralized
|
||||
// for testability.
|
||||
func buildGoogleScholarURL(query string, maxResults int) string {
|
||||
// for testability. sortBy: "relevance" (default) or "date".
|
||||
// yearLow / yearHigh: 0 means no filter. patents: nil or true includes
|
||||
// patents; false explicitly excludes them.
|
||||
func buildGoogleScholarURL(query string, maxResults int, sortBy string, yearLow, yearHigh int, patents *bool) string {
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
}
|
||||
if maxResults > 20 {
|
||||
maxResults = 20
|
||||
maxResults = defaultGoogleScholarTopN
|
||||
}
|
||||
q := url.Values{}
|
||||
q.Set("q", query)
|
||||
q.Set("hl", "en")
|
||||
q.Set("num", strconv.Itoa(maxResults))
|
||||
|
||||
if sortBy == "date" {
|
||||
q.Set("scisbd", "1")
|
||||
}
|
||||
if yearLow > 0 {
|
||||
q.Set("as_ylo", strconv.Itoa(yearLow))
|
||||
}
|
||||
if yearHigh > 0 {
|
||||
q.Set("as_yhi", strconv.Itoa(yearHigh))
|
||||
}
|
||||
if patents != nil && !*patents {
|
||||
q.Set("as_vis", "1")
|
||||
}
|
||||
|
||||
return googleScholarEndpoint + "?" + q.Encode()
|
||||
}
|
||||
|
||||
@@ -125,12 +156,13 @@ func (g *GoogleScholarTool) InvokableRun(ctx context.Context, argsJSON string, _
|
||||
return googleScholarErrJSON(fmt.Errorf("google_scholar: parse arguments: %w", err)),
|
||||
fmt.Errorf("google_scholar: parse arguments: %w", err)
|
||||
}
|
||||
p = mergeGoogleScholarDefaults(g.defaults, p)
|
||||
if strings.TrimSpace(p.Query) == "" {
|
||||
return googleScholarErrJSON(fmt.Errorf("query is required")),
|
||||
fmt.Errorf("google_scholar: query is required")
|
||||
}
|
||||
|
||||
endpoint := buildGoogleScholarURL(p.Query, p.MaxResults)
|
||||
endpoint := buildGoogleScholarURL(p.Query, p.TopN, p.SortBy, p.YearLow, p.YearHigh, p.Patents)
|
||||
headers := map[string]string{
|
||||
// Scholar blocks obviously non-browser UAs.
|
||||
"User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)",
|
||||
@@ -148,7 +180,7 @@ func (g *GoogleScholarTool) InvokableRun(ctx context.Context, argsJSON string, _
|
||||
fmt.Errorf("google_scholar: upstream returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
results, err := parseGoogleScholarHTML(resp.Body, p.MaxResults)
|
||||
results, err := parseGoogleScholarHTML(resp.Body, p.TopN)
|
||||
if err != nil {
|
||||
return googleScholarErrJSON(fmt.Errorf("google_scholar: parse html: %w", err)),
|
||||
fmt.Errorf("google_scholar: parse html: %w", err)
|
||||
@@ -156,6 +188,28 @@ func (g *GoogleScholarTool) InvokableRun(ctx context.Context, argsJSON string, _
|
||||
return googleScholarJSON(googleScholarEnvelope{Results: results}), nil
|
||||
}
|
||||
|
||||
func mergeGoogleScholarDefaults(defaults, p googleScholarParams) googleScholarParams {
|
||||
if p.Query == "" {
|
||||
p.Query = defaults.Query
|
||||
}
|
||||
if p.TopN == 0 {
|
||||
p.TopN = defaults.TopN
|
||||
}
|
||||
if p.SortBy == "" {
|
||||
p.SortBy = defaults.SortBy
|
||||
}
|
||||
if p.YearLow == 0 {
|
||||
p.YearLow = defaults.YearLow
|
||||
}
|
||||
if p.YearHigh == 0 {
|
||||
p.YearHigh = defaults.YearHigh
|
||||
}
|
||||
if p.Patents == nil {
|
||||
p.Patents = defaults.Patents
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// parseGoogleScholarHTML walks the Scholar search-results HTML and
|
||||
// extracts the conventional .gs_rt / .gs_a / .gs_rs fields. We
|
||||
// deliberately stay defensive: Scholar's markup changes without
|
||||
@@ -165,7 +219,7 @@ func parseGoogleScholarHTML(body interface {
|
||||
Read(p []byte) (int, error)
|
||||
}, maxResults int) ([]googleScholarResult, error) {
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
maxResults = defaultGoogleScholarTopN
|
||||
}
|
||||
doc, err := html.Parse(body)
|
||||
if err != nil {
|
||||
|
||||
@@ -52,42 +52,104 @@ func TestGoogleScholar_BuildURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
max int
|
||||
wantNum string
|
||||
wantHost string
|
||||
wantQuery string
|
||||
name string
|
||||
query string
|
||||
topN int
|
||||
sortBy string
|
||||
yearLow int
|
||||
yearHigh int
|
||||
patents *bool
|
||||
wantNum string
|
||||
wantHost string
|
||||
wantQuery string
|
||||
wantParams map[string]string
|
||||
}{
|
||||
{
|
||||
name: "default",
|
||||
query: "transformer",
|
||||
max: 0,
|
||||
wantNum: "5",
|
||||
wantNum: "12",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "transformer",
|
||||
},
|
||||
{
|
||||
name: "clamped high",
|
||||
name: "explicit high",
|
||||
query: "x",
|
||||
max: 99,
|
||||
wantNum: "20",
|
||||
topN: 99,
|
||||
wantNum: "99",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "x",
|
||||
},
|
||||
{
|
||||
name: "explicit",
|
||||
query: "a b",
|
||||
max: 3,
|
||||
topN: 3,
|
||||
wantNum: "3",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "a b",
|
||||
},
|
||||
{
|
||||
name: "sort by date",
|
||||
query: "test",
|
||||
topN: 5,
|
||||
sortBy: "date",
|
||||
wantNum: "5",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "test",
|
||||
wantParams: map[string]string{"scisbd": "1"},
|
||||
},
|
||||
{
|
||||
name: "year range",
|
||||
query: "ml",
|
||||
topN: 10,
|
||||
yearLow: 2020,
|
||||
yearHigh: 2024,
|
||||
wantNum: "10",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "ml",
|
||||
wantParams: map[string]string{"as_ylo": "2020", "as_yhi": "2024"},
|
||||
},
|
||||
{
|
||||
name: "exclude patents",
|
||||
query: "ai",
|
||||
topN: 5,
|
||||
patents: boolPtr(false),
|
||||
wantNum: "5",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "ai",
|
||||
wantParams: map[string]string{"as_vis": "1"},
|
||||
},
|
||||
{
|
||||
name: "include patents (default)",
|
||||
query: "nlp",
|
||||
topN: 5,
|
||||
patents: boolPtr(true),
|
||||
wantNum: "5",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "nlp",
|
||||
},
|
||||
{
|
||||
name: "all params combined",
|
||||
query: "deep learning",
|
||||
topN: 8,
|
||||
sortBy: "date",
|
||||
yearLow: 2019,
|
||||
yearHigh: 2023,
|
||||
patents: boolPtr(false),
|
||||
wantNum: "8",
|
||||
wantHost: "scholar.google.com",
|
||||
wantQuery: "deep learning",
|
||||
wantParams: map[string]string{
|
||||
"scisbd": "1",
|
||||
"as_ylo": "2019",
|
||||
"as_yhi": "2023",
|
||||
"as_vis": "1",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := buildGoogleScholarURL(tc.query, tc.max)
|
||||
got := buildGoogleScholarURL(tc.query, tc.topN, tc.sortBy, tc.yearLow, tc.yearHigh, tc.patents)
|
||||
u, err := url.Parse(got)
|
||||
if err != nil {
|
||||
t.Fatalf("url.Parse(%q): %v", got, err)
|
||||
@@ -105,6 +167,11 @@ func TestGoogleScholar_BuildURL(t *testing.T) {
|
||||
if q.Get("num") != tc.wantNum {
|
||||
t.Errorf("num = %q, want %q", q.Get("num"), tc.wantNum)
|
||||
}
|
||||
for k, v := range tc.wantParams {
|
||||
if q.Get(k) != v {
|
||||
t.Errorf("%s = %q, want %q", k, q.Get(k), v)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -123,7 +190,7 @@ func TestGoogleScholar_ParseResults(t *testing.T) {
|
||||
})
|
||||
tool := NewGoogleScholarToolWith(helper)
|
||||
out, err := tool.InvokableRun(context.Background(),
|
||||
`{"query":"transformer","max_results":5}`)
|
||||
`{"query":"transformer","top_n":5,"sort_by":"relevance","year_low":2020,"year_high":2024,"patents":true}`)
|
||||
if err != nil {
|
||||
t.Fatalf("InvokableRun: %v", err)
|
||||
}
|
||||
@@ -186,3 +253,48 @@ func TestGoogleScholar_Info(t *testing.T) {
|
||||
t.Errorf("Desc = %q, want to mention Scholar", info.Desc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleScholar_MergesNodeLevelDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
if q.Get("q") != "rag" {
|
||||
t.Errorf("q = %q, want rag", q.Get("q"))
|
||||
}
|
||||
if q.Get("num") != "7" {
|
||||
t.Errorf("num = %q, want 7", q.Get("num"))
|
||||
}
|
||||
if q.Get("scisbd") != "1" {
|
||||
t.Errorf("scisbd = %q, want 1", q.Get("scisbd"))
|
||||
}
|
||||
if q.Get("as_ylo") != "2020" {
|
||||
t.Errorf("as_ylo = %q, want 2020", q.Get("as_ylo"))
|
||||
}
|
||||
if q.Get("as_yhi") != "2024" {
|
||||
t.Errorf("as_yhi = %q, want 2024", q.Get("as_yhi"))
|
||||
}
|
||||
if q.Get("as_vis") != "1" {
|
||||
t.Errorf("as_vis = %q, want 1", q.Get("as_vis"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=UTF-8")
|
||||
_, _ = w.Write([]byte(cannedScholarHTML))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
helper := NewHTTPHelper().WithClient(&http.Client{
|
||||
Transport: rewriteHostTransport(srv.URL),
|
||||
})
|
||||
tool := NewGoogleScholarToolWithDefaults(helper, googleScholarParams{
|
||||
Query: "rag",
|
||||
TopN: 7,
|
||||
SortBy: "date",
|
||||
YearLow: 2020,
|
||||
YearHigh: 2024,
|
||||
Patents: boolPtr(false),
|
||||
})
|
||||
_, err := tool.InvokableRun(context.Background(), `{}`)
|
||||
if err != nil {
|
||||
t.Fatalf("InvokableRun with defaults: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,34 +29,35 @@ import (
|
||||
type Factory func(params map[string]any) (einotool.BaseTool, error)
|
||||
|
||||
var registry = map[string]Factory{
|
||||
"akshare": buildAkShareTool,
|
||||
"arxiv": noConfig("arxiv", func() einotool.BaseTool { return NewArxivTool() }),
|
||||
"bgpt": noConfig("bgpt", func() einotool.BaseTool { return NewBGPTTool() }),
|
||||
"code_exec": noConfig("code_exec", func() einotool.BaseTool { return NewCodeExecTool() }),
|
||||
"crawler": noConfig("crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
|
||||
"deepl": noConfig("deepl", func() einotool.BaseTool { return NewDeepLTool() }),
|
||||
"duckduckgo": noConfig("duckduckgo", func() einotool.BaseTool { return NewDuckDuckGoTool() }),
|
||||
"email": noConfig("email", func() einotool.BaseTool { return NewEmailTool() }),
|
||||
"execute_sql": buildExeSQLTool,
|
||||
"exesql": buildExeSQLTool,
|
||||
"github": noConfig("github", func() einotool.BaseTool { return NewGitHubTool() }),
|
||||
"google": buildGoogleTool,
|
||||
"google_scholar": noConfig("google_scholar", func() einotool.BaseTool { return NewGoogleScholarTool() }),
|
||||
"jin10": noConfig("jin10", func() einotool.BaseTool { return NewJin10Tool() }),
|
||||
"keenable": buildKeenableTool,
|
||||
"pubmed": noConfig("pubmed", func() einotool.BaseTool { return NewPubMedTool() }),
|
||||
"qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }),
|
||||
"retrieval": noConfig("retrieval", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"search_my_dataset": noConfig("search_my_dataset", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"search_my_dateset": noConfig("search_my_dateset", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"searxng": noConfig("searxng", func() einotool.BaseTool { return NewSearXNGTool() }),
|
||||
"tavily": noConfig("tavily", func() einotool.BaseTool { return NewTavilyTool() }),
|
||||
"tavily_extract": noConfig("tavily_extract", func() einotool.BaseTool { return NewTavilyExtractTool() }),
|
||||
"tushare": noConfig("tushare", func() einotool.BaseTool { return NewTushareTool() }),
|
||||
"wencai": noConfig("wencai", func() einotool.BaseTool { return NewWencaiTool() }),
|
||||
"web_crawler": noConfig("web_crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
|
||||
"wikipedia": noConfig("wikipedia", func() einotool.BaseTool { return NewWikipediaTool() }),
|
||||
"yahoo_finance": noConfig("yahoo_finance", func() einotool.BaseTool { return NewYahooFinanceTool() }),
|
||||
"akshare": buildAkShareTool,
|
||||
"arxiv": noConfig("arxiv", func() einotool.BaseTool { return NewArxivTool() }),
|
||||
"bgpt": noConfig("bgpt", func() einotool.BaseTool { return NewBGPTTool() }),
|
||||
"code_exec": noConfig("code_exec", func() einotool.BaseTool { return NewCodeExecTool() }),
|
||||
"crawler": noConfig("crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
|
||||
"deepl": noConfig("deepl", func() einotool.BaseTool { return NewDeepLTool() }),
|
||||
"duckduckgo": noConfig("duckduckgo", func() einotool.BaseTool { return NewDuckDuckGoTool() }),
|
||||
"email": noConfig("email", func() einotool.BaseTool { return NewEmailTool() }),
|
||||
"execute_sql": buildExeSQLTool,
|
||||
"exesql": buildExeSQLTool,
|
||||
"github": noConfig("github", func() einotool.BaseTool { return NewGitHubTool() }),
|
||||
"google": buildGoogleTool,
|
||||
"google_scholar": buildGoogleScholarTool,
|
||||
"google_scholar_search": buildGoogleScholarTool,
|
||||
"jin10": noConfig("jin10", func() einotool.BaseTool { return NewJin10Tool() }),
|
||||
"keenable": buildKeenableTool,
|
||||
"pubmed": noConfig("pubmed", func() einotool.BaseTool { return NewPubMedTool() }),
|
||||
"qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }),
|
||||
"retrieval": noConfig("retrieval", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"search_my_dataset": noConfig("search_my_dataset", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"search_my_dateset": noConfig("search_my_dateset", func() einotool.BaseTool { return NewRetrievalTool() }),
|
||||
"searxng": noConfig("searxng", func() einotool.BaseTool { return NewSearXNGTool() }),
|
||||
"tavily": noConfig("tavily", func() einotool.BaseTool { return NewTavilyTool() }),
|
||||
"tavily_extract": noConfig("tavily_extract", func() einotool.BaseTool { return NewTavilyExtractTool() }),
|
||||
"tushare": noConfig("tushare", func() einotool.BaseTool { return NewTushareTool() }),
|
||||
"wencai": noConfig("wencai", func() einotool.BaseTool { return NewWencaiTool() }),
|
||||
"web_crawler": noConfig("web_crawler", func() einotool.BaseTool { return NewCrawlerTool() }),
|
||||
"wikipedia": noConfig("wikipedia", func() einotool.BaseTool { return NewWikipediaTool() }),
|
||||
"yahoo_finance": noConfig("yahoo_finance", func() einotool.BaseTool { return NewYahooFinanceTool() }),
|
||||
}
|
||||
|
||||
func noConfig(name string, fn func() einotool.BaseTool) Factory {
|
||||
@@ -167,6 +168,39 @@ func buildGoogleTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
return NewGoogleToolWithDefaults(nil, defaults), nil
|
||||
}
|
||||
|
||||
func buildGoogleScholarTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
if len(params) == 0 {
|
||||
return NewGoogleScholarTool(), nil
|
||||
}
|
||||
for key := range params {
|
||||
switch key {
|
||||
case "query", "top_n", "sort_by", "year_low", "year_high", "patents":
|
||||
default:
|
||||
return nil, fmt.Errorf("agent tool: tool %q does not accept node-level param %s", "google_scholar", key)
|
||||
}
|
||||
}
|
||||
defaults := googleScholarParams{}
|
||||
if v, ok := stringParam(params, "query"); ok {
|
||||
defaults.Query = v
|
||||
}
|
||||
if v, ok := intParam(params, "top_n"); ok {
|
||||
defaults.TopN = v
|
||||
}
|
||||
if v, ok := stringParam(params, "sort_by"); ok {
|
||||
defaults.SortBy = v
|
||||
}
|
||||
if v, ok := intParam(params, "year_low"); ok {
|
||||
defaults.YearLow = v
|
||||
}
|
||||
if v, ok := intParam(params, "year_high"); ok {
|
||||
defaults.YearHigh = v
|
||||
}
|
||||
if v, ok := boolParam(params, "patents"); ok {
|
||||
defaults.Patents = &v
|
||||
}
|
||||
return NewGoogleScholarToolWithDefaults(nil, defaults), nil
|
||||
}
|
||||
|
||||
func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
if len(params) == 0 {
|
||||
return NewKeenableTool(), nil
|
||||
@@ -245,3 +279,12 @@ func intParam(params map[string]any, key string) (int, bool) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func boolParam(params map[string]any, key string) (bool, bool) {
|
||||
v, ok := params[key]
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
b, ok := v.(bool)
|
||||
return b, ok
|
||||
}
|
||||
|
||||
@@ -41,11 +41,11 @@ func TestBuildAll_UnknownTool(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildAll_AllRegisteredTools(t *testing.T) {
|
||||
// Every key in registry (28 entries, 24 unique canonical tools).
|
||||
// Every key in registry.
|
||||
names := []string{
|
||||
"akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl",
|
||||
"duckduckgo", "email", "exesql", "execute_sql", "github", "google",
|
||||
"google_scholar", "jin10", "keenable", "pubmed", "qweather",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
|
||||
"yahoo_finance",
|
||||
@@ -116,13 +116,11 @@ func TestBuildAll_KeenableRejectsEmptyNodeAPIKey(t *testing.T) {
|
||||
func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Every entry the registry advertises. 28 names, 24 unique
|
||||
// canonical tools (execute_sql == exesql, retrieval ==
|
||||
// search_my_dataset == search_my_dateset, crawler == web_crawler).
|
||||
// Every entry the registry advertises.
|
||||
names := []string{
|
||||
"akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl",
|
||||
"duckduckgo", "email", "execute_sql", "exesql", "github", "google",
|
||||
"google_scholar", "jin10", "keenable", "pubmed", "qweather",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia",
|
||||
"yahoo_finance",
|
||||
@@ -181,13 +179,15 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
// search_my_dateset and crawler/web_crawler. A bug here would mean
|
||||
// an alias was accidentally pointed at a different tool.
|
||||
canonicalByAlias := map[string]string{
|
||||
"execute_sql": "execute_sql",
|
||||
"exesql": "execute_sql",
|
||||
"retrieval": "search_my_dateset",
|
||||
"search_my_dataset": "search_my_dateset",
|
||||
"search_my_dateset": "search_my_dateset",
|
||||
"crawler": "web_crawler",
|
||||
"web_crawler": "web_crawler",
|
||||
"execute_sql": "execute_sql",
|
||||
"exesql": "execute_sql",
|
||||
"google_scholar": "google_scholar",
|
||||
"google_scholar_search": "google_scholar",
|
||||
"retrieval": "search_my_dateset",
|
||||
"search_my_dataset": "search_my_dateset",
|
||||
"search_my_dateset": "search_my_dateset",
|
||||
"crawler": "web_crawler",
|
||||
"web_crawler": "web_crawler",
|
||||
}
|
||||
for _, name := range names {
|
||||
canonical, ok := canonicalByAlias[name]
|
||||
|
||||
Reference in New Issue
Block a user