mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-14 20:54:30 +08:00
feat(agent): add Querit Contents tool (#18156)
### Summary Add Querit Contents as a built-in page content tool for RAGFlow Agents and Canvas workflows. Querit Contents crawls one or more known URLs and returns their page content and optional metadata. It complements the existing Querit Search tool: Search discovers relevant pages, while Contents retrieves pages already selected by an Agent or workflow. This integration supports two usage modes: - A standalone `QueritContents` node in Canvas workflows. - An embedded content tool available to RAGFlow Agents.
This commit is contained in:
@@ -134,6 +134,7 @@ var toolComponentRegistrations = []struct {
|
||||
{componentName: "GoogleScholar", toolName: "google_scholar"},
|
||||
{componentName: "KeenableSearch", toolName: "keenable"},
|
||||
{componentName: "PubMed", toolName: "pubmed"},
|
||||
{componentName: "QueritContents", toolName: "querit_contents"},
|
||||
{componentName: "QueritSearch", toolName: "querit_search"},
|
||||
{componentName: "SearXNG", toolName: "searxng"},
|
||||
{componentName: "TavilySearch", toolName: "tavily"},
|
||||
|
||||
@@ -193,6 +193,13 @@ func TestToolBackedComponentRegisteredFactories(t *testing.T) {
|
||||
outputKey: "success",
|
||||
inputKey: "to_email",
|
||||
},
|
||||
{
|
||||
name: "QueritContents",
|
||||
toolName: "QueritContents",
|
||||
params: map[string]any{"api_key": "stored-key", "format": "markdown", "crawl_timeout": float64(10), "extras_meta": true, "outputs": map[string]any{"json": map[string]any{}}},
|
||||
outputKey: "json",
|
||||
inputKey: "urls",
|
||||
},
|
||||
{
|
||||
name: "QueritSearch",
|
||||
toolName: "QueritSearch",
|
||||
@@ -283,7 +290,7 @@ func TestToolBackedComponentWenCaiInvoke(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestToolBackedComponentRegisteredBuildWorkflow(t *testing.T) {
|
||||
for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritSearch", "SearXNG", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} {
|
||||
for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "SearXNG", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} {
|
||||
t.Run(componentName, func(t *testing.T) {
|
||||
c := &canvas.Canvas{
|
||||
Components: map[string]canvas.CanvasComponent{
|
||||
|
||||
@@ -248,44 +248,58 @@ func (q *QueritTool) InvokableRun(ctx context.Context, argsJSON string, _ ...too
|
||||
if err != nil {
|
||||
return queritErrorJSON(fmt.Errorf("encode request: %w", err), apiKey), nil
|
||||
}
|
||||
raw, requestErr := doQueritRequest(ctx, q.helper, q.retryWait, queritEndpoint, body, apiKey)
|
||||
if requestErr != nil {
|
||||
return queritErrorJSON(requestErr, apiKey), nil
|
||||
}
|
||||
if _, err := decodeQueritResponse(raw); err != nil {
|
||||
return queritErrorJSON(err, apiKey), nil
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func doQueritRequest(
|
||||
ctx context.Context,
|
||||
helper *HTTPHelper,
|
||||
retryWait func(context.Context, int) bool,
|
||||
endpoint string,
|
||||
body []byte,
|
||||
apiKey string,
|
||||
) ([]byte, error) {
|
||||
for attempt := 1; attempt <= queritMaxAttempts; attempt++ {
|
||||
resp, requestErr := q.helper.Do(
|
||||
resp, err := helper.Do(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
queritEndpoint,
|
||||
endpoint,
|
||||
string(body),
|
||||
"application/json",
|
||||
map[string]string{"Authorization": "Bearer " + apiKey},
|
||||
)
|
||||
if requestErr != nil {
|
||||
return queritErrorJSON(fmt.Errorf("request failed: %w", requestErr), apiKey), nil
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if attempt == queritMaxAttempts || !q.retryWait(ctx, attempt) {
|
||||
return queritErrorJSON(fmt.Errorf("upstream returned %d after %d attempts", resp.StatusCode, attempt), apiKey), nil
|
||||
if attempt == queritMaxAttempts || !retryWait(ctx, attempt) {
|
||||
return nil, fmt.Errorf("upstream returned %d after %d attempts", resp.StatusCode, attempt)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return queritErrorJSON(fmt.Errorf("upstream returned %d", resp.StatusCode), apiKey), nil
|
||||
return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
raw, readErr := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if readErr != nil {
|
||||
return queritErrorJSON(fmt.Errorf("read response: %w", readErr), apiKey), nil
|
||||
return nil, fmt.Errorf("read response: %w", readErr)
|
||||
}
|
||||
if _, err := decodeQueritResponse(raw); err != nil {
|
||||
return queritErrorJSON(err, apiKey), nil
|
||||
}
|
||||
return string(raw), nil
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
return queritErrorJSON(fmt.Errorf("request exhausted retries"), apiKey), nil
|
||||
return nil, fmt.Errorf("request exhausted retries")
|
||||
}
|
||||
|
||||
func mergeQueritParams(defaults, runtimeParams queritParams, provided map[string]json.RawMessage) queritParams {
|
||||
@@ -360,21 +374,9 @@ func buildQueritRequest(params queritParams) queritRequest {
|
||||
}
|
||||
|
||||
func decodeQueritResponse(raw []byte) (map[string]any, error) {
|
||||
var decoded any
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
if err == nil {
|
||||
return nil, fmt.Errorf("decode response: expected exactly one JSON value")
|
||||
}
|
||||
return nil, fmt.Errorf("decode response: trailing content: %w", err)
|
||||
}
|
||||
response, ok := decoded.(map[string]any)
|
||||
if !ok || response == nil {
|
||||
return nil, fmt.Errorf("decode response: expected a JSON object")
|
||||
response, err := decodeQueritJSONObject(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resultsValue, exists := response["results"]
|
||||
if !exists {
|
||||
@@ -394,6 +396,26 @@ func decodeQueritResponse(raw []byte) (map[string]any, error) {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func decodeQueritJSONObject(raw []byte) (map[string]any, error) {
|
||||
var decoded any
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
if err == nil {
|
||||
return nil, fmt.Errorf("decode response: expected exactly one JSON value")
|
||||
}
|
||||
return nil, fmt.Errorf("decode response: trailing content: %w", err)
|
||||
}
|
||||
response, ok := decoded.(map[string]any)
|
||||
if !ok || response == nil {
|
||||
return nil, fmt.Errorf("decode response: expected a JSON object")
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func waitForQueritRetry(ctx context.Context, attempt int) bool {
|
||||
delay := 200 * time.Millisecond
|
||||
for current := 1; current < attempt; current++ {
|
||||
@@ -588,9 +610,13 @@ func queritStringSlice(value any) ([]string, bool) {
|
||||
}
|
||||
|
||||
func queritErrorJSON(err error, apiKeys ...string) string {
|
||||
message := "querit_search: unknown error"
|
||||
return queritToolErrorJSON(queritToolName, err, apiKeys...)
|
||||
}
|
||||
|
||||
func queritToolErrorJSON(toolName string, err error, apiKeys ...string) string {
|
||||
message := toolName + ": unknown error"
|
||||
if err != nil {
|
||||
message = "querit_search: " + err.Error()
|
||||
message = toolName + ": " + err.Error()
|
||||
}
|
||||
for _, apiKey := range apiKeys {
|
||||
if apiKey = strings.TrimSpace(apiKey); apiKey != "" {
|
||||
@@ -599,7 +625,7 @@ func queritErrorJSON(err error, apiKeys ...string) string {
|
||||
}
|
||||
raw, marshalErr := json.Marshal(map[string]any{"_ERROR": message})
|
||||
if marshalErr != nil {
|
||||
return `{"_ERROR":"querit_search: marshal error"}`
|
||||
return fmt.Sprintf(`{"_ERROR":%q}`, toolName+": marshal error")
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
271
internal/agent/tool/querit_contents.go
Normal file
271
internal/agent/tool/querit_contents.go
Normal file
@@ -0,0 +1,271 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package tool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/components/tool"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
queritContentsToolName = "querit_contents"
|
||||
queritContentsEndpoint = "https://api.querit.ai/v1/contents"
|
||||
)
|
||||
|
||||
type queritContentsParams struct {
|
||||
APIKey string `json:"api_key"`
|
||||
URLs any `json:"urls"`
|
||||
Format string `json:"format"`
|
||||
CrawlTimeout int `json:"crawl_timeout"`
|
||||
ExtrasMeta bool `json:"extras_meta"`
|
||||
}
|
||||
|
||||
type queritContentsRequest struct {
|
||||
URLs []string `json:"urls"`
|
||||
Format string `json:"format"`
|
||||
CrawlTimeout int `json:"crawlTimeout"`
|
||||
ExtrasMeta bool `json:"extrasMeta"`
|
||||
}
|
||||
|
||||
// QueritContentsTool crawls public web pages through the Querit Contents API.
|
||||
type QueritContentsTool struct {
|
||||
helper *HTTPHelper
|
||||
envKey func() string
|
||||
defaults queritContentsParams
|
||||
retryWait func(context.Context, int) bool
|
||||
}
|
||||
|
||||
var _ ToolComponent = (*QueritContentsTool)(nil)
|
||||
|
||||
func NewQueritContentsTool() *QueritContentsTool {
|
||||
return newQueritContentsTool(nil, nil, queritContentsParams{}, nil)
|
||||
}
|
||||
|
||||
func NewQueritContentsToolWith(helper *HTTPHelper) *QueritContentsTool {
|
||||
return newQueritContentsTool(helper, nil, queritContentsParams{}, nil)
|
||||
}
|
||||
|
||||
func NewQueritContentsToolWithEnvKey(helper *HTTPHelper, envKey func() string) *QueritContentsTool {
|
||||
return newQueritContentsTool(helper, envKey, queritContentsParams{}, nil)
|
||||
}
|
||||
|
||||
func newQueritContentsTool(
|
||||
helper *HTTPHelper,
|
||||
envKey func() string,
|
||||
defaults queritContentsParams,
|
||||
retryWait func(context.Context, int) bool,
|
||||
) *QueritContentsTool {
|
||||
if helper == nil {
|
||||
helper = NewHTTPHelper()
|
||||
helper.client.Timeout = 65 * time.Second
|
||||
}
|
||||
if envKey == nil {
|
||||
envKey = defaultQueritEnvKey
|
||||
}
|
||||
if defaults.Format == "" {
|
||||
defaults.Format = "markdown"
|
||||
}
|
||||
if defaults.CrawlTimeout == 0 {
|
||||
defaults.CrawlTimeout = 10
|
||||
}
|
||||
if retryWait == nil {
|
||||
retryWait = waitForQueritRetry
|
||||
}
|
||||
return &QueritContentsTool{helper: helper, envKey: envKey, defaults: defaults, retryWait: retryWait}
|
||||
}
|
||||
|
||||
func (q *QueritContentsTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
return &schema.ToolInfo{
|
||||
Name: queritContentsToolName,
|
||||
Desc: "Crawl one or more web pages with Querit and return their contents.",
|
||||
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
|
||||
"urls": {
|
||||
Type: schema.Array,
|
||||
ElemInfo: &schema.ParameterInfo{Type: schema.String},
|
||||
Desc: "One to ten absolute HTTP or HTTPS URLs to crawl.",
|
||||
Required: true,
|
||||
},
|
||||
"format": {
|
||||
Type: schema.String,
|
||||
Desc: `Content format: "text", "markdown" (default), or "html".`,
|
||||
Required: false,
|
||||
},
|
||||
"crawl_timeout": {
|
||||
Type: schema.Integer,
|
||||
Desc: "Per-page crawl timeout in seconds. Defaults to 10 and must be between 1 and 60.",
|
||||
Required: false,
|
||||
},
|
||||
"extras_meta": {
|
||||
Type: schema.Boolean,
|
||||
Desc: "Whether to include page metadata in each result.",
|
||||
Required: false,
|
||||
},
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *QueritContentsTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) {
|
||||
provided := make(map[string]json.RawMessage)
|
||||
if err := json.Unmarshal([]byte(argsJSON), &provided); err != nil {
|
||||
return queritContentsErrorJSON(fmt.Errorf("parse arguments: %w", err)), nil
|
||||
}
|
||||
var runtimeParams queritContentsParams
|
||||
if err := json.Unmarshal([]byte(argsJSON), &runtimeParams); err != nil {
|
||||
return queritContentsErrorJSON(fmt.Errorf("parse arguments: %w", err)), nil
|
||||
}
|
||||
params := mergeQueritContentsParams(q.defaults, runtimeParams, provided)
|
||||
urls := normalizeQueritContentURLs(params.URLs)
|
||||
if err := validateQueritContentsParams(params, urls); err != nil {
|
||||
return queritContentsErrorJSON(err, params.APIKey), nil
|
||||
}
|
||||
|
||||
apiKey := strings.TrimSpace(params.APIKey)
|
||||
if apiKey == "" {
|
||||
apiKey = strings.TrimSpace(q.envKey())
|
||||
}
|
||||
if apiKey == "" {
|
||||
return queritContentsErrorJSON(fmt.Errorf("api_key is required (or set QUERIT_API_KEY)")), nil
|
||||
}
|
||||
|
||||
body, err := json.Marshal(queritContentsRequest{
|
||||
URLs: urls,
|
||||
Format: params.Format,
|
||||
CrawlTimeout: params.CrawlTimeout,
|
||||
ExtrasMeta: params.ExtrasMeta,
|
||||
})
|
||||
if err != nil {
|
||||
return queritContentsErrorJSON(fmt.Errorf("encode request: %w", err), apiKey), nil
|
||||
}
|
||||
raw, requestErr := doQueritRequest(ctx, q.helper, q.retryWait, queritContentsEndpoint, body, apiKey)
|
||||
if requestErr != nil {
|
||||
return queritContentsErrorJSON(requestErr, apiKey), nil
|
||||
}
|
||||
if _, err := decodeQueritContentsResponse(raw); err != nil {
|
||||
return queritContentsErrorJSON(err, apiKey), nil
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func mergeQueritContentsParams(
|
||||
defaults queritContentsParams,
|
||||
runtimeParams queritContentsParams,
|
||||
provided map[string]json.RawMessage,
|
||||
) queritContentsParams {
|
||||
merged := defaults
|
||||
if _, ok := provided["urls"]; ok {
|
||||
merged.URLs = runtimeParams.URLs
|
||||
}
|
||||
if _, ok := provided["format"]; ok {
|
||||
merged.Format = runtimeParams.Format
|
||||
}
|
||||
if _, ok := provided["crawl_timeout"]; ok {
|
||||
merged.CrawlTimeout = runtimeParams.CrawlTimeout
|
||||
}
|
||||
if _, ok := provided["extras_meta"]; ok {
|
||||
merged.ExtrasMeta = runtimeParams.ExtrasMeta
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func validateQueritContentsParams(params queritContentsParams, urls []string) error {
|
||||
if len(urls) < 1 || len(urls) > 10 {
|
||||
return fmt.Errorf("urls must contain between 1 and 10 values")
|
||||
}
|
||||
for _, rawURL := range urls {
|
||||
parsed, err := url.ParseRequestURI(rawURL)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||||
return fmt.Errorf("urls must be absolute HTTP or HTTPS URLs")
|
||||
}
|
||||
}
|
||||
if params.Format != "text" && params.Format != "markdown" && params.Format != "html" {
|
||||
return fmt.Errorf("format must be text, markdown, or html")
|
||||
}
|
||||
if params.CrawlTimeout < 1 || params.CrawlTimeout > 60 {
|
||||
return fmt.Errorf("crawl_timeout must be between 1 and 60")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeQueritContentURLs(raw any) []string {
|
||||
switch value := raw.(type) {
|
||||
case string:
|
||||
return compactStrings(strings.Split(value, ","))
|
||||
case []string:
|
||||
return compactStrings(value)
|
||||
case []any:
|
||||
urls := make([]string, 0, len(value))
|
||||
for _, item := range value {
|
||||
text, ok := item.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
urls = append(urls, text)
|
||||
}
|
||||
return compactStrings(urls)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func decodeQueritContentsResponse(raw []byte) (map[string]any, error) {
|
||||
response, err := decodeQueritJSONObject(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, field := range []string{"results", "statuses"} {
|
||||
if value, exists := response[field]; exists {
|
||||
if _, ok := value.([]any); !ok {
|
||||
return nil, fmt.Errorf("decode response: %s must be a JSON array", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (q *QueritContentsTool) ComponentSpec() ComponentSpec {
|
||||
return ComponentSpec{
|
||||
PreserveJSONNumbers: true,
|
||||
Inputs: map[string]string{
|
||||
"api_key": "Querit API key. Uses QUERIT_API_KEY when empty.",
|
||||
"urls": "One to ten absolute HTTP or HTTPS URLs to crawl.",
|
||||
"format": `Content format: "text", "markdown", or "html".`,
|
||||
"crawl_timeout": "Per-page crawl timeout in seconds.",
|
||||
"extras_meta": "Whether to include page metadata.",
|
||||
},
|
||||
Outputs: map[string]string{
|
||||
"json": "Complete raw Querit Contents JSON response.",
|
||||
},
|
||||
InputForm: map[string]any{
|
||||
"urls": map[string]any{"name": "URLs", "type": "line"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (q *QueritContentsTool) BuildComponentOutputs(response map[string]any) map[string]any {
|
||||
return map[string]any{"json": response}
|
||||
}
|
||||
|
||||
func queritContentsErrorJSON(err error, apiKeys ...string) string {
|
||||
return queritToolErrorJSON(queritContentsToolName, err, apiKeys...)
|
||||
}
|
||||
188
internal/agent/tool/querit_contents_test.go
Normal file
188
internal/agent/tool/querit_contents_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package tool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueritContentsBuildsRequestAndPreservesResponse(t *testing.T) {
|
||||
var gotMethod, gotPath, gotAuthorization string
|
||||
var gotBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
gotMethod = request.Method
|
||||
gotPath = request.URL.Path
|
||||
gotAuthorization = request.Header.Get("Authorization")
|
||||
_ = json.NewDecoder(request.Body).Decode(&gotBody)
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = writer.Write([]byte(`{"error_code":0,"search_id":"crawl-1","results":[{"id":"1","url":"https://example.com","content":"# Example"}],"statuses":[{"id":"1","status":"success"}],"searchTime":1}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)})
|
||||
contents := newQueritContentsTool(helper, func() string { return "" }, queritContentsParams{APIKey: "key-test"}, nil)
|
||||
out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"],"format":"html","crawl_timeout":20,"extras_meta":true}`)
|
||||
if err != nil {
|
||||
t.Fatalf("InvokableRun: %v", err)
|
||||
}
|
||||
if gotMethod != http.MethodPost || gotPath != "/v1/contents" {
|
||||
t.Fatalf("request = %s %s, want POST /v1/contents", gotMethod, gotPath)
|
||||
}
|
||||
if gotAuthorization != "Bearer key-test" {
|
||||
t.Fatalf("Authorization = %q", gotAuthorization)
|
||||
}
|
||||
if gotBody["format"] != "html" || gotBody["crawlTimeout"] != float64(20) || gotBody["extrasMeta"] != true {
|
||||
t.Fatalf("request body = %#v", gotBody)
|
||||
}
|
||||
urls, ok := gotBody["urls"].([]any)
|
||||
if !ok || len(urls) != 1 || urls[0] != "https://example.com" {
|
||||
t.Fatalf("urls = %#v", gotBody["urls"])
|
||||
}
|
||||
if !strings.Contains(out, `"search_id":"crawl-1"`) || !strings.Contains(out, `"statuses"`) {
|
||||
t.Fatalf("complete response was not retained: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueritContentsMergesDefaultsAndExplicitFalse(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
_ = json.NewDecoder(request.Body).Decode(&gotBody)
|
||||
_, _ = writer.Write([]byte(`{"results":[],"statuses":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)})
|
||||
contents := newQueritContentsTool(helper, func() string { return "" }, queritContentsParams{
|
||||
APIKey: "stored-key",
|
||||
URLs: []string{"https://stored.example"},
|
||||
Format: "text",
|
||||
CrawlTimeout: 30,
|
||||
ExtrasMeta: true,
|
||||
}, nil)
|
||||
_, err := contents.InvokableRun(context.Background(), `{"urls":"https://runtime.example","extras_meta":false}`)
|
||||
if err != nil {
|
||||
t.Fatalf("InvokableRun: %v", err)
|
||||
}
|
||||
if gotBody["format"] != "text" || gotBody["crawlTimeout"] != float64(30) || gotBody["extrasMeta"] != false {
|
||||
t.Fatalf("merged defaults = %#v", gotBody)
|
||||
}
|
||||
if gotBody["urls"].([]any)[0] != "https://runtime.example" {
|
||||
t.Fatalf("runtime urls = %#v", gotBody["urls"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueritContentsValidatesInputsAndAPIKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args string
|
||||
want string
|
||||
}{
|
||||
{name: "missing urls", args: `{}`, want: "urls must contain"},
|
||||
{name: "too many urls", args: `{"urls":["https://1.example","https://2.example","https://3.example","https://4.example","https://5.example","https://6.example","https://7.example","https://8.example","https://9.example","https://10.example","https://11.example"]}`, want: "between 1 and 10"},
|
||||
{name: "relative url", args: `{"urls":["example.com"]}`, want: "absolute HTTP or HTTPS"},
|
||||
{name: "unsupported scheme", args: `{"urls":["file:///tmp/page"]}`, want: "absolute HTTP or HTTPS"},
|
||||
{name: "bad format", args: `{"urls":["https://example.com"],"format":"xml"}`, want: "format must be"},
|
||||
{name: "bad timeout", args: `{"urls":["https://example.com"],"crawl_timeout":61}`, want: "between 1 and 60"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
contents := NewQueritContentsToolWithEnvKey(NewHTTPHelper(), func() string { return "key-test" })
|
||||
out, err := contents.InvokableRun(context.Background(), test.args)
|
||||
if err != nil || !strings.Contains(out, "_ERROR") || !strings.Contains(out, test.want) {
|
||||
t.Fatalf("result = %s, err = %v", out, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
contents := NewQueritContentsToolWithEnvKey(NewHTTPHelper(), func() string { return "" })
|
||||
out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`)
|
||||
if err != nil || !strings.Contains(out, "api_key is required") {
|
||||
t.Fatalf("missing key result = %s, err = %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueritContentsRejectsMalformedResponses(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
{name: "top-level array", body: `[]`, want: "JSON object"},
|
||||
{name: "results object", body: `{"results":{}}`, want: "results must be a JSON array"},
|
||||
{name: "statuses object", body: `{"results":[],"statuses":{}}`, want: "statuses must be a JSON array"},
|
||||
{name: "trailing content", body: `{"results":[]} trailing`, want: "trailing content"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write([]byte(test.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)})
|
||||
contents := NewQueritContentsToolWithEnvKey(helper, func() string { return "key-test" })
|
||||
out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`)
|
||||
if err != nil || !strings.Contains(out, test.want) {
|
||||
t.Fatalf("result = %s, err = %v", out, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueritContentsRedactsAPIKey(t *testing.T) {
|
||||
const secret = "secret-contents-key"
|
||||
helper := NewHTTPHelper().WithClient(&http.Client{Transport: roundTripperErrorFunc(func(*http.Request) error {
|
||||
return errors.New("failed with " + secret)
|
||||
})})
|
||||
contents := NewQueritContentsToolWithEnvKey(helper, func() string { return secret })
|
||||
out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`)
|
||||
if err != nil || strings.Contains(out, secret) || !strings.Contains(out, "[REDACTED]") {
|
||||
t.Fatalf("result = %s, err = %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueritContentsInfoAndComponentContract(t *testing.T) {
|
||||
contents := NewQueritContentsTool()
|
||||
if contents.helper.client.Timeout != 65*time.Second {
|
||||
t.Fatalf("HTTP timeout = %s, want 65s", contents.helper.client.Timeout)
|
||||
}
|
||||
info, err := contents.Info(context.Background())
|
||||
if err != nil || info.Name != queritContentsToolName || info.ParamsOneOf == nil {
|
||||
t.Fatalf("Info = %#v, %v", info, err)
|
||||
}
|
||||
encoded, _ := json.Marshal(info)
|
||||
if strings.Contains(string(encoded), "api_key") {
|
||||
t.Fatalf("Info exposed API key: %s", encoded)
|
||||
}
|
||||
spec := contents.ComponentSpec()
|
||||
if spec.Inputs["urls"] == "" || spec.Outputs["json"] == "" || !spec.PreserveJSONNumbers {
|
||||
t.Fatalf("ComponentSpec = %#v", spec)
|
||||
}
|
||||
if len(spec.InputForm) != 1 || spec.InputForm["urls"] == nil {
|
||||
t.Fatalf("InputForm = %#v, want URLs only", spec.InputForm)
|
||||
}
|
||||
response := map[string]any{"search_id": "crawl-1", "results": []any{map[string]any{"content": "page"}}}
|
||||
outputs := contents.BuildComponentOutputs(response)
|
||||
if outputs["json"].(map[string]any)["search_id"] != "crawl-1" {
|
||||
t.Fatalf("outputs = %#v", outputs)
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,7 @@ var registry = map[string]Factory{
|
||||
"pubmed": buildPubMedTool,
|
||||
"qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }),
|
||||
"querit": buildQueritTool,
|
||||
"querit_contents": buildQueritContentsTool,
|
||||
"querit_search": buildQueritTool,
|
||||
"retrieval": buildRetrievalTool,
|
||||
"search_my_dataset": buildRetrievalTool,
|
||||
@@ -78,6 +79,7 @@ var canvasToolNames = map[string]string{
|
||||
"codeexec": "code_exec",
|
||||
"googlescholar": "google_scholar",
|
||||
"keenablesearch": "keenable",
|
||||
"queritcontents": "querit_contents",
|
||||
"queritsearch": "querit_search",
|
||||
"tavilyextract": "tavily_extract",
|
||||
"tavilysearch": "tavily",
|
||||
@@ -657,6 +659,42 @@ func buildQueritTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
return newQueritTool(nil, nil, defaults, nil), nil
|
||||
}
|
||||
|
||||
func buildQueritContentsTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
defaults := queritContentsParams{}
|
||||
if value, exists := params["api_key"]; exists {
|
||||
apiKey, valid := value.(string)
|
||||
if !valid {
|
||||
return nil, fmt.Errorf("agent tool: tool %q requires string node-level param api_key", queritContentsToolName)
|
||||
}
|
||||
defaults.APIKey = apiKey
|
||||
}
|
||||
if value, exists := params["urls"]; exists {
|
||||
defaults.URLs = value
|
||||
}
|
||||
if value, exists := params["format"]; exists {
|
||||
format, valid := value.(string)
|
||||
if !valid || (format != "text" && format != "markdown" && format != "html") {
|
||||
return nil, fmt.Errorf("agent tool: tool %q has unsupported format %q", queritContentsToolName, format)
|
||||
}
|
||||
defaults.Format = format
|
||||
}
|
||||
if value, exists := params["crawl_timeout"]; exists {
|
||||
crawlTimeout, valid := strictInt(value)
|
||||
if !valid || crawlTimeout < 1 || crawlTimeout > 60 {
|
||||
return nil, fmt.Errorf("agent tool: tool %q requires integer node-level param crawl_timeout within [1, 60]", queritContentsToolName)
|
||||
}
|
||||
defaults.CrawlTimeout = crawlTimeout
|
||||
}
|
||||
if value, exists := params["extras_meta"]; exists {
|
||||
extrasMeta, valid := value.(bool)
|
||||
if !valid {
|
||||
return nil, fmt.Errorf("agent tool: tool %q requires boolean node-level param extras_meta", queritContentsToolName)
|
||||
}
|
||||
defaults.ExtrasMeta = extrasMeta
|
||||
}
|
||||
return newQueritContentsTool(nil, nil, defaults, nil), nil
|
||||
}
|
||||
|
||||
func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {
|
||||
defaults := keenableParams{}
|
||||
apiKey := ""
|
||||
|
||||
@@ -48,6 +48,7 @@ func TestBuildByName_CanvasComponentNames(t *testing.T) {
|
||||
{name: "CodeExec", wantToolName: "execute_code"},
|
||||
{name: "GoogleScholar", wantToolName: "google_scholar_search"},
|
||||
{name: "KeenableSearch", wantToolName: "keenable_search"},
|
||||
{name: "QueritContents", wantToolName: "querit_contents"},
|
||||
{name: "QueritSearch", wantToolName: "querit_search"},
|
||||
{name: "TavilyExtract", wantToolName: "tavily_extract"},
|
||||
{name: "TavilySearch", wantToolName: "tavily_search"},
|
||||
@@ -134,13 +135,34 @@ func TestBuildByName_QueritAliases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildByName_QueritContentsAliases(t *testing.T) {
|
||||
for _, name := range []string{"querit_contents", "queritcontents", "QueritContents"} {
|
||||
built, err := BuildByName(name, map[string]any{
|
||||
"api_key": "stored-key",
|
||||
"format": "html",
|
||||
"crawl_timeout": float64(20),
|
||||
"extras_meta": true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildByName(%q): %v", name, err)
|
||||
}
|
||||
contents, ok := built.(*QueritContentsTool)
|
||||
if !ok {
|
||||
t.Fatalf("BuildByName(%q) returned %T, want *QueritContentsTool", name, built)
|
||||
}
|
||||
if contents.defaults.Format != "html" || contents.defaults.CrawlTimeout != 20 || !contents.defaults.ExtrasMeta {
|
||||
t.Fatalf("BuildByName(%q) defaults = %#v", name, contents.defaults)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAll_AllRegisteredTools(t *testing.T) {
|
||||
// Every key in registry.
|
||||
names := []string{
|
||||
"akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl",
|
||||
"duckduckgo", "email", "exesql", "execute_sql", "github", "google",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"querit", "querit_search",
|
||||
"querit", "querit_contents", "querit_search",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
|
||||
"yahoo_finance",
|
||||
@@ -204,7 +226,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
"akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl",
|
||||
"duckduckgo", "email", "execute_sql", "exesql", "github", "google",
|
||||
"google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather",
|
||||
"querit", "querit_search",
|
||||
"querit", "querit_contents", "querit_search",
|
||||
"retrieval", "search_my_dataset", "search_my_dateset", "searxng",
|
||||
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
|
||||
"yahoo_finance",
|
||||
@@ -275,6 +297,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
|
||||
"wikipedia": "wikipedia_search",
|
||||
"wikipedia_search": "wikipedia_search",
|
||||
"querit": "querit_search",
|
||||
"querit_contents": "querit_contents",
|
||||
"querit_search": "querit_search",
|
||||
}
|
||||
for _, name := range names {
|
||||
|
||||
Reference in New Issue
Block a user