From 1d8b9c74522cb3df3e49c548bb943fddd80f2aea Mon Sep 17 00:00:00 2001 From: EthanZhang Date: Wed, 12 Aug 2026 21:03:21 +0800 Subject: [PATCH] feat(agent): add Querit Contents tool (#18156) ### Summary Add Querit Contents as a built-in page content tool for RAGFlow Agents and Canvas workflows. Querit Contents crawls one or more known URLs and returns their page content and optional metadata. It complements the existing Querit Search tool: Search discovers relevant pages, while Contents retrieves pages already selected by an Agent or workflow. This integration supports two usage modes: - A standalone `QueritContents` node in Canvas workflows. - An embedded content tool available to RAGFlow Agents. --- agent/tools/querit.py | 189 ++++++++++-- internal/agent/component/tool_component.go | 1 + .../agent/component/tool_component_test.go | 9 +- internal/agent/tool/querit.go | 90 +++--- internal/agent/tool/querit_contents.go | 271 ++++++++++++++++++ internal/agent/tool/querit_contents_test.go | 188 ++++++++++++ internal/agent/tool/registry.go | 38 +++ internal/agent/tool/registry_test.go | 27 +- .../agent/tools/test_querit_contents_unit.py | 212 ++++++++++++++ web/src/components/operator-icon.tsx | 3 +- web/src/constants/agent.tsx | 1 + web/src/locales/en.ts | 13 + web/src/locales/zh.ts | 12 + .../node/dropdown/accordion-operators.tsx | 1 + web/src/pages/agent/constant/index.tsx | 22 ++ .../agent/form-sheet/form-config-map.tsx | 4 + .../agent-form/tool-popover/tool-command.tsx | 1 + .../agent/form/querit-contents-form/index.tsx | 138 +++++++++ .../pages/agent/form/tool-form/constant.tsx | 7 +- web/src/pages/agent/hooks/use-add-node.ts | 2 + .../hooks/use-agent-tool-initial-values.ts | 2 + .../pages/agent/log-sheet/tool-name.test.ts | 8 + web/src/pages/agent/log-sheet/tool-name.ts | 3 + .../agent/log-sheet/tool-timeline-item.tsx | 1 + .../utils/clear-sensitive-fields.test.ts | 42 +-- .../agent/utils/clear-sensitive-fields.ts | 3 +- 26 files changed, 1200 insertions(+), 88 deletions(-) create mode 100644 internal/agent/tool/querit_contents.go create mode 100644 internal/agent/tool/querit_contents_test.go create mode 100644 test/unit_test/agent/tools/test_querit_contents_unit.py create mode 100644 web/src/pages/agent/form/querit-contents-form/index.tsx diff --git a/agent/tools/querit.py b/agent/tools/querit.py index 6725342a94..51822e7965 100644 --- a/agent/tools/querit.py +++ b/agent/tools/querit.py @@ -20,6 +20,7 @@ import re import time from abc import ABC from typing import Any +from urllib.parse import urlparse import requests @@ -28,8 +29,10 @@ from common.connection_utils import timeout from common.http_client import DEFAULT_TIMEOUT QUERIT_SEARCH_URL = "https://api.querit.ai/v1/search" +QUERIT_CONTENTS_URL = "https://api.querit.ai/v1/contents" QUERIT_MAX_ATTEMPTS = 3 QUERIT_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} +QUERIT_CONTENT_FORMATS = {"text", "markdown", "html"} TIME_RANGE_PATTERN = re.compile(r"^([dwmy][1-9][0-9]*|\d{4}-\d{2}-\d{2}to\d{4}-\d{2}-\d{2})$") logger = logging.getLogger(__name__) @@ -192,35 +195,7 @@ class QueritSearch(ToolBase, ABC): return self._fail(_safe_error_message(error, api_key)) def _search(self, payload: dict[str, Any], api_key: str) -> Any: - headers = { - "Accept": "application/json", - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } - for attempt in range(QUERIT_MAX_ATTEMPTS): - if self.check_if_canceled("QueritSearch processing"): - raise _QueritCanceled - try: - response = requests.post( - QUERIT_SEARCH_URL, - headers=headers, - json=payload, - timeout=DEFAULT_TIMEOUT, - ) - if response.status_code in QUERIT_RETRYABLE_STATUS_CODES and attempt + 1 < QUERIT_MAX_ATTEMPTS: - self._wait_before_retry() - continue - response.raise_for_status() - return response.json() - except requests.JSONDecodeError: - raise - except requests.HTTPError: - raise - except requests.RequestException: - if attempt + 1 >= QUERIT_MAX_ATTEMPTS: - raise - self._wait_before_retry() - raise RuntimeError("Querit request failed after three attempts.") + return _post_querit(self, QUERIT_SEARCH_URL, payload, api_key, "QueritSearch") def _wait_before_retry(self) -> None: if self.check_if_canceled("QueritSearch processing"): @@ -236,6 +211,96 @@ class QueritSearch(ToolBase, ABC): return "Searching Querit for `{}`.".format(self.get_input().get("query", "-_-!")) +class QueritContentsParam(ToolParamBase): + def __init__(self): + self.meta: ToolMeta = { + "name": "querit_contents", + "description": "Crawl one or more web pages with Querit and return their contents.", + "parameters": { + "urls": { + "type": "array", + "description": "The absolute HTTP or HTTPS URLs to crawl. Supports 1 to 10 URLs.", + "default": [], + "items": {"type": "string"}, + "required": True, + }, + "format": { + "type": "string", + "description": "Content format: text, markdown, or html. Defaults to markdown.", + "enum": ["text", "markdown", "html"], + "default": "markdown", + "required": False, + }, + "crawl_timeout": { + "type": "integer", + "description": "Per-page crawl timeout in seconds. Must be between 1 and 60.", + "default": 10, + "required": False, + }, + "extras_meta": { + "type": "boolean", + "description": "Whether to include page metadata in each result.", + "default": False, + "required": False, + }, + }, + } + super().__init__() + self.api_key = "" + + def check(self): + self.urls = _normalize_contents_urls(self.urls) + _validate_contents_inputs(self.urls, self.format, self.crawl_timeout, self.extras_meta) + + def get_input_form(self) -> dict[str, dict]: + return {"urls": {"name": "URLs", "type": "line"}} + + +class QueritContents(ToolBase, ABC): + component_name = "QueritContents" + + @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", "70"))) + def _invoke(self, **kwargs): + if self.check_if_canceled("QueritContents processing"): + return + + values = {name: kwargs[name] if name in kwargs else getattr(self._param, name) for name in ("urls", "format", "crawl_timeout", "extras_meta")} + values["urls"] = _normalize_contents_urls(values["urls"]) + + node_api_key = (self._param.api_key or "").strip() + api_key = node_api_key or (os.environ.get("QUERIT_API_KEY") or "").strip() + if not api_key: + return self._fail("Querit API key is required. Configure api_key or set QUERIT_API_KEY.") + + try: + _validate_contents_inputs(**values) + response_data = self._request(_build_contents_payload(**values), api_key) + _validate_contents_response(response_data) + self.set_output("json", response_data) + return self.output("json") + except _QueritCanceled: + return + except (requests.RequestException, RuntimeError, TypeError, ValueError) as error: + return self._fail(_safe_error_message(error, api_key)) + + def _request(self, payload: dict[str, Any], api_key: str) -> Any: + request_timeout = max(DEFAULT_TIMEOUT, payload["crawlTimeout"] + 5) + return _post_querit(self, QUERIT_CONTENTS_URL, payload, api_key, "QueritContents", request_timeout) + + def _wait_before_retry(self) -> None: + if self.check_if_canceled("QueritContents processing"): + raise _QueritCanceled + time.sleep(self._param.delay_after_error) + + def _fail(self, message: str) -> str: + self.set_output("_ERROR", message) + logger.error("Querit contents failed: %s", message) + return f"Querit contents error: {message}" + + def thoughts(self) -> str: + return "Reading web page contents with Querit." + + def _build_payload(query: str, **values: Any) -> dict[str, Any]: payload: dict[str, Any] = { "query": query, @@ -262,6 +327,72 @@ def _build_payload(query: str, **values: Any) -> dict[str, Any]: return payload +def _post_querit(tool: Any, endpoint: str, payload: dict[str, Any], api_key: str, operation: str, request_timeout: float = DEFAULT_TIMEOUT) -> Any: + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + for attempt in range(QUERIT_MAX_ATTEMPTS): + if tool.check_if_canceled(f"{operation} processing"): + raise _QueritCanceled + try: + response = requests.post(endpoint, headers=headers, json=payload, timeout=request_timeout) + if response.status_code in QUERIT_RETRYABLE_STATUS_CODES and attempt + 1 < QUERIT_MAX_ATTEMPTS: + tool._wait_before_retry() + continue + response.raise_for_status() + return response.json() + except requests.JSONDecodeError: + raise + except requests.HTTPError: + raise + except requests.RequestException: + if attempt + 1 >= QUERIT_MAX_ATTEMPTS: + raise + tool._wait_before_retry() + raise RuntimeError("Querit request failed after three attempts.") + + +def _build_contents_payload(urls: list[str], format: str, crawl_timeout: int, extras_meta: bool) -> dict[str, Any]: + return { + "urls": urls, + "format": format, + "crawlTimeout": crawl_timeout, + "extrasMeta": extras_meta, + } + + +def _normalize_contents_urls(urls: Any) -> Any: + if isinstance(urls, str): + return [url.strip() for url in urls.split(",") if url.strip()] + return urls + + +def _validate_contents_inputs(urls: Any, format: Any, crawl_timeout: Any, extras_meta: Any) -> None: + if not isinstance(urls, list) or not 1 <= len(urls) <= 10 or any(not isinstance(url, str) or not url.strip() for url in urls): + raise ValueError("Querit urls must contain between 1 and 10 non-empty strings.") + for url in urls: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("Querit urls must be absolute HTTP or HTTPS URLs.") + if format not in QUERIT_CONTENT_FORMATS: + raise ValueError("Querit format must be text, markdown, or html.") + if type(crawl_timeout) is not int or not 1 <= crawl_timeout <= 60: + raise ValueError("Querit crawl_timeout must be an integer from 1 to 60.") + if type(extras_meta) is not bool: + raise ValueError("Querit extras_meta must be a boolean.") + + +def _validate_contents_response(response_data: Any) -> None: + if not isinstance(response_data, dict): + raise TypeError("Querit API response must be a JSON object.") + if "results" in response_data and not isinstance(response_data["results"], list): + raise TypeError("Querit API response field results must be an array.") + if "statuses" in response_data and not isinstance(response_data["statuses"], list): + raise TypeError("Querit API response field statuses must be an array.") + + def _validate_search_inputs( count: Any, chunks_per_doc: Any, diff --git a/internal/agent/component/tool_component.go b/internal/agent/component/tool_component.go index 2eaca204b5..9d7ac01859 100644 --- a/internal/agent/component/tool_component.go +++ b/internal/agent/component/tool_component.go @@ -134,6 +134,7 @@ var toolComponentRegistrations = []struct { {componentName: "GoogleScholar", toolName: "google_scholar"}, {componentName: "KeenableSearch", toolName: "keenable"}, {componentName: "PubMed", toolName: "pubmed"}, + {componentName: "QueritContents", toolName: "querit_contents"}, {componentName: "QueritSearch", toolName: "querit_search"}, {componentName: "SearXNG", toolName: "searxng"}, {componentName: "TavilySearch", toolName: "tavily"}, diff --git a/internal/agent/component/tool_component_test.go b/internal/agent/component/tool_component_test.go index 64decfa181..13ffaf5c01 100644 --- a/internal/agent/component/tool_component_test.go +++ b/internal/agent/component/tool_component_test.go @@ -193,6 +193,13 @@ func TestToolBackedComponentRegisteredFactories(t *testing.T) { outputKey: "success", inputKey: "to_email", }, + { + name: "QueritContents", + toolName: "QueritContents", + params: map[string]any{"api_key": "stored-key", "format": "markdown", "crawl_timeout": float64(10), "extras_meta": true, "outputs": map[string]any{"json": map[string]any{}}}, + outputKey: "json", + inputKey: "urls", + }, { name: "QueritSearch", toolName: "QueritSearch", @@ -283,7 +290,7 @@ func TestToolBackedComponentWenCaiInvoke(t *testing.T) { } func TestToolBackedComponentRegisteredBuildWorkflow(t *testing.T) { - for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritSearch", "SearXNG", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} { + for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "SearXNG", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} { t.Run(componentName, func(t *testing.T) { c := &canvas.Canvas{ Components: map[string]canvas.CanvasComponent{ diff --git a/internal/agent/tool/querit.go b/internal/agent/tool/querit.go index 5555c97f6e..a94d40a0ca 100644 --- a/internal/agent/tool/querit.go +++ b/internal/agent/tool/querit.go @@ -248,44 +248,58 @@ func (q *QueritTool) InvokableRun(ctx context.Context, argsJSON string, _ ...too if err != nil { return queritErrorJSON(fmt.Errorf("encode request: %w", err), apiKey), nil } + raw, requestErr := doQueritRequest(ctx, q.helper, q.retryWait, queritEndpoint, body, apiKey) + if requestErr != nil { + return queritErrorJSON(requestErr, apiKey), nil + } + if _, err := decodeQueritResponse(raw); err != nil { + return queritErrorJSON(err, apiKey), nil + } + return string(raw), nil +} + +func doQueritRequest( + ctx context.Context, + helper *HTTPHelper, + retryWait func(context.Context, int) bool, + endpoint string, + body []byte, + apiKey string, +) ([]byte, error) { for attempt := 1; attempt <= queritMaxAttempts; attempt++ { - resp, requestErr := q.helper.Do( + resp, err := helper.Do( ctx, http.MethodPost, - queritEndpoint, + endpoint, string(body), "application/json", map[string]string{"Authorization": "Bearer " + apiKey}, ) - if requestErr != nil { - return queritErrorJSON(fmt.Errorf("request failed: %w", requestErr), apiKey), nil + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) } if resp.StatusCode == http.StatusTooManyRequests { _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() - if attempt == queritMaxAttempts || !q.retryWait(ctx, attempt) { - return queritErrorJSON(fmt.Errorf("upstream returned %d after %d attempts", resp.StatusCode, attempt), apiKey), nil + if attempt == queritMaxAttempts || !retryWait(ctx, attempt) { + return nil, fmt.Errorf("upstream returned %d after %d attempts", resp.StatusCode, attempt) } continue } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() - return queritErrorJSON(fmt.Errorf("upstream returned %d", resp.StatusCode), apiKey), nil + return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) } raw, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() if readErr != nil { - return queritErrorJSON(fmt.Errorf("read response: %w", readErr), apiKey), nil + return nil, fmt.Errorf("read response: %w", readErr) } - if _, err := decodeQueritResponse(raw); err != nil { - return queritErrorJSON(err, apiKey), nil - } - return string(raw), nil + return raw, nil } - - return queritErrorJSON(fmt.Errorf("request exhausted retries"), apiKey), nil + return nil, fmt.Errorf("request exhausted retries") } func mergeQueritParams(defaults, runtimeParams queritParams, provided map[string]json.RawMessage) queritParams { @@ -360,21 +374,9 @@ func buildQueritRequest(params queritParams) queritRequest { } func decodeQueritResponse(raw []byte) (map[string]any, error) { - var decoded any - decoder := json.NewDecoder(strings.NewReader(string(raw))) - decoder.UseNumber() - if err := decoder.Decode(&decoded); err != nil { - return nil, fmt.Errorf("decode response: %w", err) - } - if err := decoder.Decode(&struct{}{}); err != io.EOF { - if err == nil { - return nil, fmt.Errorf("decode response: expected exactly one JSON value") - } - return nil, fmt.Errorf("decode response: trailing content: %w", err) - } - response, ok := decoded.(map[string]any) - if !ok || response == nil { - return nil, fmt.Errorf("decode response: expected a JSON object") + response, err := decodeQueritJSONObject(raw) + if err != nil { + return nil, err } resultsValue, exists := response["results"] if !exists { @@ -394,6 +396,26 @@ func decodeQueritResponse(raw []byte) (map[string]any, error) { return response, nil } +func decodeQueritJSONObject(raw []byte) (map[string]any, error) { + var decoded any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&decoded); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("decode response: expected exactly one JSON value") + } + return nil, fmt.Errorf("decode response: trailing content: %w", err) + } + response, ok := decoded.(map[string]any) + if !ok || response == nil { + return nil, fmt.Errorf("decode response: expected a JSON object") + } + return response, nil +} + func waitForQueritRetry(ctx context.Context, attempt int) bool { delay := 200 * time.Millisecond for current := 1; current < attempt; current++ { @@ -588,9 +610,13 @@ func queritStringSlice(value any) ([]string, bool) { } func queritErrorJSON(err error, apiKeys ...string) string { - message := "querit_search: unknown error" + return queritToolErrorJSON(queritToolName, err, apiKeys...) +} + +func queritToolErrorJSON(toolName string, err error, apiKeys ...string) string { + message := toolName + ": unknown error" if err != nil { - message = "querit_search: " + err.Error() + message = toolName + ": " + err.Error() } for _, apiKey := range apiKeys { if apiKey = strings.TrimSpace(apiKey); apiKey != "" { @@ -599,7 +625,7 @@ func queritErrorJSON(err error, apiKeys ...string) string { } raw, marshalErr := json.Marshal(map[string]any{"_ERROR": message}) if marshalErr != nil { - return `{"_ERROR":"querit_search: marshal error"}` + return fmt.Sprintf(`{"_ERROR":%q}`, toolName+": marshal error") } return string(raw) } diff --git a/internal/agent/tool/querit_contents.go b/internal/agent/tool/querit_contents.go new file mode 100644 index 0000000000..20ed212dff --- /dev/null +++ b/internal/agent/tool/querit_contents.go @@ -0,0 +1,271 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package tool + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +const ( + queritContentsToolName = "querit_contents" + queritContentsEndpoint = "https://api.querit.ai/v1/contents" +) + +type queritContentsParams struct { + APIKey string `json:"api_key"` + URLs any `json:"urls"` + Format string `json:"format"` + CrawlTimeout int `json:"crawl_timeout"` + ExtrasMeta bool `json:"extras_meta"` +} + +type queritContentsRequest struct { + URLs []string `json:"urls"` + Format string `json:"format"` + CrawlTimeout int `json:"crawlTimeout"` + ExtrasMeta bool `json:"extrasMeta"` +} + +// QueritContentsTool crawls public web pages through the Querit Contents API. +type QueritContentsTool struct { + helper *HTTPHelper + envKey func() string + defaults queritContentsParams + retryWait func(context.Context, int) bool +} + +var _ ToolComponent = (*QueritContentsTool)(nil) + +func NewQueritContentsTool() *QueritContentsTool { + return newQueritContentsTool(nil, nil, queritContentsParams{}, nil) +} + +func NewQueritContentsToolWith(helper *HTTPHelper) *QueritContentsTool { + return newQueritContentsTool(helper, nil, queritContentsParams{}, nil) +} + +func NewQueritContentsToolWithEnvKey(helper *HTTPHelper, envKey func() string) *QueritContentsTool { + return newQueritContentsTool(helper, envKey, queritContentsParams{}, nil) +} + +func newQueritContentsTool( + helper *HTTPHelper, + envKey func() string, + defaults queritContentsParams, + retryWait func(context.Context, int) bool, +) *QueritContentsTool { + if helper == nil { + helper = NewHTTPHelper() + helper.client.Timeout = 65 * time.Second + } + if envKey == nil { + envKey = defaultQueritEnvKey + } + if defaults.Format == "" { + defaults.Format = "markdown" + } + if defaults.CrawlTimeout == 0 { + defaults.CrawlTimeout = 10 + } + if retryWait == nil { + retryWait = waitForQueritRetry + } + return &QueritContentsTool{helper: helper, envKey: envKey, defaults: defaults, retryWait: retryWait} +} + +func (q *QueritContentsTool) Info(_ context.Context) (*schema.ToolInfo, error) { + return &schema.ToolInfo{ + Name: queritContentsToolName, + Desc: "Crawl one or more web pages with Querit and return their contents.", + ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{ + "urls": { + Type: schema.Array, + ElemInfo: &schema.ParameterInfo{Type: schema.String}, + Desc: "One to ten absolute HTTP or HTTPS URLs to crawl.", + Required: true, + }, + "format": { + Type: schema.String, + Desc: `Content format: "text", "markdown" (default), or "html".`, + Required: false, + }, + "crawl_timeout": { + Type: schema.Integer, + Desc: "Per-page crawl timeout in seconds. Defaults to 10 and must be between 1 and 60.", + Required: false, + }, + "extras_meta": { + Type: schema.Boolean, + Desc: "Whether to include page metadata in each result.", + Required: false, + }, + }), + }, nil +} + +func (q *QueritContentsTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) { + provided := make(map[string]json.RawMessage) + if err := json.Unmarshal([]byte(argsJSON), &provided); err != nil { + return queritContentsErrorJSON(fmt.Errorf("parse arguments: %w", err)), nil + } + var runtimeParams queritContentsParams + if err := json.Unmarshal([]byte(argsJSON), &runtimeParams); err != nil { + return queritContentsErrorJSON(fmt.Errorf("parse arguments: %w", err)), nil + } + params := mergeQueritContentsParams(q.defaults, runtimeParams, provided) + urls := normalizeQueritContentURLs(params.URLs) + if err := validateQueritContentsParams(params, urls); err != nil { + return queritContentsErrorJSON(err, params.APIKey), nil + } + + apiKey := strings.TrimSpace(params.APIKey) + if apiKey == "" { + apiKey = strings.TrimSpace(q.envKey()) + } + if apiKey == "" { + return queritContentsErrorJSON(fmt.Errorf("api_key is required (or set QUERIT_API_KEY)")), nil + } + + body, err := json.Marshal(queritContentsRequest{ + URLs: urls, + Format: params.Format, + CrawlTimeout: params.CrawlTimeout, + ExtrasMeta: params.ExtrasMeta, + }) + if err != nil { + return queritContentsErrorJSON(fmt.Errorf("encode request: %w", err), apiKey), nil + } + raw, requestErr := doQueritRequest(ctx, q.helper, q.retryWait, queritContentsEndpoint, body, apiKey) + if requestErr != nil { + return queritContentsErrorJSON(requestErr, apiKey), nil + } + if _, err := decodeQueritContentsResponse(raw); err != nil { + return queritContentsErrorJSON(err, apiKey), nil + } + return string(raw), nil +} + +func mergeQueritContentsParams( + defaults queritContentsParams, + runtimeParams queritContentsParams, + provided map[string]json.RawMessage, +) queritContentsParams { + merged := defaults + if _, ok := provided["urls"]; ok { + merged.URLs = runtimeParams.URLs + } + if _, ok := provided["format"]; ok { + merged.Format = runtimeParams.Format + } + if _, ok := provided["crawl_timeout"]; ok { + merged.CrawlTimeout = runtimeParams.CrawlTimeout + } + if _, ok := provided["extras_meta"]; ok { + merged.ExtrasMeta = runtimeParams.ExtrasMeta + } + return merged +} + +func validateQueritContentsParams(params queritContentsParams, urls []string) error { + if len(urls) < 1 || len(urls) > 10 { + return fmt.Errorf("urls must contain between 1 and 10 values") + } + for _, rawURL := range urls { + parsed, err := url.ParseRequestURI(rawURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return fmt.Errorf("urls must be absolute HTTP or HTTPS URLs") + } + } + if params.Format != "text" && params.Format != "markdown" && params.Format != "html" { + return fmt.Errorf("format must be text, markdown, or html") + } + if params.CrawlTimeout < 1 || params.CrawlTimeout > 60 { + return fmt.Errorf("crawl_timeout must be between 1 and 60") + } + return nil +} + +func normalizeQueritContentURLs(raw any) []string { + switch value := raw.(type) { + case string: + return compactStrings(strings.Split(value, ",")) + case []string: + return compactStrings(value) + case []any: + urls := make([]string, 0, len(value)) + for _, item := range value { + text, ok := item.(string) + if !ok { + return nil + } + urls = append(urls, text) + } + return compactStrings(urls) + default: + return nil + } +} + +func decodeQueritContentsResponse(raw []byte) (map[string]any, error) { + response, err := decodeQueritJSONObject(raw) + if err != nil { + return nil, err + } + for _, field := range []string{"results", "statuses"} { + if value, exists := response[field]; exists { + if _, ok := value.([]any); !ok { + return nil, fmt.Errorf("decode response: %s must be a JSON array", field) + } + } + } + return response, nil +} + +func (q *QueritContentsTool) ComponentSpec() ComponentSpec { + return ComponentSpec{ + PreserveJSONNumbers: true, + Inputs: map[string]string{ + "api_key": "Querit API key. Uses QUERIT_API_KEY when empty.", + "urls": "One to ten absolute HTTP or HTTPS URLs to crawl.", + "format": `Content format: "text", "markdown", or "html".`, + "crawl_timeout": "Per-page crawl timeout in seconds.", + "extras_meta": "Whether to include page metadata.", + }, + Outputs: map[string]string{ + "json": "Complete raw Querit Contents JSON response.", + }, + InputForm: map[string]any{ + "urls": map[string]any{"name": "URLs", "type": "line"}, + }, + } +} + +func (q *QueritContentsTool) BuildComponentOutputs(response map[string]any) map[string]any { + return map[string]any{"json": response} +} + +func queritContentsErrorJSON(err error, apiKeys ...string) string { + return queritToolErrorJSON(queritContentsToolName, err, apiKeys...) +} diff --git a/internal/agent/tool/querit_contents_test.go b/internal/agent/tool/querit_contents_test.go new file mode 100644 index 0000000000..76645d57da --- /dev/null +++ b/internal/agent/tool/querit_contents_test.go @@ -0,0 +1,188 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package tool + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestQueritContentsBuildsRequestAndPreservesResponse(t *testing.T) { + var gotMethod, gotPath, gotAuthorization string + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + gotMethod = request.Method + gotPath = request.URL.Path + gotAuthorization = request.Header.Get("Authorization") + _ = json.NewDecoder(request.Body).Decode(&gotBody) + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"error_code":0,"search_id":"crawl-1","results":[{"id":"1","url":"https://example.com","content":"# Example"}],"statuses":[{"id":"1","status":"success"}],"searchTime":1}`)) + })) + defer server.Close() + + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)}) + contents := newQueritContentsTool(helper, func() string { return "" }, queritContentsParams{APIKey: "key-test"}, nil) + out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"],"format":"html","crawl_timeout":20,"extras_meta":true}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if gotMethod != http.MethodPost || gotPath != "/v1/contents" { + t.Fatalf("request = %s %s, want POST /v1/contents", gotMethod, gotPath) + } + if gotAuthorization != "Bearer key-test" { + t.Fatalf("Authorization = %q", gotAuthorization) + } + if gotBody["format"] != "html" || gotBody["crawlTimeout"] != float64(20) || gotBody["extrasMeta"] != true { + t.Fatalf("request body = %#v", gotBody) + } + urls, ok := gotBody["urls"].([]any) + if !ok || len(urls) != 1 || urls[0] != "https://example.com" { + t.Fatalf("urls = %#v", gotBody["urls"]) + } + if !strings.Contains(out, `"search_id":"crawl-1"`) || !strings.Contains(out, `"statuses"`) { + t.Fatalf("complete response was not retained: %s", out) + } +} + +func TestQueritContentsMergesDefaultsAndExplicitFalse(t *testing.T) { + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _ = json.NewDecoder(request.Body).Decode(&gotBody) + _, _ = writer.Write([]byte(`{"results":[],"statuses":[]}`)) + })) + defer server.Close() + + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)}) + contents := newQueritContentsTool(helper, func() string { return "" }, queritContentsParams{ + APIKey: "stored-key", + URLs: []string{"https://stored.example"}, + Format: "text", + CrawlTimeout: 30, + ExtrasMeta: true, + }, nil) + _, err := contents.InvokableRun(context.Background(), `{"urls":"https://runtime.example","extras_meta":false}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if gotBody["format"] != "text" || gotBody["crawlTimeout"] != float64(30) || gotBody["extrasMeta"] != false { + t.Fatalf("merged defaults = %#v", gotBody) + } + if gotBody["urls"].([]any)[0] != "https://runtime.example" { + t.Fatalf("runtime urls = %#v", gotBody["urls"]) + } +} + +func TestQueritContentsValidatesInputsAndAPIKey(t *testing.T) { + tests := []struct { + name string + args string + want string + }{ + {name: "missing urls", args: `{}`, want: "urls must contain"}, + {name: "too many urls", args: `{"urls":["https://1.example","https://2.example","https://3.example","https://4.example","https://5.example","https://6.example","https://7.example","https://8.example","https://9.example","https://10.example","https://11.example"]}`, want: "between 1 and 10"}, + {name: "relative url", args: `{"urls":["example.com"]}`, want: "absolute HTTP or HTTPS"}, + {name: "unsupported scheme", args: `{"urls":["file:///tmp/page"]}`, want: "absolute HTTP or HTTPS"}, + {name: "bad format", args: `{"urls":["https://example.com"],"format":"xml"}`, want: "format must be"}, + {name: "bad timeout", args: `{"urls":["https://example.com"],"crawl_timeout":61}`, want: "between 1 and 60"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + contents := NewQueritContentsToolWithEnvKey(NewHTTPHelper(), func() string { return "key-test" }) + out, err := contents.InvokableRun(context.Background(), test.args) + if err != nil || !strings.Contains(out, "_ERROR") || !strings.Contains(out, test.want) { + t.Fatalf("result = %s, err = %v", out, err) + } + }) + } + + contents := NewQueritContentsToolWithEnvKey(NewHTTPHelper(), func() string { return "" }) + out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`) + if err != nil || !strings.Contains(out, "api_key is required") { + t.Fatalf("missing key result = %s, err = %v", out, err) + } +} + +func TestQueritContentsRejectsMalformedResponses(t *testing.T) { + for _, test := range []struct { + name string + body string + want string + }{ + {name: "top-level array", body: `[]`, want: "JSON object"}, + {name: "results object", body: `{"results":{}}`, want: "results must be a JSON array"}, + {name: "statuses object", body: `{"results":[],"statuses":{}}`, want: "statuses must be a JSON array"}, + {name: "trailing content", body: `{"results":[]} trailing`, want: "trailing content"}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write([]byte(test.body)) + })) + defer server.Close() + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)}) + contents := NewQueritContentsToolWithEnvKey(helper, func() string { return "key-test" }) + out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`) + if err != nil || !strings.Contains(out, test.want) { + t.Fatalf("result = %s, err = %v", out, err) + } + }) + } +} + +func TestQueritContentsRedactsAPIKey(t *testing.T) { + const secret = "secret-contents-key" + helper := NewHTTPHelper().WithClient(&http.Client{Transport: roundTripperErrorFunc(func(*http.Request) error { + return errors.New("failed with " + secret) + })}) + contents := NewQueritContentsToolWithEnvKey(helper, func() string { return secret }) + out, err := contents.InvokableRun(context.Background(), `{"urls":["https://example.com"]}`) + if err != nil || strings.Contains(out, secret) || !strings.Contains(out, "[REDACTED]") { + t.Fatalf("result = %s, err = %v", out, err) + } +} + +func TestQueritContentsInfoAndComponentContract(t *testing.T) { + contents := NewQueritContentsTool() + if contents.helper.client.Timeout != 65*time.Second { + t.Fatalf("HTTP timeout = %s, want 65s", contents.helper.client.Timeout) + } + info, err := contents.Info(context.Background()) + if err != nil || info.Name != queritContentsToolName || info.ParamsOneOf == nil { + t.Fatalf("Info = %#v, %v", info, err) + } + encoded, _ := json.Marshal(info) + if strings.Contains(string(encoded), "api_key") { + t.Fatalf("Info exposed API key: %s", encoded) + } + spec := contents.ComponentSpec() + if spec.Inputs["urls"] == "" || spec.Outputs["json"] == "" || !spec.PreserveJSONNumbers { + t.Fatalf("ComponentSpec = %#v", spec) + } + if len(spec.InputForm) != 1 || spec.InputForm["urls"] == nil { + t.Fatalf("InputForm = %#v, want URLs only", spec.InputForm) + } + response := map[string]any{"search_id": "crawl-1", "results": []any{map[string]any{"content": "page"}}} + outputs := contents.BuildComponentOutputs(response) + if outputs["json"].(map[string]any)["search_id"] != "crawl-1" { + t.Fatalf("outputs = %#v", outputs) + } +} diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.go index f5143b9ec2..4f8eb71d75 100644 --- a/internal/agent/tool/registry.go +++ b/internal/agent/tool/registry.go @@ -55,6 +55,7 @@ var registry = map[string]Factory{ "pubmed": buildPubMedTool, "qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }), "querit": buildQueritTool, + "querit_contents": buildQueritContentsTool, "querit_search": buildQueritTool, "retrieval": buildRetrievalTool, "search_my_dataset": buildRetrievalTool, @@ -78,6 +79,7 @@ var canvasToolNames = map[string]string{ "codeexec": "code_exec", "googlescholar": "google_scholar", "keenablesearch": "keenable", + "queritcontents": "querit_contents", "queritsearch": "querit_search", "tavilyextract": "tavily_extract", "tavilysearch": "tavily", @@ -657,6 +659,42 @@ func buildQueritTool(params map[string]any) (einotool.BaseTool, error) { return newQueritTool(nil, nil, defaults, nil), nil } +func buildQueritContentsTool(params map[string]any) (einotool.BaseTool, error) { + defaults := queritContentsParams{} + if value, exists := params["api_key"]; exists { + apiKey, valid := value.(string) + if !valid { + return nil, fmt.Errorf("agent tool: tool %q requires string node-level param api_key", queritContentsToolName) + } + defaults.APIKey = apiKey + } + if value, exists := params["urls"]; exists { + defaults.URLs = value + } + if value, exists := params["format"]; exists { + format, valid := value.(string) + if !valid || (format != "text" && format != "markdown" && format != "html") { + return nil, fmt.Errorf("agent tool: tool %q has unsupported format %q", queritContentsToolName, format) + } + defaults.Format = format + } + if value, exists := params["crawl_timeout"]; exists { + crawlTimeout, valid := strictInt(value) + if !valid || crawlTimeout < 1 || crawlTimeout > 60 { + return nil, fmt.Errorf("agent tool: tool %q requires integer node-level param crawl_timeout within [1, 60]", queritContentsToolName) + } + defaults.CrawlTimeout = crawlTimeout + } + if value, exists := params["extras_meta"]; exists { + extrasMeta, valid := value.(bool) + if !valid { + return nil, fmt.Errorf("agent tool: tool %q requires boolean node-level param extras_meta", queritContentsToolName) + } + defaults.ExtrasMeta = extrasMeta + } + return newQueritContentsTool(nil, nil, defaults, nil), nil +} + func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) { defaults := keenableParams{} apiKey := "" diff --git a/internal/agent/tool/registry_test.go b/internal/agent/tool/registry_test.go index 1f71a40f9f..a1d2151ba9 100644 --- a/internal/agent/tool/registry_test.go +++ b/internal/agent/tool/registry_test.go @@ -48,6 +48,7 @@ func TestBuildByName_CanvasComponentNames(t *testing.T) { {name: "CodeExec", wantToolName: "execute_code"}, {name: "GoogleScholar", wantToolName: "google_scholar_search"}, {name: "KeenableSearch", wantToolName: "keenable_search"}, + {name: "QueritContents", wantToolName: "querit_contents"}, {name: "QueritSearch", wantToolName: "querit_search"}, {name: "TavilyExtract", wantToolName: "tavily_extract"}, {name: "TavilySearch", wantToolName: "tavily_search"}, @@ -134,13 +135,34 @@ func TestBuildByName_QueritAliases(t *testing.T) { } } +func TestBuildByName_QueritContentsAliases(t *testing.T) { + for _, name := range []string{"querit_contents", "queritcontents", "QueritContents"} { + built, err := BuildByName(name, map[string]any{ + "api_key": "stored-key", + "format": "html", + "crawl_timeout": float64(20), + "extras_meta": true, + }) + if err != nil { + t.Fatalf("BuildByName(%q): %v", name, err) + } + contents, ok := built.(*QueritContentsTool) + if !ok { + t.Fatalf("BuildByName(%q) returned %T, want *QueritContentsTool", name, built) + } + if contents.defaults.Format != "html" || contents.defaults.CrawlTimeout != 20 || !contents.defaults.ExtrasMeta { + t.Fatalf("BuildByName(%q) defaults = %#v", name, contents.defaults) + } + } +} + func TestBuildAll_AllRegisteredTools(t *testing.T) { // Every key in registry. names := []string{ "akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl", "duckduckgo", "email", "exesql", "execute_sql", "github", "google", "google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather", - "querit", "querit_search", + "querit", "querit_contents", "querit_search", "retrieval", "search_my_dataset", "search_my_dateset", "searxng", "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search", "yahoo_finance", @@ -204,7 +226,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) { "akshare", "arxiv", "bgpt", "code_exec", "crawler", "deepl", "duckduckgo", "email", "execute_sql", "exesql", "github", "google", "google_scholar", "google_scholar_search", "jin10", "keenable", "pubmed", "qweather", - "querit", "querit_search", + "querit", "querit_contents", "querit_search", "retrieval", "search_my_dataset", "search_my_dateset", "searxng", "tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search", "yahoo_finance", @@ -275,6 +297,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) { "wikipedia": "wikipedia_search", "wikipedia_search": "wikipedia_search", "querit": "querit_search", + "querit_contents": "querit_contents", "querit_search": "querit_search", } for _, name := range names { diff --git a/test/unit_test/agent/tools/test_querit_contents_unit.py b/test/unit_test/agent/tools/test_querit_contents_unit.py new file mode 100644 index 0000000000..dbf4f8df8c --- /dev/null +++ b/test/unit_test/agent/tools/test_querit_contents_unit.py @@ -0,0 +1,212 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +import pytest + +import agent.tools.querit as querit_module +from agent.tools.querit import QueritContents, QueritContentsParam + + +class _FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise querit_module.requests.HTTPError(f"{self.status_code} error", response=self) + + +def _make_tool(api_key="test-api-key"): + tool = QueritContents.__new__(QueritContents) + param = QueritContentsParam() + param.api_key = api_key + param.delay_after_error = 0 + tool._param = param + tool.check_if_canceled = lambda *args, **kwargs: False + + outputs = {} + tool.set_output = lambda key, value: outputs.__setitem__(key, value) + tool.output = lambda key=None: outputs.get(key) if key else outputs + return tool, outputs + + +def test_contents_posts_documented_defaults_and_preserves_response(monkeypatch): + raw_response = { + "error_code": 0, + "error_msg": "", + "search_id": "crawl-1", + "results": [ + { + "id": "1", + "url": "https://example.com/article", + "content": "# Article", + "extrasMeta": {"title": "Article", "siteName": "Example"}, + } + ], + "statuses": [{"id": "1", "status": "success"}], + "searchTime": 1, + } + calls = [] + + def fake_post(url, **kwargs): + calls.append((url, kwargs)) + return _FakeResponse(raw_response) + + monkeypatch.setattr(querit_module.requests, "post", fake_post) + tool, outputs = _make_tool() + + result = tool._invoke(urls=["https://example.com/article"]) + + assert result == raw_response + assert calls == [ + ( + "https://api.querit.ai/v1/contents", + { + "headers": { + "Accept": "application/json", + "Authorization": "Bearer test-api-key", + "Content-Type": "application/json", + }, + "json": { + "urls": ["https://example.com/article"], + "format": "markdown", + "crawlTimeout": 10, + "extrasMeta": False, + }, + "timeout": querit_module.DEFAULT_TIMEOUT, + }, + ) + ] + assert outputs["json"] == raw_response + + +def test_contents_accepts_canvas_url_string_and_runtime_options(monkeypatch): + calls = [] + + def fake_post(url, **kwargs): + calls.append(kwargs) + return _FakeResponse({"results": [], "statuses": []}) + + monkeypatch.setattr(querit_module.requests, "post", fake_post) + tool, _ = _make_tool() + + tool._invoke( + urls="https://one.example, https://two.example", + format="html", + crawl_timeout=60, + extras_meta=True, + ) + + assert calls[0]["json"] == { + "urls": ["https://one.example", "https://two.example"], + "format": "html", + "crawlTimeout": 60, + "extrasMeta": True, + } + assert calls[0]["timeout"] == 65 + + +def test_contents_param_normalizes_canvas_url_string_before_validation(): + param = QueritContentsParam() + param.urls = "https://one.example, https://two.example" + param.format = "markdown" + param.crawl_timeout = 10 + param.extras_meta = False + + param.check() + + assert param.urls == ["https://one.example", "https://two.example"] + assert param.get_input_form() == {"urls": {"name": "URLs", "type": "line"}} + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"urls": []}, "between 1 and 10"), + ({"urls": [f"https://{index}.example" for index in range(11)]}, "between 1 and 10"), + ({"urls": ["example.com"]}, "absolute HTTP or HTTPS"), + ({"urls": ["file:///tmp/page"]}, "absolute HTTP or HTTPS"), + ({"urls": ["https://example.com"], "format": "xml"}, "format must be"), + ({"urls": ["https://example.com"], "crawl_timeout": 0}, "integer from 1 to 60"), + ({"urls": ["https://example.com"], "extras_meta": 1}, "must be a boolean"), + ], +) +def test_contents_rejects_invalid_inputs(kwargs, message): + tool, outputs = _make_tool() + + result = tool._invoke(**kwargs) + + assert message in result + assert message in outputs["_ERROR"] + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + ([], "JSON object"), + ({"results": {}}, "results must be an array"), + ({"results": [], "statuses": {}}, "statuses must be an array"), + ], +) +def test_contents_rejects_malformed_response(monkeypatch, payload, message): + monkeypatch.setattr(querit_module.requests, "post", lambda *args, **kwargs: _FakeResponse(payload)) + tool, outputs = _make_tool() + + result = tool._invoke(urls=["https://example.com"]) + + assert message in result + assert message in outputs["_ERROR"] + + +def test_contents_redacts_api_key_from_errors_and_logs(monkeypatch, caplog): + secret = f"secret-contents-key-{id(caplog)}" + + def fake_post(*args, **kwargs): + raise querit_module.requests.ConnectionError(f"connection failed for {secret}") + + monkeypatch.setattr(querit_module.requests, "post", fake_post) + tool, outputs = _make_tool(api_key=secret) + + with caplog.at_level(logging.ERROR): + result = tool._invoke(urls=["https://example.com"]) + + assert secret not in result + assert secret not in outputs["_ERROR"] + assert secret not in caplog.text + assert "[REDACTED]" in result + + +def test_contents_contract_and_dynamic_discovery(): + import agent.tools as tools_package + + metadata = QueritContentsParam().get_meta()["function"] + assert metadata["name"] == "querit_contents" + assert metadata["parameters"]["required"] == ["urls"] + assert set(metadata["parameters"]["properties"]) == { + "urls", + "format", + "crawl_timeout", + "extras_meta", + } + assert "api_key" not in metadata["parameters"]["properties"] + assert tools_package.QueritContents is QueritContents + assert tools_package.QueritContentsParam is QueritContentsParam diff --git a/web/src/components/operator-icon.tsx b/web/src/components/operator-icon.tsx index 79982d10b3..9e8528c37e 100644 --- a/web/src/components/operator-icon.tsx +++ b/web/src/components/operator-icon.tsx @@ -74,6 +74,7 @@ export const SVGIconMap = { [Operator.KeenableSearch]: 'keenable', [Operator.TavilyExtract]: 'tavily', [Operator.TavilySearch]: 'tavily', + [Operator.QueritContents]: 'querit', [Operator.QueritSearch]: 'querit', [Operator.Wikipedia]: 'wikipedia', [Operator.YahooFinance]: 'yahoo-finance', @@ -123,7 +124,7 @@ const OperatorIcon = ({ name, className }: IProps) => { const svgIcon = SVGIconMap[name as keyof typeof SVGIconMap]; const LucideIcon = LucideIconMap[name as keyof typeof LucideIconMap]; - if (name === Operator.QueritSearch) { + if (name === Operator.QueritContents || name === Operator.QueritSearch) { return ( >({ + defaultValues: values, + resolver: zodResolver(FormSchema), + mode: 'onChange', + }); + + useWatchFormChange(node?.id, form); + + return ( +
+ + + + ( + + + {t('queritContentsUrls')} + + + + + + + )} + /> + ( + + {t('format')} + + + + + + )} + /> + ( + + + {t('queritContentsTimeout')} + + + + + + + )} + /> + ( + + + {t('queritContentsMetadata')} + + + + + + )} + /> + + +
+ +
+
+ ); +} + +export default memo(QueritContentsForm); diff --git a/web/src/pages/agent/form/tool-form/constant.tsx b/web/src/pages/agent/form/tool-form/constant.tsx index 772ed10c6a..e6b369429d 100644 --- a/web/src/pages/agent/form/tool-form/constant.tsx +++ b/web/src/pages/agent/form/tool-form/constant.tsx @@ -14,7 +14,7 @@ import QueritForm from './querit-form'; import BGPTForm from './bgpt-form'; import RetrievalForm from './retrieval-form'; import SearXNGForm from './searxng-form'; -import TavilyForm from './tavily-form'; +import ApiKeyToolForm from './tavily-form'; import WenCaiForm from './wencai-form'; import WikipediaForm from './wikipedia-form'; import YahooFinanceForm from './yahoo-finance-form'; @@ -35,8 +35,9 @@ export const ToolFormConfigMap = { [Operator.YahooFinance]: YahooFinanceForm, [Operator.Crawler]: CrawlerForm, [Operator.Email]: EmailForm, - [Operator.TavilySearch]: TavilyForm, - [Operator.TavilyExtract]: TavilyForm, + [Operator.TavilySearch]: ApiKeyToolForm, + [Operator.TavilyExtract]: ApiKeyToolForm, + [Operator.QueritContents]: ApiKeyToolForm, [Operator.QueritSearch]: QueritForm, [Operator.WenCai]: WenCaiForm, [Operator.SearXNG]: SearXNGForm, diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts index 1cbe828239..cc9b6dfd45 100644 --- a/web/src/pages/agent/hooks/use-add-node.ts +++ b/web/src/pages/agent/hooks/use-add-node.ts @@ -38,6 +38,7 @@ import { initialParserValues, initialPubMedValues, initialBGPTValues, + initialQueritContentsValues, initialQueritValues, initialRetrievalValues, initialRewriteQuestionValues, @@ -168,6 +169,7 @@ export const useInitializeOperatorParams = () => { [Operator.Agent]: { ...initialAgentValues, llm_id: llmId }, [Operator.Tool]: {}, [Operator.TavilySearch]: initialTavilyValues, + [Operator.QueritContents]: initialQueritContentsValues, [Operator.QueritSearch]: initialQueritValues, [Operator.KeenableSearch]: initialKeenableValues, [Operator.UserFillUp]: initialUserFillUpValues, diff --git a/web/src/pages/agent/hooks/use-agent-tool-initial-values.ts b/web/src/pages/agent/hooks/use-agent-tool-initial-values.ts index 3250a55fde..814afc3d72 100644 --- a/web/src/pages/agent/hooks/use-agent-tool-initial-values.ts +++ b/web/src/pages/agent/hooks/use-agent-tool-initial-values.ts @@ -23,6 +23,8 @@ export function useAgentToolInitialValues() { }; case Operator.QueritSearch: return getQueritAgentInitialValues(initialValues); + case Operator.QueritContents: + return { api_key: '' }; case Operator.ExeSQL: return omit(initialValues, 'sql'); case Operator.Bing: diff --git a/web/src/pages/agent/log-sheet/tool-name.test.ts b/web/src/pages/agent/log-sheet/tool-name.test.ts index 2f99622724..538cff9b9e 100644 --- a/web/src/pages/agent/log-sheet/tool-name.test.ts +++ b/web/src/pages/agent/log-sheet/tool-name.test.ts @@ -1,5 +1,6 @@ jest.mock('@/constants/agent', () => ({ Operator: { + QueritContents: 'QueritContents', QueritSearch: 'QueritSearch', }, })); @@ -15,6 +16,13 @@ describe('getToolOperatorName', () => { }, ); + it.each(['QueritContents', 'querit_contents'])( + 'maps the Querit Contents timeline name %p to its operator', + (toolName) => { + expect(getToolOperatorName(toolName)).toBe(Operator.QueritContents); + }, + ); + it.each([undefined, null, ''])( 'returns an empty name for the missing value %p', (toolName) => { diff --git a/web/src/pages/agent/log-sheet/tool-name.ts b/web/src/pages/agent/log-sheet/tool-name.ts index 80e87f2c15..3bcb718060 100644 --- a/web/src/pages/agent/log-sheet/tool-name.ts +++ b/web/src/pages/agent/log-sheet/tool-name.ts @@ -9,6 +9,9 @@ export function getToolOperatorName(toolName?: string | null) { if (normalizedName === Operator.QueritSearch.toLowerCase()) { return Operator.QueritSearch; } + if (normalizedName === Operator.QueritContents.toLowerCase()) { + return Operator.QueritContents; + } return toolName .split('_') diff --git a/web/src/pages/agent/log-sheet/tool-timeline-item.tsx b/web/src/pages/agent/log-sheet/tool-timeline-item.tsx index f7d7d71a6c..6666f641c5 100644 --- a/web/src/pages/agent/log-sheet/tool-timeline-item.tsx +++ b/web/src/pages/agent/log-sheet/tool-timeline-item.tsx @@ -29,6 +29,7 @@ type IToolIcon = | Operator.BGPT | Operator.TavilyExtract | Operator.TavilySearch + | Operator.QueritContents | Operator.QueritSearch | Operator.KeenableSearch | Operator.Wikipedia diff --git a/web/src/pages/agent/utils/clear-sensitive-fields.test.ts b/web/src/pages/agent/utils/clear-sensitive-fields.test.ts index 1fd1a4fd1e..f19b77469c 100644 --- a/web/src/pages/agent/utils/clear-sensitive-fields.test.ts +++ b/web/src/pages/agent/utils/clear-sensitive-fields.test.ts @@ -5,6 +5,7 @@ jest.mock('@/constants/agent', () => ({ Google: 'Google', KeenableSearch: 'KeenableSearch', BGPT: 'Bing', + QueritContents: 'QueritContents', QueritSearch: 'QueritSearch', }, })); @@ -44,28 +45,31 @@ describe('clearSensitiveFields', () => { expect(dsl.tools[0].params.api_key).toBe('querit-secret'); }); - it.each(['querit', 'querit_search', 'queritsearch'])( - 'clears a Querit API key for the %s registry alias', - (componentName) => { - const dsl = { - tools: [ - { - component_name: componentName, - params: { - api_key: 'querit-secret', - count: 5, - }, + it.each([ + 'querit', + 'querit_contents', + 'queritcontents', + 'querit_search', + 'queritsearch', + ])('clears a Querit API key for the %s registry alias', (componentName) => { + const dsl = { + tools: [ + { + component_name: componentName, + params: { + api_key: 'querit-secret', + count: 5, }, - ], - }; + }, + ], + }; - const sanitized = clearSensitiveFields(dsl); + const sanitized = clearSensitiveFields(dsl); - expect(sanitized.tools[0].params.api_key).toBe(''); - expect(sanitized.tools[0].params.count).toBe(5); - expect(dsl.tools[0].params.api_key).toBe('querit-secret'); - }, - ); + expect(sanitized.tools[0].params.api_key).toBe(''); + expect(sanitized.tools[0].params.count).toBe(5); + expect(dsl.tools[0].params.api_key).toBe('querit-secret'); + }); it('clears a standalone Querit Canvas key from graph and components', () => { const dsl = { diff --git a/web/src/pages/agent/utils/clear-sensitive-fields.ts b/web/src/pages/agent/utils/clear-sensitive-fields.ts index 2129cd39f0..d7d6c59d1b 100644 --- a/web/src/pages/agent/utils/clear-sensitive-fields.ts +++ b/web/src/pages/agent/utils/clear-sensitive-fields.ts @@ -7,6 +7,7 @@ const apiKeyOperators = [ Operator.Google, Operator.KeenableSearch, Operator.BGPT, + Operator.QueritContents, Operator.QueritSearch, ]; @@ -15,7 +16,7 @@ function isQueritOperator(value: unknown) { return false; } - return ['querit', 'queritsearch'].includes( + return ['querit', 'queritcontents', 'queritsearch'].includes( value.replace(/_/g, '').toLowerCase(), ); }