feat(chat): add Querit web search provider (#17813)

This commit is contained in:
EthanZhang
2026-08-05 09:54:46 +08:00
committed by GitHub
parent 4d68e154ce
commit bdcd8aadde
32 changed files with 1253 additions and 134 deletions

View File

@@ -28,7 +28,6 @@ import (
"errors"
"fmt"
"hash/fnv"
"strings"
"sync"
"ragflow/internal/agent/canvas"
@@ -83,7 +82,7 @@ func NewBotService(agentSvc *AgentService, llmSvc *LLMService) *BotService {
// it (TenantID match), and Status must equal common.StatusDialogValid
// (the python StatusEnum.VALID.value).
func (s *BotService) ChatbotInfo(ctx context.Context, tenantID, dialogID string) (
title, avatar, prologue, llmID string, hasTavilyKey bool, ec common.ErrorCode, err error,
title, avatar, prologue, llmID string, hasWebSearch bool, ec common.ErrorCode, err error,
) {
dialog, err := s.chatDAO.GetDialogByID(ctx, dao.DB, dialogID)
if err != nil {
@@ -97,14 +96,13 @@ func (s *BotService) ChatbotInfo(ctx context.Context, tenantID, dialogID string)
pc := dialog.PromptConfig
// Defensive lookups mirroring python's
// dialog.prompt_config.get("prologue", "") and
// dialog.prompt_config.get("tavily_api_key", "").strip()
// resolveWebSearchProvider(dialog.prompt_config) != nil
// semantics. A hard type assertion here would panic on a missing
// or non-string prologue field — this endpoint is public over
// persisted JSON config and the schema is not guaranteed.
prologue = stringFromMap(pc, "prologue")
tk := stringFromMap(pc, "tavily_api_key")
return botDerefStr(dialog.Name), botDerefStr(dialog.Icon), prologue,
dialog.LLMID, strings.TrimSpace(tk) != "", common.CodeSuccess, nil
dialog.LLMID, resolveWebSearchProvider(pc) != nil, common.CodeSuccess, nil
}
// AgentbotInputs returns the public metadata of an agentbot canvas.

View File

@@ -125,12 +125,12 @@ type AsyncChatResult struct {
// │ │
// │ reasoning=true? │
// │ YES → DeepResearcher (recursive, maxDepth=3) │
// │ each layer: KB → Web(Tavily) → KG(use_kg) │
// │ each layer: KB → Web search → KG(use_kg)
// │ → sufficiencyCheck → multiQueriesGen → recurse│
// │ NO → Standard vector retrieval │
// │ vector/hybrid search → rerank → │
// │ TOC enhance → child chunk retrieval → │
// │ Tavily web search → KG retrieval (prepend) │
// │ Web search → KG retrieval (prepend)
// │ │
// │ enrichChunksWithMetadata (doc metadata) │
// │ kbPrompt (build knowledge blocks) │
@@ -184,7 +184,7 @@ func (s *ChatPipelineService) AsyncChat(
if useWebSearch {
common.Debug("web_search",
zap.Bool("kb", hasKBs),
zap.Bool("tavily", chat.PromptConfig != nil && chat.PromptConfig["tavily_api_key"] != "" && chat.PromptConfig["tavily_api_key"] != nil),
zap.Bool("configured", resolveWebSearchProvider(chat.PromptConfig) != nil),
zap.Any("internet", kwargs["internet"]),
zap.Bool("enabled", useWebSearch))
}
@@ -612,7 +612,7 @@ func (s *ChatPipelineService) AsyncChat(
// b) Otherwise: standard retrieval, then:
// - TOC enhancement (if toc_enhance is enabled).
// - Child chunk retrieval.
// - Tavily web search (if internet is enabled).
// - Web search provider (if internet is enabled).
// - Knowledge graph retrieval (if use_kg is enabled).
// Populates kbinfos (chunks + doc_aggs) and knowledges.
// When false, the entire block is skipped.
@@ -772,21 +772,21 @@ func (s *ChatPipelineService) AsyncChat(
kbinfos["chunks"] = nlp.RetrievalByChildren(existingChunks, kbTenantIDStrings(kbs), engine.Get(), ctx)
}
// Web search via Tavily
// Web search
if s.shouldUseWebSearch(chat, kwargs["internet"]) {
tavilyKey, _ := chat.PromptConfig["tavily_api_key"].(string)
tavResult, tavErr := s.tavilyRetrieve(ctx, tavilyKey, searchQuestion)
if tavErr != nil {
common.Warn("Tavily web search failed", zap.Error(tavErr))
provider := resolveWebSearchProvider(chat.PromptConfig)
webResult, webErr := s.retrieveWebSearch(ctx, provider, searchQuestion)
if webErr != nil {
common.Warn("Web search failed", zap.Error(webErr))
} else {
// Extend chunks and doc_aggs with web search results.
if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
if newChunks, ok := tavResult["chunks"].([]map[string]interface{}); ok {
if newChunks, ok := webResult["chunks"].([]map[string]interface{}); ok {
kbinfos["chunks"] = append(existingChunks, newChunks...)
}
}
if existingAggs, ok := kbinfos["doc_aggs"].([]interface{}); ok {
if newAggs, ok := tavResult["doc_aggs"].([]interface{}); ok {
if newAggs, ok := webResult["doc_aggs"].([]interface{}); ok {
kbinfos["doc_aggs"] = append(existingAggs, newAggs...)
}
}
@@ -1755,7 +1755,7 @@ func normalizeInternetFlag(v interface{}) *bool {
// shouldUseWebSearch returns true if web search should be enabled.
// Mirrors Python's _should_use_web_search (dialog_service.py:122-126):
// Tavily key must be present on chat.PromptConfig AND the internet
// A web search provider must be configured on chat.PromptConfig AND the internet
// flag must normalize to explicit true.
//
// The second parameter takes the raw internet value (typically
@@ -1765,8 +1765,7 @@ func (s *ChatPipelineService) shouldUseWebSearch(chat *entity.Chat, internet int
if chat.PromptConfig == nil {
return false
}
tavilyKey, _ := chat.PromptConfig["tavily_api_key"].(string)
if tavilyKey == "" {
if resolveWebSearchProvider(chat.PromptConfig) == nil {
return false
}
normalized := normalizeInternetFlag(internet)

View File

@@ -139,7 +139,7 @@ type DeepResearcher struct {
PromptConfig map[string]interface{}
KBRetrieve KBRetrieveFunc
InternetEnabled bool
TavilyAPIKey string
WebSearch *webSearchProviderConfig
// Fields needed for KG retrieval (mirrors async_chat.go usage).
DocEngine engine.DocEngine
@@ -168,7 +168,7 @@ func NewDeepResearcher(
PromptConfig: promptConfig,
KBRetrieve: kbRetrieve,
InternetEnabled: internetEnabled,
TavilyAPIKey: mapStringValue(promptConfig, "tavily_api_key"),
WebSearch: resolveWebSearchProvider(promptConfig),
DocEngine: docEngine,
KbIDs: kbIDs,
TenantIDs: tenantIDs,
@@ -367,17 +367,17 @@ func (dr *DeepResearcher) retrieveInformation(ctx context.Context, query string)
}
}
// 2. Web retrieval (Tavily)
if dr.InternetEnabled && dr.TavilyAPIKey != "" {
tavRes, err := dr.tavilyRetrieve(ctx, query)
// 2. Web retrieval
if dr.InternetEnabled && dr.WebSearch != nil {
webRes, err := dr.retrieveWebSearch(ctx, dr.WebSearch, query)
if err != nil {
common.Warn("DeepResearcher: web retrieval error", zap.Error(err))
} else if tavRes != nil {
if chunks, ok := tavRes["chunks"].([]map[string]interface{}); ok {
} else if webRes != nil {
if chunks, ok := webRes["chunks"].([]map[string]interface{}); ok {
existing, _ := kbinfos["chunks"].([]map[string]interface{})
kbinfos["chunks"] = append(existing, chunks...)
}
if aggs, ok := tavRes["doc_aggs"].([]interface{}); ok {
if aggs, ok := webRes["doc_aggs"].([]interface{}); ok {
existing, _ := kbinfos["doc_aggs"].([]interface{})
kbinfos["doc_aggs"] = append(existing, aggs...)
}
@@ -410,10 +410,10 @@ func (dr *DeepResearcher) retrieveInformation(ctx context.Context, query string)
}
// tavilyRetrieve calls the Tavily Search API.
func (dr *DeepResearcher) tavilyRetrieve(ctx context.Context, query string) (map[string]interface{}, error) {
func (dr *DeepResearcher) tavilyRetrieve(ctx context.Context, apiKey, query string) (map[string]interface{}, error) {
reqBody := map[string]interface{}{
"query": query,
"api_key": dr.TavilyAPIKey,
"api_key": apiKey,
"search_depth": "advanced",
"max_results": 6,
}
@@ -836,16 +836,6 @@ func getMapString(m map[string]interface{}, keys ...string) string {
return ""
}
// mapStringValue extracts a string value from a map by key.
func mapStringValue(m map[string]interface{}, key string) string {
if v, ok := m[key]; ok {
if s, ok := v.(string); ok {
return s
}
}
return ""
}
// chunksFromKBInfos extracts chunks list from kbinfos for counting.
func chunksFromKBInfos(kbinfos map[string]interface{}) []map[string]interface{} {
if ch, ok := kbinfos["chunks"].([]map[string]interface{}); ok {

View File

@@ -0,0 +1,246 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const (
webSearchProviderTavily = "tavily"
webSearchProviderQuerit = "querit"
queritWebSearchEndpoint = "https://api.querit.ai/v1/search"
)
var queritWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
type webSearchProviderConfig struct {
Provider string
APIKey string
}
func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchProviderConfig {
if promptConfig == nil {
return nil
}
provider := webSearchProviderTavily
if configuredProvider, exists := promptConfig["web_search_provider"]; exists {
var ok bool
provider, ok = configuredProvider.(string)
if !ok {
return nil
}
}
apiKeyField := ""
switch provider {
case webSearchProviderTavily:
apiKeyField = "tavily_api_key"
case webSearchProviderQuerit:
apiKeyField = "querit_api_key"
default:
return nil
}
apiKey, _ := promptConfig[apiKeyField].(string)
apiKey = strings.TrimSpace(apiKey)
if apiKey == "" {
return nil
}
return &webSearchProviderConfig{
Provider: provider,
APIKey: apiKey,
}
}
func (s *ChatPipelineService) retrieveWebSearch(
ctx context.Context,
provider *webSearchProviderConfig,
question string,
) (map[string]interface{}, error) {
if provider == nil {
return nil, fmt.Errorf("web search provider is not configured")
}
switch provider.Provider {
case webSearchProviderTavily:
return s.tavilyRetrieve(ctx, provider.APIKey, question)
case webSearchProviderQuerit:
return retrieveQueritWebSearch(
ctx,
queritWebSearchHTTPClient,
queritWebSearchEndpoint,
provider.APIKey,
question,
)
default:
return nil, fmt.Errorf("unsupported web search provider %q", provider.Provider)
}
}
func (dr *DeepResearcher) retrieveWebSearch(
ctx context.Context,
provider *webSearchProviderConfig,
query string,
) (map[string]interface{}, error) {
if provider == nil {
return nil, fmt.Errorf("web search provider is not configured")
}
switch provider.Provider {
case webSearchProviderTavily:
return dr.tavilyRetrieve(ctx, provider.APIKey, query)
case webSearchProviderQuerit:
return retrieveQueritWebSearch(
ctx,
queritWebSearchHTTPClient,
queritWebSearchEndpoint,
provider.APIKey,
query,
)
default:
return nil, fmt.Errorf("unsupported web search provider %q", provider.Provider)
}
}
type queritWebSearchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
func retrieveQueritWebSearch(
ctx context.Context,
client *http.Client,
endpoint string,
apiKey string,
query string,
) (map[string]interface{}, error) {
requestBody, err := json.Marshal(map[string]interface{}{
"query": query,
"count": 6,
"chunksPerDoc": 1,
})
if err != nil {
return nil, fmt.Errorf("querit: marshal request: %w", err)
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(requestBody))
if err != nil {
return nil, fmt.Errorf("querit: new request: %w", err)
}
request.Header.Set("Accept", "application/json")
request.Header.Set("Authorization", "Bearer "+apiKey)
request.Header.Set("Content-Type", "application/json")
response, err := client.Do(request)
if err != nil {
return nil, fmt.Errorf("querit: do request: %w", err)
}
defer response.Body.Close()
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
return nil, fmt.Errorf("querit: status %d", response.StatusCode)
}
responseBody, err := io.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("querit: read response: %w", err)
}
results, err := decodeQueritWebSearchResults(responseBody)
if err != nil {
return nil, err
}
chunks := make([]map[string]interface{}, 0, len(results))
docAggs := make([]interface{}, 0, len(results))
for _, result := range results {
if result.Snippet == "" {
continue
}
chunkID := "querit-" + result.URL
chunks = append(chunks, map[string]interface{}{
"chunk_id": chunkID,
"content_ltks": tokenizeText(result.Snippet),
"content_with_weight": result.Snippet,
"doc_id": chunkID,
"docnm_kwd": result.Title,
"kb_id": []interface{}{},
"important_kwd": []interface{}{},
"image_id": "",
"similarity": float64(1),
"vector_similarity": float64(1),
"term_similarity": float64(0),
"vector": []float64{},
"positions": []interface{}{},
"url": result.URL,
})
docAggs = append(docAggs, map[string]interface{}{
"doc_name": result.Title,
"doc_id": chunkID,
"count": 1,
"url": result.URL,
})
}
return map[string]interface{}{
"chunks": chunks,
"doc_aggs": docAggs,
}, nil
}
func decodeQueritWebSearchResults(responseBody []byte) ([]queritWebSearchResult, error) {
var envelope map[string]json.RawMessage
if err := json.Unmarshal(responseBody, &envelope); err != nil {
return nil, fmt.Errorf("querit: decode response: %w", err)
}
if envelope == nil {
return nil, fmt.Errorf("querit: response must be an object")
}
resultsValue, exists := envelope["results"]
if !exists {
return []queritWebSearchResult{}, nil
}
if strings.TrimSpace(string(resultsValue)) == "null" {
return nil, fmt.Errorf("querit: response field results must be an object")
}
var resultsContainer map[string]json.RawMessage
if err := json.Unmarshal(resultsValue, &resultsContainer); err != nil {
return nil, fmt.Errorf("querit: response field results must be an object: %w", err)
}
resultValue, exists := resultsContainer["result"]
if !exists {
return []queritWebSearchResult{}, nil
}
if strings.TrimSpace(string(resultValue)) == "null" {
return nil, fmt.Errorf("querit: response field results.result must be an array")
}
var results []queritWebSearchResult
if err := json.Unmarshal(resultValue, &results); err != nil {
return nil, fmt.Errorf("querit: response field results.result must be an array: %w", err)
}
return results, nil
}

View File

@@ -0,0 +1,222 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package service
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestResolveWebSearchProviderUsesExistingTavilyConfig(t *testing.T) {
provider := resolveWebSearchProvider(map[string]interface{}{
"tavily_api_key": "tvly-test",
})
if provider == nil {
t.Fatal("provider is nil")
}
if provider.Provider != webSearchProviderTavily {
t.Fatalf("provider = %q, want %q", provider.Provider, webSearchProviderTavily)
}
if provider.APIKey != "tvly-test" {
t.Fatalf("api key = %q, want %q", provider.APIKey, "tvly-test")
}
}
func TestResolveWebSearchProviderReturnsNilWithoutTavilyKey(t *testing.T) {
cases := []struct {
name string
config map[string]interface{}
}{
{name: "nil config", config: nil},
{name: "empty config", config: map[string]interface{}{}},
{name: "empty key", config: map[string]interface{}{"tavily_api_key": ""}},
{name: "whitespace key", config: map[string]interface{}{"tavily_api_key": " "}},
{name: "non-string key", config: map[string]interface{}{"tavily_api_key": 1}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if provider := resolveWebSearchProvider(tc.config); provider != nil {
t.Fatalf("provider = %+v, want nil", provider)
}
})
}
}
func TestResolveWebSearchProviderUsesSelectedQueritConfig(t *testing.T) {
provider := resolveWebSearchProvider(map[string]interface{}{
"web_search_provider": "querit",
"querit_api_key": "querit-test",
"tavily_api_key": "tvly-test",
})
if provider == nil {
t.Fatal("provider is nil")
}
if provider.Provider != webSearchProviderQuerit {
t.Fatalf("provider = %q, want %q", provider.Provider, webSearchProviderQuerit)
}
if provider.APIKey != "querit-test" {
t.Fatalf("api key = %q, want %q", provider.APIKey, "querit-test")
}
}
func TestResolveWebSearchProviderTrimsSelectedKey(t *testing.T) {
provider := resolveWebSearchProvider(map[string]interface{}{
"web_search_provider": "querit",
"querit_api_key": " querit-test ",
})
if provider == nil {
t.Fatal("provider is nil")
}
if provider.APIKey != "querit-test" {
t.Fatalf("api key = %q, want %q", provider.APIKey, "querit-test")
}
}
func TestResolveWebSearchProviderRequiresKeyForSelectedProvider(t *testing.T) {
cases := []struct {
name string
config map[string]interface{}
}{
{name: "tavily", config: map[string]interface{}{"web_search_provider": "tavily"}},
{name: "querit", config: map[string]interface{}{"web_search_provider": "querit"}},
{
name: "querit whitespace key",
config: map[string]interface{}{
"web_search_provider": "querit",
"querit_api_key": " ",
},
},
{
name: "querit does not fall back to tavily",
config: map[string]interface{}{
"web_search_provider": "querit",
"tavily_api_key": "tvly-test",
},
},
{
name: "unsupported provider",
config: map[string]interface{}{
"web_search_provider": "unsupported",
"querit_api_key": "querit-test",
"tavily_api_key": "tvly-test",
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if provider := resolveWebSearchProvider(tc.config); provider != nil {
t.Fatalf("provider = %+v, want nil", provider)
}
})
}
}
func TestRetrieveQueritWebSearchUsesChatDefaultsAndReturnsReferenceShape(t *testing.T) {
var requestBody map[string]interface{}
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if got := request.Header.Get("Authorization"); got != "Bearer querit-test" {
t.Errorf("Authorization = %q, want %q", got, "Bearer querit-test")
}
if err := json.NewDecoder(request.Body).Decode(&requestBody); err != nil {
t.Errorf("decode request: %v", err)
return
}
response.Header().Set("Content-Type", "application/json")
_, _ = response.Write([]byte(`{
"results": {
"result": [{
"title": "RAGFlow",
"url": "https://example.com/ragflow",
"snippet": "RAGFlow is an open-source RAG engine."
}]
}
}`))
}))
defer server.Close()
result, err := retrieveQueritWebSearch(
context.Background(),
server.Client(),
server.URL,
"querit-test",
"What is RAGFlow?",
)
if err != nil {
t.Fatalf("retrieve Querit web search: %v", err)
}
if requestBody["query"] != "What is RAGFlow?" {
t.Fatalf("query = %#v, want %q", requestBody["query"], "What is RAGFlow?")
}
if requestBody["count"] != float64(6) {
t.Fatalf("count = %#v, want 6", requestBody["count"])
}
if requestBody["chunksPerDoc"] != float64(1) {
t.Fatalf("chunksPerDoc = %#v, want 1", requestBody["chunksPerDoc"])
}
chunks, ok := result["chunks"].([]map[string]interface{})
if !ok || len(chunks) != 1 {
t.Fatalf("chunks = %#v, want one chunk", result["chunks"])
}
if chunks[0]["content_with_weight"] != "RAGFlow is an open-source RAG engine." {
t.Fatalf("content = %#v", chunks[0]["content_with_weight"])
}
if chunks[0]["docnm_kwd"] != "RAGFlow" {
t.Fatalf("title = %#v", chunks[0]["docnm_kwd"])
}
if chunks[0]["url"] != "https://example.com/ragflow" {
t.Fatalf("url = %#v", chunks[0]["url"])
}
if chunks[0]["similarity"] != float64(1) {
t.Fatalf("similarity = %#v, want 1", chunks[0]["similarity"])
}
aggs, ok := result["doc_aggs"].([]interface{})
if !ok || len(aggs) != 1 {
t.Fatalf("doc_aggs = %#v, want one aggregate", result["doc_aggs"])
}
}
func TestDecodeQueritWebSearchResultsRejectsMalformedContainers(t *testing.T) {
cases := []struct {
name string
body string
}{
{name: "null response", body: `null`},
{name: "null results", body: `{"results":null}`},
{name: "array results", body: `{"results":[]}`},
{name: "null result list", body: `{"results":{"result":null}}`},
{name: "object result list", body: `{"results":{"result":{}}}`},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if _, err := decodeQueritWebSearchResults([]byte(tc.body)); err == nil {
t.Fatal("error is nil")
}
})
}
}