mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-04 14:50:30 +08:00
refactor(go-agent): unify tool-backed canvas components (#16912)
## Summary
This PR consolidates Eino-backed Agent tools behind the shared
ToolBackedComponent implementation and aligns their Canvas
configuration,
runtime inputs, output conversion, validation, and registration.
## What changed
- Migrated these Canvas components to the unified tool-backed path:
- Tavily Search and Extract
- Execute SQL
- Google
- Yahoo Finance
- Email
- DuckDuckGo
- Wikipedia
- Google Scholar
- ArXiv
- PubMed
- BGPT
- GitHub
- WenCai
- SearXNG
- Keenable Search
- Removed superseded component wrappers and their duplicate tests.
- Added dedicated registry builders for node-level configuration and
validation.
- Kept model-emitted runtime inputs separate from Canvas node
configuration.
- Moved Email defaults, template resolution, recipient parsing, SMTP
execution, and output conversion into the owning tool.
- Added complete ToolComponent specifications for Canvas inputs,
outputs, and input forms.
- Preserved raw upstream fields where downstream workflows may depend on
them.
- Added workflow registration coverage for all migrated component names.
- Kept HTTP Request, Docs Generator, and Browser as standalone
components because they are not Eino-backed tools.
## Testing
Passed:
bash build.sh --test ./internal/agent/tool/...
bash build.sh --test ./internal/agent/component/...
bash build.sh --test ./internal/agent/runtime/...
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/c728d7a3-9d15-4c5c-b0eb-6b77ad0e41ac"
/>
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/ef13119a-ef92-4b43-9061-eee9511bf492"
/>
<img width="2057" height="1111" alt="image"
src="https://github.com/user-attachments/assets/f50f2c30-eab0-463c-b0b3-0a02523219f1"
/>
This commit is contained in:
@@ -18,14 +18,21 @@ package tool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/cloudwego/eino/components/tool"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
|
||||
"ragflow/internal/tokenizer"
|
||||
)
|
||||
|
||||
const githubToolName = "github_search"
|
||||
@@ -33,10 +40,13 @@ const githubToolName = "github_search"
|
||||
const githubToolDescription = "GitHub repository search finds repositories, projects, and codebases hosted on GitHub."
|
||||
|
||||
const (
|
||||
defaultGitHubTopN = 10
|
||||
maxGitHubTopN = 100
|
||||
defaultGitHubTopN = 10
|
||||
maxGitHubTopN = 100
|
||||
githubPromptMaxTokens = 200000
|
||||
)
|
||||
|
||||
const githubQueryDescription = "The search keywords to execute with GitHub. Use the most important terms and synonyms from the original request."
|
||||
|
||||
// githubParams mirrors Python GitHubParam. Info() exposes only Query to the
|
||||
// model, while TopN is a canvas-side configuration value merged with defaults.
|
||||
type githubParams struct {
|
||||
@@ -66,6 +76,10 @@ type GitHubTool struct {
|
||||
defaults githubParams
|
||||
}
|
||||
|
||||
var _ ToolInvoker = (*GitHubTool)(nil)
|
||||
var _ ToolComponent = (*GitHubTool)(nil)
|
||||
var _ ReferenceBuilder = (*GitHubTool)(nil)
|
||||
|
||||
// NewGitHubTool returns a GitHubTool using the default HTTPHelper.
|
||||
func NewGitHubTool() *GitHubTool {
|
||||
return NewGitHubToolWithDefaults(nil, githubParams{})
|
||||
@@ -101,7 +115,7 @@ func (g *GitHubTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
|
||||
"query": {
|
||||
Type: schema.String,
|
||||
Desc: "The search keywords to execute with GitHub. Use the most important terms and synonyms from the original request.",
|
||||
Desc: githubQueryDescription,
|
||||
Required: true,
|
||||
},
|
||||
}),
|
||||
@@ -130,8 +144,7 @@ func (g *GitHubTool) InvokableRun(ctx context.Context, argsJSON string, _ ...too
|
||||
fmt.Errorf("github: parse arguments: %w", err)
|
||||
}
|
||||
if p.Query == "" {
|
||||
return githubErrJSON(fmt.Errorf("query is required")),
|
||||
fmt.Errorf("github: query is required")
|
||||
return githubJSON(githubEnvelope{Results: []map[string]any{}}), nil
|
||||
}
|
||||
p = mergeGitHubDefaults(g.defaults, p)
|
||||
|
||||
@@ -178,3 +191,135 @@ func githubJSON(env githubEnvelope) string {
|
||||
func githubErrJSON(err error) string {
|
||||
return githubJSON(githubEnvelope{Error: err.Error()})
|
||||
}
|
||||
|
||||
// ComponentSpec returns the Python-compatible GitHub Canvas surface.
|
||||
func (g *GitHubTool) ComponentSpec() ComponentSpec {
|
||||
return ComponentSpec{
|
||||
Inputs: map[string]string{
|
||||
"query": githubQueryDescription,
|
||||
},
|
||||
Outputs: map[string]string{
|
||||
"formalized_content": "GitHub repositories formatted for downstream prompts.",
|
||||
"json": "Raw GitHub repository items.",
|
||||
},
|
||||
InputForm: map[string]any{
|
||||
"query": map[string]any{
|
||||
"type": "line",
|
||||
"name": "Query",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildReferences creates the chunks and document aggregates Python's
|
||||
// ToolBase._retrieve_chunks records for GitHub results.
|
||||
func (g *GitHubTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {
|
||||
return buildGitHubReferences(envelope)
|
||||
}
|
||||
|
||||
func buildGitHubReferences(envelope map[string]any) ([]map[string]any, []map[string]any) {
|
||||
results := envelopeSlice(envelope, "results")
|
||||
chunks := make([]map[string]any, 0, len(results))
|
||||
docAggs := make([]map[string]any, 0, len(results))
|
||||
for _, result := range results {
|
||||
repository, ok := result.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
content := truncateGitHubRunes(githubValueString(repository["description"])+"\n stars:"+githubValueString(repository["watchers"]), 10000)
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
documentID := strconv.FormatInt(githubHashInt(content, 100000000), 10)
|
||||
title := githubValueString(repository["name"])
|
||||
resultURL := githubValueString(repository["html_url"])
|
||||
displayID := strconv.FormatInt(githubHashInt(documentID, 500), 10)
|
||||
chunks = append(chunks, map[string]any{
|
||||
"id": displayID,
|
||||
"chunk_id": documentID,
|
||||
"content": content,
|
||||
"doc_id": documentID,
|
||||
"document_id": documentID,
|
||||
"docnm_kwd": title,
|
||||
"document_name": title,
|
||||
"similarity": 1,
|
||||
"score": 1,
|
||||
"url": resultURL,
|
||||
})
|
||||
docAggs = append(docAggs, map[string]any{
|
||||
"doc_name": title,
|
||||
"doc_id": documentID,
|
||||
"count": 1,
|
||||
"url": resultURL,
|
||||
})
|
||||
}
|
||||
return chunks, docAggs
|
||||
}
|
||||
|
||||
// BuildComponentOutputs constructs GitHub's complete Canvas output map.
|
||||
func (g *GitHubTool) BuildComponentOutputs(envelope map[string]any) map[string]any {
|
||||
results := envelopeSlice(envelope, "results")
|
||||
chunks, _ := buildGitHubReferences(envelope)
|
||||
return map[string]any{
|
||||
"json": results,
|
||||
"formalized_content": renderGitHubReferences(chunks),
|
||||
}
|
||||
}
|
||||
|
||||
func renderGitHubReferences(chunks []map[string]any) string {
|
||||
chunks = limitGitHubReferences(chunks, githubPromptMaxTokens)
|
||||
blocks := make([]string, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
blocks = append(blocks, strings.Join([]string{
|
||||
"\nID: " + githubValueString(chunk["id"]),
|
||||
"├── Title: " + githubValueString(chunk["docnm_kwd"]),
|
||||
"├── URL: " + githubValueString(chunk["url"]),
|
||||
"└── Content:\n" + githubValueString(chunk["content"]),
|
||||
}, "\n"))
|
||||
}
|
||||
return strings.Join(blocks, "\n")
|
||||
}
|
||||
|
||||
func limitGitHubReferences(chunks []map[string]any, maxTokens int) []map[string]any {
|
||||
if maxTokens <= 0 {
|
||||
return nil
|
||||
}
|
||||
usedTokens := 0
|
||||
for index, chunk := range chunks {
|
||||
content := githubValueString(chunk["content"])
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
usedTokens += tokenizer.NumTokensFromString(content)
|
||||
if float64(maxTokens)*0.97 < float64(usedTokens) {
|
||||
return chunks[:index+1]
|
||||
}
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
func githubValueString(value any) string {
|
||||
if value == nil {
|
||||
return "None"
|
||||
}
|
||||
if boolean, ok := value.(bool); ok {
|
||||
if boolean {
|
||||
return "True"
|
||||
}
|
||||
return "False"
|
||||
}
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
|
||||
func githubHashInt(value string, modulus int64) int64 {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
number := new(big.Int).SetBytes(sum[:])
|
||||
return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
|
||||
}
|
||||
|
||||
func truncateGitHubRunes(value string, limit int) string {
|
||||
if utf8.RuneCountInString(value) <= limit {
|
||||
return value
|
||||
}
|
||||
return string([]rune(value)[:limit])
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user