ragflow/internal/agent/tool/github.go

//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package tool

import (
	"context"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"math/big"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/cloudwego/eino/components/tool"
	"github.com/cloudwego/eino/schema"

	"ragflow/internal/tokenizer"
)

const githubToolName = "github_search"

const githubToolDescription = "GitHub repository search finds repositories, projects, and codebases hosted on GitHub."

const (
	defaultGitHubTopN     = 10
	maxGitHubTopN         = 100
	githubPromptMaxTokens = 200000
)

const githubQueryDescription = "The search keywords to execute with GitHub. Use the most important terms and synonyms from the original request."

// githubParams mirrors Python GitHubParam. Info() exposes only Query to the
// model, while TopN is a canvas-side configuration value merged with defaults.
type githubParams struct {
	Query string `json:"query"`
	TopN  int    `json:"top_n"`
}

// githubResponse keeps GitHub's raw repository objects intact. The Python
// component stores response["items"] in its json output, so narrowing this to
// selected fields would change downstream DSL behaviour.
type githubResponse struct {
	Items []map[string]any `json:"items"`
}

// githubEnvelope is the shared tool-to-component transport shape.
type githubEnvelope struct {
	Results []map[string]any `json:"results"`
	Error   string           `json:"_ERROR,omitempty"`
}

// GitHubTool is the GitHub
// repository search tool. It
// GETs the GitHub Search API via the shared HTTPHelper and returns the
// top N repository matches.
type GitHubTool struct {
	helper   *HTTPHelper
	defaults githubParams
}

var _ ToolInvoker = (*GitHubTool)(nil)
var _ ToolComponent = (*GitHubTool)(nil)
var _ ReferenceBuilder = (*GitHubTool)(nil)

// NewGitHubTool returns a GitHubTool using the default HTTPHelper.
func NewGitHubTool() *GitHubTool {
	return NewGitHubToolWithDefaults(nil, githubParams{})
}

// NewGitHubToolWith returns a GitHubTool that uses the provided
// HTTPHelper. Useful for tests.
func NewGitHubToolWith(h *HTTPHelper) *GitHubTool {
	return NewGitHubToolWithDefaults(h, githubParams{})
}

// NewGitHubToolWithDefaults returns a GitHubTool with component-level
// defaults. It follows NewPubMedToolWithDefaults: the constructor owns
// defaults while Info() exposes only the model-call input schema.
func NewGitHubToolWithDefaults(h *HTTPHelper, defaults githubParams) *GitHubTool {
	if h == nil {
		// Python uses DEFAULT_TIMEOUT (15 seconds) and ToolParamBase starts
		// with max_retries=0, so a default GitHub component makes one request.
		h = NewHTTPHelperWithRetry(RetryConfig{MaxAttempts: 1})
		h.client.Timeout = 15 * time.Second
	}
	if defaults.TopN == 0 {
		defaults.TopN = defaultGitHubTopN
	}
	return &GitHubTool{helper: h, defaults: defaults}
}

// Info returns the tool's metadata for the chat model.
func (g *GitHubTool) Info(_ context.Context) (*schema.ToolInfo, error) {
	return &schema.ToolInfo{
		Name: githubToolName,
		Desc: githubToolDescription,
		ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
			"query": {
				Type:     schema.String,
				Desc:     githubQueryDescription,
				Required: true,
			},
		}),
	}, nil
}

// buildGitHubURL constructs the repository search URL used by the Python
// GitHub component: most-starred repositories first, with per_page=top_n.
func buildGitHubURL(query string, topN int) string {
	if topN <= 0 {
		topN = defaultGitHubTopN
	}
	q := url.Values{}
	q.Set("q", query)
	q.Set("sort", "stars")
	q.Set("order", "desc")
	q.Set("per_page", fmt.Sprintf("%d", topN))
	return "https://api.github.com/search/repositories?" + q.Encode()
}

// InvokableRun performs the GitHub repository search.
func (g *GitHubTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) {
	var p githubParams
	if err := json.Unmarshal([]byte(argsJSON), &p); err != nil {
		return githubErrJSON(fmt.Errorf("github: parse arguments: %w", err)),
			fmt.Errorf("github: parse arguments: %w", err)
	}
	if p.Query == "" {
		return githubJSON(githubEnvelope{Results: []map[string]any{}}), nil
	}
	p = mergeGitHubDefaults(g.defaults, p)

	endpoint := buildGitHubURL(p.Query, p.TopN)
	headers := map[string]string{
		"Content-Type":         "application/vnd.github+json",
		"X-GitHub-Api-Version": "2022-11-28",
	}

	resp, err := g.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)
	if err != nil {
		return githubErrJSON(err), err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return githubErrJSON(fmt.Errorf("github: upstream returned %d", resp.StatusCode)),
			fmt.Errorf("github: upstream returned %d", resp.StatusCode)
	}

	var raw githubResponse
	if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
		return githubErrJSON(fmt.Errorf("github: decode response: %w", err)),
			fmt.Errorf("github: decode response: %w", err)
	}
	return githubJSON(githubEnvelope{Results: raw.Items}), nil
}

func mergeGitHubDefaults(defaults, params githubParams) githubParams {
	if params.TopN == 0 {
		params.TopN = defaults.TopN
	}
	return params
}

func githubJSON(env githubEnvelope) string {
	b, err := json.Marshal(env)
	if err != nil {
		return fmt.Sprintf(`{"_ERROR":"github: marshal result: %s"}`, err)
	}
	return string(b)
}

func githubErrJSON(err error) string {
	return githubJSON(githubEnvelope{Error: err.Error()})
}

// ComponentSpec returns the Python-compatible GitHub Canvas surface.
func (g *GitHubTool) ComponentSpec() ComponentSpec {
	return ComponentSpec{
		Inputs: map[string]string{
			"query": githubQueryDescription,
		},
		Outputs: map[string]string{
			"formalized_content": "GitHub repositories formatted for downstream prompts.",
			"json":               "Raw GitHub repository items.",
		},
		InputForm: map[string]any{
			"query": map[string]any{
				"type": "line",
				"name": "Query",
			},
		},
	}
}

// BuildReferences creates the chunks and document aggregates Python's
// ToolBase._retrieve_chunks records for GitHub results.
func (g *GitHubTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {
	return buildGitHubReferences(envelope)
}

func buildGitHubReferences(envelope map[string]any) ([]map[string]any, []map[string]any) {
	results := envelopeSlice(envelope, "results")
	chunks := make([]map[string]any, 0, len(results))
	docAggs := make([]map[string]any, 0, len(results))
	for _, result := range results {
		repository, ok := result.(map[string]any)
		if !ok {
			continue
		}
		content := truncateGitHubRunes(githubValueString(repository["description"])+"\n stars:"+githubValueString(repository["watchers"]), 10000)
		if content == "" {
			continue
		}
		documentID := strconv.FormatInt(githubHashInt(content, 100000000), 10)
		title := githubValueString(repository["name"])
		resultURL := githubValueString(repository["html_url"])
		displayID := strconv.FormatInt(githubHashInt(documentID, 500), 10)
		chunks = append(chunks, map[string]any{
			"id":            displayID,
			"chunk_id":      documentID,
			"content":       content,
			"doc_id":        documentID,
			"document_id":   documentID,
			"docnm_kwd":     title,
			"document_name": title,
			"similarity":    1,
			"score":         1,
			"url":           resultURL,
		})
		docAggs = append(docAggs, map[string]any{
			"doc_name": title,
			"doc_id":   documentID,
			"count":    1,
			"url":      resultURL,
		})
	}
	return chunks, docAggs
}

// BuildComponentOutputs constructs GitHub's complete Canvas output map.
func (g *GitHubTool) BuildComponentOutputs(envelope map[string]any) map[string]any {
	results := envelopeSlice(envelope, "results")
	chunks, _ := buildGitHubReferences(envelope)
	return map[string]any{
		"json":               results,
		"formalized_content": renderGitHubReferences(chunks),
	}
}

func renderGitHubReferences(chunks []map[string]any) string {
	chunks = limitGitHubReferences(chunks, githubPromptMaxTokens)
	blocks := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		blocks = append(blocks, strings.Join([]string{
			"\nID: " + githubValueString(chunk["id"]),
			"├── Title: " + githubValueString(chunk["docnm_kwd"]),
			"├── URL: " + githubValueString(chunk["url"]),
			"└── Content:\n" + githubValueString(chunk["content"]),
		}, "\n"))
	}
	return strings.Join(blocks, "\n")
}

func limitGitHubReferences(chunks []map[string]any, maxTokens int) []map[string]any {
	if maxTokens <= 0 {
		return nil
	}
	usedTokens := 0
	for index, chunk := range chunks {
		content := githubValueString(chunk["content"])
		if content == "" {
			continue
		}
		usedTokens += tokenizer.NumTokensFromString(content)
		if float64(maxTokens)*0.97 < float64(usedTokens) {
			return chunks[:index+1]
		}
	}
	return chunks
}

func githubValueString(value any) string {
	if value == nil {
		return "None"
	}
	if boolean, ok := value.(bool); ok {
		if boolean {
			return "True"
		}
		return "False"
	}
	return fmt.Sprint(value)
}

func githubHashInt(value string, modulus int64) int64 {
	sum := sha1.Sum([]byte(value))
	number := new(big.Int).SetBytes(sum[:])
	return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
}

func truncateGitHubRunes(value string, limit int) string {
	if utf8.RuneCountInString(value) <= limit {
		return value
	}
	return string([]rune(value)[:limit])
}