zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-04 23:00:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: add skills space to context engine (#13908)

Browse Source 
### What problem does this PR solve?

issue #13714

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Yingfeng
2026-04-30 12:36:03 +08:00
committed by GitHub
parent bb3b99f0a5
commit 4ee0702aed
101 changed files with 19161 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
internal/cli/README.md
View File

@@ -7,12 +7,12 @@ This is the Go implementation of the RAGFlow command-line interface, compatible
- Interactive mode and single command execution
- Full compatibility with Python CLI syntax
- Recursive descent parser for SQL-like commands
- Context Engine (Virtual Filesystem) for intuitive resource management
- Virtual Filesystem for intuitive resource management
- Support for all major commands:
- User management: LOGIN, REGISTER, CREATE USER, DROP USER, LIST USERS, etc.
- Service management: LIST SERVICES, SHOW SERVICE, STARTUP/SHUTDOWN/RESTART SERVICE
- Role management: CREATE ROLE, DROP ROLE, LIST ROLES, GRANT/REVOKE PERMISSION
- Dataset management via Context Engine: `ls`, `search`, `mkdir`, `cat`, `rm`
- Dataset management via Virtual Filesystem: `ls`, `search`, `mkdir`, `cat`, `rm`
- Model management: SET/RESET DEFAULT LLM/VLM/EMBEDDING/etc.
- And more...
@@ -30,24 +30,24 @@ go build -o ragflow_cli ./cmd/ragflow_cli.go
```
internal/cli/
├── cli.go # Main CLI loop and interaction
├── client.go # RAGFlowClient with Context Engine integration
├── client.go # RAGFlowClient with Filesystem integration
├── http_client.go # HTTP client for API communication
├── parser/ # Command parser package
│ ├── types.go # Token and Command types
│ ├── lexer.go # Lexical analyzer
│ └── parser.go # Recursive descent parser
└── contextengine/ # Context Engine (Virtual Filesystem)
└── filesystem/ # Virtual Filesystem
├── engine.go # Core engine: path resolution, command routing
├── types.go # Node, Command, Result types
├── provider.go # Provider interface definition
├── dataset_provider.go # Dataset provider implementation
├── file_provider.go # File manager provider implementation
├── base.go # Provider interface definition
├── dataset.go # Dataset provider implementation
├── file.go # File manager provider implementation
└── utils.go # Helper functions
```
## Context Engine
## Virtual Filesystem
The Context Engine provides a unified virtual filesystem interface over RAGFlow's RESTful APIs.
The Virtual Filesystem provides a unified filesystem interface over RAGFlow's RESTful APIs.
### Design Principles
@@ -90,11 +90,7 @@ ls datasets/kb1 -n 50 # List 50 files in kb1 dataset
Semantic search in datasets.
**Options:**
- `-d, --dir <path>` - Directory to search in (can be specified multiple times)
- `-q, --query <query>` - Search query (required)
- `-k, --top-k <number>` - Number of top results to return (default: 10)
- `-t, --threshold <num>` - Similarity threshold, 0.0-1.0 (default: 0.2)
- `-h, --help` - Show search help message
- `-n, --number` - Number of top results to return (default: 10)
**Output Formats:**
- Default: JSON format
@@ -103,10 +99,10 @@ Semantic search in datasets.
**Examples:**
```bash
search -q "machine learning" # Search all datasets (JSON output)
search -d datasets/kb1 -q "neural networks" # Search in kb1
search -d datasets/kb1 -q "AI" --output plain # Plain text output
search -q "RAG" -k 20 -t 0.5 # Return 20 results with threshold 0.5
search "machine learning" # Search all datasets (JSON output)
search "neural networks" datasets/kb1 # Search in kb1
search "AI" datasets/kb1 --output plain # Plain text output
search "RAG" -n 20 # Return 20 results
```
#### `cat <path>` - Display content
@@ -155,20 +151,6 @@ SET DEFAULT LLM 'gpt-4';
SET DEFAULT EMBEDDING 'text-embedding-ada-002';
RESET DEFAULT LLM;
-- Context Engine (Virtual Filesystem)
ls; -- List all datasets (default 10)
ls -n 20; -- List 20 datasets
ls datasets/my_dataset; -- List documents in dataset
ls datasets/my_dataset -n 50; -- List 50 documents
ls datasets/my_dataset/info; -- Show dataset info
search -q "test"; -- Search all datasets (JSON output)
search -d datasets/my_dataset -q "test"; -- Search in specific dataset
-- Meta commands
\? -- Show help
\q -- Quit
\c -- Clear screen
```
## Parser Implementation

447
internal/cli/cli.go
View File

@@ -23,6 +23,7 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
@@ -31,7 +32,7 @@ import (
"github.com/peterh/liner"
"gopkg.in/yaml.v3"
"ragflow/internal/cli/contextengine"
"ragflow/internal/cli/filesystem"
)
// ConfigFile represents the rf.yml configuration file structure
@@ -53,17 +54,19 @@ const (
// ConnectionArgs holds the parsed command line arguments
type ConnectionArgs struct {
Host string
Port int
Password string
APIToken string
UserName string
Command *string // Original command string (for SQL mode)
CommandArgs []string // Split command arguments (for ContextEngine mode)
IsSQLMode bool // true=SQL mode (quoted), false=ContextEngine mode (unquoted)
ShowHelp bool
AdminMode bool
OutputFormat OutputFormat // Output format: table, plain, json
Host string
Port int
Password string
APIToken string
UserName string
ConfigFilePath string // Path to the config file (e.g., rf.yml)
Command *string // Original command string (for SQL mode)
CommandArgs []string // Split command arguments (for ContextEngine mode)
IsSQLMode bool // true=SQL mode (quoted), false= ContextEngine mode (unquoted)
ShowHelp bool
AdminMode bool
OutputFormat OutputFormat // Output format: table, plain, json
Verbose bool // Enable verbose logging
}
// LoadDefaultConfigFile reads the rf.yml file from current directory if it exists
@@ -124,9 +127,10 @@ func parseHostPort(hostPort string) (string, int, error) {
// ParseConnectionArgs parses command line arguments similar to Python's parse_connection_args
func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
// First, scan args to check for help, config file, and admin mode
// First, scan args to check for help, config file, admin mode, and verbose flag
var configFilePath string
var adminMode bool = false
var verboseMode bool = false
foundCommand := false
for i := 0; i < len(args); i++ {
arg := args[i]
@@ -138,9 +142,16 @@ func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
}
// Only process --help as global help if it's before any command
if !foundCommand && (arg == "--help" || arg == "-help") {
return &ConnectionArgs{ShowHelp: true}, nil
return &ConnectionArgs{ShowHelp: true, Verbose: verboseMode}, nil
} else if (arg == "-f" || arg == "--config") && i+1 < len(args) {
configFilePath = args[i+1]
// Convert to absolute path immediately
if !filepath.IsAbs(configFilePath) {
absPath, err := filepath.Abs(configFilePath)
if err == nil {
configFilePath = absPath
}
}
i++
} else if (arg == "-o" || arg == "--output") && i+1 < len(args) {
// -o/--output is allowed with config file, skip it and its value
@@ -148,6 +159,8 @@ func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
continue
} else if arg == "--admin" {
adminMode = true
} else if arg == "-v" || arg == "--verbose" {
verboseMode = true
}
}
@@ -158,7 +171,10 @@ func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
// Parse arguments manually to support both short and long forms
// and to handle priority: command line > config file > defaults
result := &ConnectionArgs{}
result := &ConnectionArgs{
Verbose: verboseMode,
ConfigFilePath: configFilePath,
}
if !adminMode {
// Only user mode read config file
@@ -256,6 +272,8 @@ func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
}
i++
}
case "-v", "--verbose":
result.Verbose = true
case "--admin", "-admin":
result.AdminMode = true
case "--help", "-help":
@@ -303,12 +321,24 @@ func ParseConnectionArgs(args []string) (*ConnectionArgs, error) {
}
}
// Get command from remaining args (non-flag arguments)
// Get command from remaining args (non-flag arguments)
if len(nonFlagArgs) > 0 {
command := strings.Join(nonFlagArgs, " ")
result.Command = &command
fmt.Printf("COMMAND: %s\n", command)
// Check if this is SQL mode or ContextEngine mode
// SQL mode: single argument that looks like SQL (e.g., "LIST DATASETS")
// ContextEngine mode: multiple arguments (e.g., "ls", "datasets")
if len(nonFlagArgs) == 1 && looksLikeSQL(nonFlagArgs[0]) {
// SQL mode: single argument that looks like SQL
result.IsSQLMode = true
command := nonFlagArgs[0]
result.Command = &command
} else {
// ContextEngine mode: multiple arguments
result.IsSQLMode = false
result.CommandArgs = nonFlagArgs
// Also store joined version for backward compatibility
command := strings.Join(nonFlagArgs, " ")
result.Command = &command
}
}
return result, nil
@@ -345,6 +375,7 @@ Options:
-p, --password string Password for authentication
-f, --config string Path to config file (YAML format)
-o, --output string Output format: table, plain, json (search defaults to json)
-v, --verbose Enable verbose logging (shows debug info)
--admin, -admin Run in admin mode
--help Show this help message
@@ -373,7 +404,11 @@ Configuration File:
Commands:
SQL commands (use quotes): "LIST USERS", "CREATE USER 'email' 'password'", etc.
Context Engine commands (no quotes): ls datasets, search "keyword", cat path, etc.
Filesystem commands (no quotes): ls datasets, search "keyword", cat path, etc.
Skill commands:
install-skill <space> <path|url> [options] Install a skill from local path or remote URL
uninstall-skill <space> <skill-name> Remove an installed skill
search skills -q <query> [--space space1] Search skills in a space
If no command is provided, CLI runs in interactive mode.`)
}
@@ -386,13 +421,13 @@ const historyFileName = ".ragflow_cli_history"
// CLI represents the command line interface
type CLI struct {
client *RAGFlowClient
contextEngine *contextengine.Engine
prompt string
running bool
line *liner.State
args *ConnectionArgs
outputFormat OutputFormat // Output format
client *RAGFlowClient
contextEngine *filesystem.Engine
prompt string
running bool
line *liner.State
args *ConnectionArgs
outputFormat OutputFormat // Output format
}
// NewCLI creates a new CLI instance
@@ -451,10 +486,11 @@ func NewCLIWithArgs(args *ConnectionArgs) (*CLI, error) {
prompt = "RAGFlow(admin)> "
}
// Create context engine and register providers
engine := contextengine.NewEngine()
engine.RegisterProvider(contextengine.NewDatasetProvider(&httpClientAdapter{client: client.HTTPClient}))
engine.RegisterProvider(contextengine.NewFileProvider(&httpClientAdapter{client: client.HTTPClient}))
// Create filesystem engine and register providers
engine := filesystem.NewEngine()
engine.RegisterProvider(filesystem.NewDatasetProvider(&httpClientAdapter{client: client.HTTPClient}))
engine.RegisterProvider(filesystem.NewFileProvider(&httpClientAdapter{client: client.HTTPClient}))
engine.RegisterProvider(filesystem.NewSkillProvider(&httpClientAdapter{client: client.HTTPClient}))
return &CLI{
prompt: prompt,
@@ -587,7 +623,7 @@ func (c *CLI) execute(input string) error {
}
}
// Check if we should use SQL mode or ContextEngine mode
// Check if we should use SQL mode or Filesystem mode
isSQLMode := false
if c.args != nil && len(c.args.CommandArgs) > 0 {
// Non-interactive mode: use pre-determined mode from args
@@ -617,12 +653,12 @@ func (c *CLI) execute(input string) error {
return err
}
// ContextEngine mode: execute context engine command
return c.executeContextEngine(input)
// Filesystem mode: execute filesystem command
return c.executeFilesystem(input)
}
// executeContextEngine executes a Context Engine command
func (c *CLI) executeContextEngine(input string) error {
// executeFilesystem executes a Filesystem command
func (c *CLI) executeFilesystem(input string) error {
// Parse input into arguments
var args []string
if c.args != nil && len(c.args.CommandArgs) > 0 {
@@ -630,23 +666,23 @@ func (c *CLI) executeContextEngine(input string) error {
args = c.args.CommandArgs
} else {
// Interactive mode: parse input
args = parseContextEngineArgs(input)
args = parseFilesystemArgs(input)
}
if len(args) == 0 {
return fmt.Errorf("no command provided")
}
// Check if we have a context engine
// Check if we have a filesystem engine
if c.contextEngine == nil {
return fmt.Errorf("context engine not available")
return fmt.Errorf("filesystem engine not available")
}
cmdType := args[0]
cmdArgs := args[1:]
// Build context engine command
var ceCmd *contextengine.Command
// Build filesystem command
var ceCmd *filesystem.Command
switch cmdType {
case "ls", "list":
@@ -659,8 +695,8 @@ func (c *CLI) executeContextEngine(input string) error {
// Help was printed
return nil
}
ceCmd = &contextengine.Command{
Type: contextengine.CommandList,
ceCmd = &filesystem.Command{
Type: filesystem.CommandList,
Path: listOpts.Path,
Params: map[string]interface{}{
"limit": listOpts.Limit,
@@ -682,8 +718,45 @@ func (c *CLI) executeContextEngine(input string) error {
if len(searchOpts.Dirs) > 0 {
searchPath = searchOpts.Dirs[0]
}
ceCmd = &contextengine.Command{
Type: contextengine.CommandSearch,
// Check if searching skills (supports: "skills" or "skills/space1")
if searchPath == "skills" || strings.HasPrefix(searchPath, "skills/") {
// Parse space ID from path (e.g., "skills/space1" -> "space1")
spaceID := "default"
if strings.HasPrefix(searchPath, "skills/") {
spaceID = strings.TrimPrefix(searchPath, "skills/")
if spaceID == "" {
spaceID = "default"
}
}
// Get skill provider and perform search
provider := c.contextEngine.GetProvider("skills")
if provider == nil {
return fmt.Errorf("skill provider not available")
}
skillProvider, ok := provider.(*filesystem.SkillProvider)
if !ok {
return fmt.Errorf("invalid skill provider type")
}
pageSize := searchOpts.TopK
if pageSize <= 0 {
pageSize = 10
}
searchOptions := &filesystem.SearchOptions{
Query: searchOpts.Query,
Limit: pageSize,
Offset: 0,
TopK: pageSize,
}
result, err := skillProvider.Search(context.Background(), spaceID, searchOptions)
if err != nil {
return err
}
// Print skill search results with full details
c.printSkillSearchResults(result, c.outputFormat)
return nil
}
ceCmd = &filesystem.Command{
Type: filesystem.CommandSearch,
Path: searchPath,
Params: map[string]interface{}{
"query": searchOpts.Query,
@@ -709,8 +782,66 @@ func (c *CLI) executeContextEngine(input string) error {
fmt.Println(string(content))
return nil
case "install-skill":
// Get the file provider and skill provider from the engine
fileProvider, ok := c.contextEngine.GetProvider("files").(*filesystem.FileProvider)
if !ok {
return fmt.Errorf("file provider not available")
}
skillProvider := c.contextEngine.GetProvider("skills")
if skillProvider == nil {
return fmt.Errorf("skill provider not available")
}
// Create adapter for HTTPClient
httpAdapter := &httpClientAdapter{client: c.client.HTTPClient}
cmd := filesystem.NewInstallSkillCommand(httpAdapter, fileProvider, skillProvider)
return cmd.Execute(cmdArgs)
case "uninstall-skill":
skillProvider := c.contextEngine.GetProvider("skills")
if skillProvider == nil {
return fmt.Errorf("skill provider not available")
}
fileProvider := c.contextEngine.GetProvider("files")
if fileProvider == nil {
return fmt.Errorf("file provider not available")
}
// Create adapter for HTTPClient
httpAdapter := &httpClientAdapter{client: c.client.HTTPClient}
fileProv, _ := fileProvider.(*filesystem.FileProvider)
cmd := filesystem.NewUninstallSkillCommand(httpAdapter, skillProvider, fileProv)
return cmd.Execute(cmdArgs)
case "add-skill":
fmt.Println("⚠ Warning: 'add-skill' is deprecated. Use 'install-skill' instead.")
// Forward to install-skill
fileProvider, ok := c.contextEngine.GetProvider("files").(*filesystem.FileProvider)
if !ok {
return fmt.Errorf("file provider not available")
}
skillProvider := c.contextEngine.GetProvider("skills")
if skillProvider == nil {
return fmt.Errorf("skill provider not available")
}
httpAdapter := &httpClientAdapter{client: c.client.HTTPClient}
cmd := filesystem.NewInstallSkillCommand(httpAdapter, fileProvider, skillProvider)
return cmd.Execute(cmdArgs)
case "delete-skill":
fmt.Println("⚠ Warning: 'delete-skill' is deprecated. Use 'uninstall-skill' instead.")
// Forward to uninstall-skill
skillProvider := c.contextEngine.GetProvider("skills")
if skillProvider == nil {
return fmt.Errorf("skill provider not available")
}
fileProvider := c.contextEngine.GetProvider("files")
if fileProvider == nil {
return fmt.Errorf("file provider not available")
}
httpAdapter := &httpClientAdapter{client: c.client.HTTPClient}
fileProv, _ := fileProvider.(*filesystem.FileProvider)
cmd := filesystem.NewUninstallSkillCommand(httpAdapter, skillProvider, fileProv)
return cmd.Execute(cmdArgs)
default:
return fmt.Errorf("unknown context engine command: %s", cmdType)
return fmt.Errorf("unknown filesystem command: %s", cmdType)
}
// Execute the command
@@ -722,23 +853,23 @@ func (c *CLI) executeContextEngine(input string) error {
// Print result
// For search command, default to JSON format if not explicitly set to plain/table
format := c.outputFormat
if ceCmd.Type == contextengine.CommandSearch && format != OutputFormatPlain && format != OutputFormatTable {
if ceCmd.Type == filesystem.CommandSearch && format != OutputFormatPlain && format != OutputFormatTable {
format = OutputFormatJSON
}
// Get limit for list command
limit := 0
if ceCmd.Type == contextengine.CommandList {
if ceCmd.Type == filesystem.CommandList {
if l, ok := ceCmd.Params["limit"].(int); ok {
limit = l
}
}
c.printContextEngineResult(result, ceCmd.Type, format, limit)
c.printFilesystemResult(result, ceCmd.Type, format, limit)
return nil
}
// parseContextEngineArgs parses Context Engine command arguments
// parseFilesystemArgs parses Filesystem command arguments
// Supports simple space-separated args and quoted strings
func parseContextEngineArgs(input string) []string {
func parseFilesystemArgs(input string) []string {
var args []string
var current strings.Builder
inQuote := false
@@ -780,14 +911,14 @@ func parseContextEngineArgs(input string) []string {
return args
}
// printContextEngineResult prints the result of a context engine command
func (c *CLI) printContextEngineResult(result *contextengine.Result, cmdType contextengine.CommandType, format OutputFormat, limit int) {
// printFilesystemResult prints the result of a filesystem command
func (c *CLI) printFilesystemResult(result *filesystem.Result, cmdType filesystem.CommandType, format OutputFormat, limit int) {
if result == nil {
return
}
switch cmdType {
case contextengine.CommandList:
case filesystem.CommandList:
if len(result.Nodes) == 0 {
fmt.Println("(empty)")
return
@@ -824,7 +955,7 @@ func (c *CLI) printContextEngineResult(result *contextengine.Result, cmdType con
fmt.Printf("\n... and %d more (use -n to show more)\n", result.Total-limit)
}
fmt.Printf("Total: %d\n", result.Total)
case contextengine.CommandSearch:
case filesystem.CommandSearch:
if len(result.Nodes) == 0 {
if format == OutputFormatJSON {
fmt.Println("[]")
@@ -921,13 +1052,103 @@ func (c *CLI) printContextEngineResult(result *contextengine.Result, cmdType con
fmt.Println(sep)
fmt.Printf("Total: %d\n", result.Total)
}
case contextengine.CommandCat:
case filesystem.CommandCat:
// Cat output is handled differently - it returns []byte, not *Result
// This case should not be reached in normal flow since Cat returns []byte directly
fmt.Println("Content retrieved")
}
}
// printSkillSearchResults prints skill search results with full details
func (c *CLI) printSkillSearchResults(result *filesystem.Result, format OutputFormat) {
if result == nil || len(result.Nodes) == 0 {
if format == OutputFormatJSON {
fmt.Println("[]")
} else {
fmt.Println("No skills found")
}
return
}
// Skill search result structure
type skillSearchResult struct {
SkillID string `json:"skill_id"`
Name string `json:"name"`
Description string `json:"description"`
Tags string `json:"tags"`
Score float64 `json:"score"`
BM25Score float64 `json:"bm25_score"`
VectorScore float64 `json:"vector_score"`
}
results := make([]skillSearchResult, 0, len(result.Nodes))
for _, node := range result.Nodes {
// Extract metadata
skillID := ""
if id, ok := node.Metadata["skill_id"].(string); ok {
skillID = id
}
description := ""
if desc, ok := node.Metadata["description"].(string); ok {
description = desc
}
tags := ""
if t, ok := node.Metadata["tags"].([]string); ok {
tags = strings.Join(t, ", ")
}
var score, bm25Score, vectorScore float64
if s, ok := node.Metadata["score"].(float64); ok {
score = s
}
if b, ok := node.Metadata["bm25_score"].(float64); ok {
bm25Score = b
}
if v, ok := node.Metadata["vector_score"].(float64); ok {
vectorScore = v
}
results = append(results, skillSearchResult{
SkillID: skillID,
Name: node.Name,
Description: description,
Tags: tags,
Score: score,
BM25Score: bm25Score,
VectorScore: vectorScore,
})
}
if format == OutputFormatJSON {
jsonData, err := json.MarshalIndent(results, "", " ")
if err != nil {
fmt.Printf("Error marshaling JSON: %v\n", err)
return
}
fmt.Println(string(jsonData))
} else if format == OutputFormatPlain {
fmt.Printf("Found %d skill(s):\n", len(results))
for _, sr := range results {
fmt.Printf("\nName: %s\n", sr.Name)
fmt.Printf("Skill ID: %s\n", sr.SkillID)
fmt.Printf("Description: %s\n", sr.Description)
fmt.Printf("Tags: %s\n", sr.Tags)
fmt.Printf("Score: %.6f (BM25: %.6f, Vector: %.6f)\n", sr.Score, sr.BM25Score, sr.VectorScore)
}
} else {
// Table format
fmt.Printf("Found %d skill(s):\n", len(results))
fmt.Println()
for _, sr := range results {
fmt.Printf("Name: %s\n", sr.Name)
fmt.Printf("Skill ID: %s\n", sr.SkillID)
fmt.Printf("Description: %s\n", sr.Description)
fmt.Printf("Tags: %s\n", sr.Tags)
fmt.Printf("Score: %.6f (BM25: %.6f, Vector: %.6f)\n", sr.Score, sr.BM25Score, sr.VectorScore)
fmt.Println()
}
}
}
func (c *CLI) handleMetaCommand(cmd *Command) error {
command := cmd.Params["command"].(string)
args, _ := cmd.Params["args"].([]string)
@@ -1021,7 +1242,7 @@ Commands (User Mode):
CHAT 'message'; - Chat using current model
CHAT 'provider/instance/model' 'message'; - Chat with specified model
Context Engine Commands (no quotes):
Filesystem Commands (no quotes):
ls [path] - List resources
e.g., ls - List root (providers and folders)
e.g., ls datasets - List all datasets
@@ -1036,7 +1257,7 @@ Context Engine Commands (no quotes):
Examples:
ragflow_cli -f rf.yml "LIST USERS" # SQL mode (with quotes)
ragflow_cli -f rf.yml ls datasets # Context Engine mode (no quotes)
ragflow_cli -f rf.yml ls datasets # Filesystem mode (no quotes)
ragflow_cli -f rf.yml ls files # List files in root
ragflow_cli -f rf.yml cat datasets # Error: datasets is a directory
ragflow_cli -f rf.yml ls files/myfolder # List folder contents
@@ -1079,7 +1300,7 @@ func (c *CLI) RunSingleCommand(command *string) error {
defer c.Cleanup()
// Execute the command
if err := c.executeNew(*command); err != nil {
if err := c.execute(*command); err != nil {
return err
}
return nil
@@ -1141,7 +1362,7 @@ type ListCommandOptions struct {
}
// parseSearchCommandArgs parses search command arguments
// Format: search [-d dir1] [-d dir2] ... -q query [-k top_k] [-t threshold]
// Format: search <query> [path] [-n number]
//
// search -h|--help (shows help)
func parseSearchCommandArgs(args []string) (*SearchCommandOptions, error) {
@@ -1160,77 +1381,45 @@ func parseSearchCommandArgs(args []string) (*SearchCommandOptions, error) {
}
// Parse arguments
// Format: search <query> [path] [-n number]
i := 0
for i < len(args) {
arg := args[i]
switch arg {
case "-d", "--dir":
if i+1 >= len(args) {
return nil, fmt.Errorf("missing value for %s flag", arg)
}
opts.Dirs = append(opts.Dirs, args[i+1])
i += 2
case "-q", "--query":
if i+1 >= len(args) {
return nil, fmt.Errorf("missing value for %s flag", arg)
}
opts.Query = args[i+1]
i += 2
case "-k", "--top-k":
// Handle -n flag for number of results
if arg == "-n" || arg == "--number" {
if i+1 >= len(args) {
return nil, fmt.Errorf("missing value for %s flag", arg)
}
topK, err := strconv.Atoi(args[i+1])
if err != nil {
return nil, fmt.Errorf("invalid top-k value: %s", args[i+1])
return nil, fmt.Errorf("invalid number value: %s", args[i+1])
}
opts.TopK = topK
i += 2
case "-t", "--threshold":
if i+1 >= len(args) {
return nil, fmt.Errorf("missing value for %s flag", arg)
}
threshold, err := strconv.ParseFloat(args[i+1], 64)
if err != nil {
return nil, fmt.Errorf("invalid threshold value: %s", args[i+1])
}
opts.Threshold = threshold
i += 2
default:
// If it doesn't start with -, it might be a positional argument
if !strings.HasPrefix(arg, "-") {
// For backwards compatibility: if no -q flag and this is the last arg, treat as query
if opts.Query == "" && i == len(args)-1 {
opts.Query = arg
} else if opts.Query == "" && len(args) > 0 && i < len(args)-1 {
// Old format: search [path] query
// Treat first non-flag as path, rest as query
opts.Dirs = append(opts.Dirs, arg)
// Join remaining args as query
remainingArgs := args[i+1:]
queryParts := []string{}
for _, part := range remainingArgs {
if !strings.HasPrefix(part, "-") {
queryParts = append(queryParts, part)
}
}
opts.Query = strings.Join(queryParts, " ")
break
}
} else {
return nil, fmt.Errorf("unknown flag: %s", arg)
}
i++
continue
}
// If it starts with -, it's an unknown flag
if strings.HasPrefix(arg, "-") {
return nil, fmt.Errorf("unknown flag: %s", arg)
}
// Non-flag arguments: first is query, second is path
if opts.Query == "" {
opts.Query = arg
} else if len(opts.Dirs) == 0 {
opts.Dirs = append(opts.Dirs, arg)
}
i++
}
// Validate required parameters
if opts.Query == "" {
return nil, fmt.Errorf("query is required (use -q or --query)")
return nil, fmt.Errorf("query is required")
}
// If no directories specified, search in all datasets (empty path means all)
// If no path specified, default to "datasets"
if len(opts.Dirs) == 0 {
opts.Dirs = []string{"datasets"}
}
@@ -1240,30 +1429,34 @@ func parseSearchCommandArgs(args []string) (*SearchCommandOptions, error) {
// printSearchHelp prints help for the search command
func printSearchHelp() {
help := `Search command usage: search [options]
help := `Search command usage: search <query> [path] [-n number]
Search for content in datasets. Currently only supports searching in datasets.
Search for content in datasets or skills.
Arguments:
<query> Search query (required)
Example: "machine learning"
[path] Path to search in (default: datasets)
Supports:
- 'datasets' (all datasets)
- 'datasets/<kb_name>' (specific dataset)
- 'skills' (default skill space)
- 'skills/<space_name>' (specific skill space)
Example: skills/space1
Options:
-d, --dir <path> Directory to search in (can be specified multiple times)
Currently only supports paths under 'datasets/'
Example: -d datasets/kb1 -d datasets/kb2
-q, --query <query> Search query (required)
Example: -q "machine learning"
-k, --top-k <number> Number of top results to return (default: 10)
Example: -k 20
-t, --threshold <num> Similarity threshold, 0.0-1.0 (default: 0.2)
Example: -t 0.5
-n, --number <num> Number of results to return (default: 10)
Example: -n 20
-h, --help Show this help message
Output:
Default output format is JSON. Use --output plain or --output table for other formats.
Examples:
search -d datasets/kb1 -q "neural networks" # Search in kb1 (JSON output)
search -d datasets/kb1 -q "AI" --output plain # Search with plain text output
search -q "data mining" # Search all datasets
search -q "RAG" -k 20 -t 0.5 # Return 20 results with threshold 0.5
search "neural networks" # Search all datasets
search "AI" datasets/kb1 # Search in kb1
search "RAG" skills/space1 -n 20 # Search skills in hub1, return 20 results
search "data processing" skills # Search skills (default space)
`
fmt.Println(help)
}

19
internal/cli/client.go
View File

@@ -18,7 +18,9 @@ package cli
import (
"fmt"
ce "ragflow/internal/cli/contextengine"
"io"
ce "ragflow/internal/cli/filesystem"
)
// PasswordPromptFunc is a function type for password input
@@ -41,7 +43,6 @@ type RAGFlowClient struct {
CurrentModel *CurrentModel // Current model configuration
}
// NewRAGFlowClient creates a new RAGFlow client
func NewRAGFlowClient(serverType string) *RAGFlowClient {
httpClient := NewHTTPClient()
// Set port from configuration file based on server type
@@ -68,6 +69,8 @@ func (c *RAGFlowClient) initContextEngine() {
// Register providers
engine.RegisterProvider(ce.NewDatasetProvider(&httpClientAdapter{c.HTTPClient}))
engine.RegisterProvider(ce.NewFileProvider(&httpClientAdapter{c.HTTPClient}))
engine.RegisterProvider(ce.NewSkillProvider(&httpClientAdapter{c.HTTPClient}))
c.ContextEngine = engine
}
@@ -101,6 +104,10 @@ func (a *httpClientAdapter) Request(method, path string, useAPIBase bool, authKi
}, nil
}
func (a *httpClientAdapter) UploadMultipart(path string, contentType string, body io.Reader) error {
return a.client.UploadMultipart(path, contentType, body)
}
// ExecuteCommand executes a parsed command
// Returns benchmark result map for commands that support it (e.g., ping_server with iterations > 1)
func (c *RAGFlowClient) ExecuteCommand(cmd *Command) (ResponseIf, error) {
@@ -288,14 +295,10 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
case "remove_chunks":
return c.RemoveChunks(cmd)
// ContextEngine commands
case "context_list":
return c.ContextList(cmd)
case "context_cat":
return c.ContextCat(cmd)
case "context_search":
return c.ContextSearch(cmd)
case "ce_ls":
return c.CEList(cmd)
case "ce_cat":
return c.CECat(cmd)
case "ce_search":
return c.CESearch(cmd)
// TODO: Implement other commands

135
internal/cli/context_command.go
View File

@@ -1,135 +0,0 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package cli
import (
"fmt"
)
func (c *RAGFlowClient) ContextList(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
var path string
var ok bool
if cmd.Params["path"] != nil {
path, ok = cmd.Params["path"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'path' to string")
}
}
if path == "" {
path = "."
}
var parameter string
if cmd.Params["parameter"] != nil {
parameter, ok = cmd.Params["parameter"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'parameter' to string")
}
}
if parameter == "" {
fmt.Printf("ls %s\n", path)
} else {
fmt.Printf("ls %s -%s\n", path, parameter)
}
// Convert to response
var response ContextListResponse
response.OutputFormat = c.OutputFormat
response.Code = 0
response.Data = nil
return &response, nil
}
func (c *RAGFlowClient) ContextCat(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
path, ok := cmd.Params["filename"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'filename' to string")
}
fmt.Printf("cat %s\n", path)
// Convert to response
var response ContextListResponse
response.OutputFormat = c.OutputFormat
response.Code = 0
response.Data = nil
return &response, nil
}
func (c *RAGFlowClient) ContextSearch(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
path, ok := cmd.Params["path"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'path' to string")
}
query, ok := cmd.Params["query"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'parameter' to float64")
}
number := 10
if cmd.Params["number"] != nil {
number, ok = cmd.Params["number"].(int)
if !ok {
return nil, fmt.Errorf("fail to convert 'number' to int")
}
}
//threshold := 0.0
//if cmd.Params["threshold"] != nil {
// threshold, ok = cmd.Params["threshold"].(float64)
// if !ok {
// return nil, fmt.Errorf("fail to convert 'threshold' to float64")
// }
//}
fmt.Printf("search query: %s, path: %s, number: %d\n", query, path, number)
// Convert to response
var response ContextSearchResponse
response.OutputFormat = c.OutputFormat
response.Code = 0
response.Total = 0
response.Data = nil
return &response, nil
}

6
internal/cli/context_parser.go
View File

@@ -24,7 +24,7 @@ import (
func (p *Parser) parseContextListCommand() (*Command, error) {
p.nextToken() // consume LS
cmd := NewCommand("context_list")
cmd := NewCommand("ce_ls")
if p.curToken.Type == TokenEOF {
cmd.Params["path"] = "."
@@ -70,7 +70,7 @@ func (p *Parser) parseContextCatCommand() (*Command, error) {
return nil, fmt.Errorf("expect a filename")
}
cmd := NewCommand("context_cat")
cmd := NewCommand("ce_cat")
if p.curToken.Type == TokenIdentifier {
for p.curToken.Type != TokenEOF {
if p.curToken.Type != TokenIdentifier {
@@ -114,7 +114,7 @@ func (p *Parser) parseContextCatCommand() (*Command, error) {
func (p *Parser) parseContextSearchCommand() (*Command, error) {
p.nextToken() // consume SEARCH
cmd := NewCommand("context_search")
cmd := NewCommand("ce_search")
for p.curToken.Type != TokenEOF {
if p.curToken.Type == TokenDash {

49
internal/cli/contextengine/README.md
View File

@@ -1,49 +0,0 @@
# ContextFS - Context Engine File System
ContextFS is a context engine interface for RAGFlow, providing users with a Unix-like file system interface to manage datasets, tools, skills, and memories.
## Directory Structure
```
user_id/
├── datasets/
│ └── my_dataset/
│ └── ...
├── tools/
│ ├── registry.json
│ └── tool_name/
│ ├── DOC.md
│ └── ...
├── skills/
│ ├── registry.json
│ └── skill_name/
│ ├── SKILL.md
│ └── ...
└── memories/
└── memory_id/
├── sessions/
│ ├── messages/
│ ├── summaries/
│ │ └── session_id/
│ │ └── summary-{datetime}.md
│ └── tools/
│ └── session_id/
│ └── {tool_name}.md # User level of memory on Tools usage
├── users/
│ ├── profile.md
│ ├── preferences/
│ └── entities/
└── agents/
└── agent_space/
├── tools/
│ └── {tool_name}.md # Agent level of memory on Tools usage
└── skills/
└── {skill_name}.md # Agent level of memory on Skills usage
```
## Supported Commands
- `ls [path]` - List directory contents
- `cat <path>` - Display file contents(only for text files)
- `search <query>` - Search content

195
internal/cli/filesystem/README.md Normal file
View File

@@ -0,0 +1,195 @@
# ContextEngine Filesystem
The ContextEngine Filesystem is a filesystem interface for RAGFlow, providing users with a Unix-like file system interface to manage datasets, tools, skills, and memories.
## Directory Structure
```
user_id/
├── datasets/
│ └── my_dataset/
│ └── ...
├── tools/
│ ├── registry.json
│ └── tool_name/
│ ├── DOC.md
│ └── ...
├── skills/
│ └── skill_name/
│ └── version
. ├──SKILL.md
. └── ...
└── memories/
└── memory_id/
├── sessions/
│ ├── messages/
│ ├── summaries/
│ │ └── session_id/
│ │ └── summary-{datetime}.md
│ └── tools/
│ └── session_id/
│ └── {tool_name}.md # User level of memory on Tools usage
├── users/
│ ├── profile.md
│ ├── preferences/
│ └── entities/
└── agents/
└── agent_space/
├── tools/
│ └── {tool_name}.md # Agent level of memory on Tools usage
└── skills/
└── {skill_name}.md # Agent level of memory on Skills usage
```
## Supported Commands
- `ls [path]` - List directory contents
- `cat <path>` - Display file contents(only for text files)
- `search <query> path` - Search content
- `install-skill <space> <source> [options]` - Install a skill from multiple sources
- `uninstall-skill <space> <skill-name>` - Uninstall a skill
### Skill Management Commands
#### install-skill
Install a skill from multiple sources into a RAGFlow space.
**Usage:**
```bash
install-skill <space> <source> [options]
```
**Arguments:**
- `<space>` - Target skills space ID (required)
- `<source>` - Skill source reference (required)
**Supported Sources:**
| Source Type | Format | Example |
|------------|--------|---------|
| **Local** | `./path` or `/absolute/path` | `./my-skill`, `/home/user/skills/awesome` |
| **GitHub** | `github.com/owner/repo/path` | `github.com/openai/skills/skill-creator` |
| **ClawHub** | `clawhub://owner/skill-name` or `clawhub.ai/owner/skill-name` | `clawhub://pskoett/self-improving-agent` |
| **skills.sh** | `skill://skill-name` or `skills.sh/skill/name` | `skill://kubernetes` |
**Options:**
- `-v, --version <version>` - Specify skill version (default: from SKILL.md or 1.0.0)
- `-n, --name <name>` - Override skill name (default: from SKILL.md)
- `-f, --force` - Force reinstall if skill exists (deletes existing first and updates index)
- `--skip-verify` - Skip security verification (use with caution)
- `-h, --help` - Show help message
**Security Scanning:**
By default, all skills are scanned for potential security threats:
- **Data exfiltration**: Environment variable access, secret leakage, `.ssh` access
- **Prompt injection**: DAN mode, instruction override attempts, role hijacking
- **Destructive commands**: `rm -rf /`, `mkfs`, disk overwrite operations
- **Persistence mechanisms**: Cron jobs, shell RC modification, SSH backdoors
- **Network threats**: Reverse shells, tunneling services, exfiltration endpoints
- **Obfuscation**: Base64 piped to shell, `eval()` usage, encoded execution
**Trust Levels:**
- `builtin` - Official RAGFlow skills (always allowed)
- `trusted` - `openai/skills`, `anthropics/skills`, `microsoft/skills`, `google/skills` (caution allowed)
- `community` - All other sources (findings blocked unless `--force`)
**Examples:**
```bash
# Install from local path
install-skill my-space ./my-local-skill
# Install from GitHub
install-skill my-space github.com/openai/skills/skill-creator
# Install from ClawHub
install-skill my-space clawhub://user/web-search
# Install from Skills.sh
install-skill my-space skills.sh/xixu-me/skills/readme-i18n
# Force reinstall (delete existing and reinstall, update index)
install-skill my-space ./my-skill --force
# Force install with custom name, skip security check
install-skill my-space clawhub://unknown-skill --force --name my-skill --skip-verify
# Install specific version
install-skill my-space skill://kubernetes --version 2.1.0
```
#### uninstall-skill
Remove a skill from RAGFlow and delete its search index.
**Usage:**
```bash
uninstall-skill <space> <skill-name>
```
**Arguments:**
- `<space>` - Skills space ID (required)
- `<skill-name>` - Name of the skill to uninstall (required)
**Examples:**
```bash
uninstall-skill my-space my-skill
```
#### Deprecated Commands
- `add-skill` - Deprecated, use `install-skill` instead
- `delete-skill` - Deprecated, use `uninstall-skill` instead
## File Structure Requirements
### Skill Directory
A valid skill directory must contain:
- `SKILL.md` - Required. Skill metadata and instructions in YAML frontmatter format
Optional files:
- Additional documentation (`.md`, `.mdx`)
- Code files (`.py`, `.js`, `.ts`, etc.)
- Configuration files (`.json`, `.yaml`, `.toml`)
### SKILL.md Frontmatter
```yaml
---
name: my-skill
description: A brief description of what this skill does
version: 1.0.0
author: Your Name
tags:
- category1
- category2
---
```
## Security Architecture
The skill management system implements defense-in-depth security:
1. **Source Validation**: All remote sources use HTTPS and verify SSL certificates
2. **Quarantine**: Downloaded skills are isolated before installation
3. **Static Analysis**: Regex-based scanning for 100+ threat patterns across 6 categories:
- Exfiltration: Environment variable access, secret leakage
- Injection: Prompt injection, jailbreak attempts
- Destructive: Dangerous filesystem operations
- Persistence: Backdoors, startup file modification
- Network: Reverse shells, unauthorized tunneling
- Obfuscation: Encoded execution, download-and-run
4. **Trust Tiers**: Different security policies based on source reputation
5. **User Confirmation**: High-risk installations require explicit `--force`
6. **Audit Logging**: All installations are logged with scan results
## Validation Rules
- Total size must not exceed 50MB
- Individual files must not exceed 5MB
- Only text files are allowed (no binaries)
- Skill name must be lowercase alphanumeric with hyphens/underscores
- Hidden files and directories are ignored

2
internal/cli/contextengine/provider.go → internal/cli/filesystem/base.go
View File

@@ -14,7 +14,7 @@
// limitations under the License.
//
package contextengine
package filesystem
import (
stdctx "context"

6
internal/cli/contextengine/dataset_provider.go → internal/cli/filesystem/dataset.go
View File

@@ -14,9 +14,10 @@
// limitations under the License.
//
package contextengine
package filesystem
import (
"io"
stdctx "context"
"encoding/json"
"fmt"
@@ -36,6 +37,7 @@ type HTTPResponse struct {
// HTTPClientInterface defines the interface needed from HTTPClient
type HTTPClientInterface interface {
Request(method, path string, useAPIBase bool, authKind string, headers map[string]string, jsonBody map[string]interface{}) (*HTTPResponse, error)
UploadMultipart(path string, contentType string, body io.Reader) error
}
// DatasetProvider handles datasets and their documents
@@ -508,7 +510,7 @@ func (p *DatasetProvider) listDocuments(ctx stdctx.Context, datasetName string,
}
var apiResp struct {
Code int `json:"code"`
Code int `json:"code"`
Data struct {
Docs []map[string]interface{} `json:"docs"`
} `json:"data"`

24
internal/cli/contextengine/engine.go → internal/cli/filesystem/engine.go
View File

@@ -14,7 +14,7 @@
// limitations under the License.
//
package contextengine
package filesystem
import (
stdctx "context"
@@ -23,13 +23,13 @@ import (
"time"
)
// Engine is the core of the Context Engine
// Engine is the core of the Virtual Filesystem
// It manages providers and routes commands to the appropriate provider
type Engine struct {
providers []Provider
}
// NewEngine creates a new Context Engine
// NewEngine creates a new Virtual Filesystem Engine
func NewEngine() *Engine {
return &Engine{
providers: make([]Provider, 0),
@@ -136,6 +136,8 @@ func (e *Engine) List(ctx stdctx.Context, path string, opts *ListOptions) (*Resu
// 2. Top-level folders from files provider (file_manager)
func (e *Engine) listRoot(ctx stdctx.Context, opts *ListOptions) (*Result, error) {
nodes := make([]*Node, 0)
// Track names to avoid duplicates
seen := make(map[string]bool)
// Add built-in providers first (like datasets)
for _, p := range e.providers {
@@ -152,6 +154,7 @@ func (e *Engine) listRoot(ctx stdctx.Context, opts *ListOptions) (*Result, error
"description": p.Description(),
},
})
seen[p.Name()] = true
}
// Add top-level folders from files provider (file_manager)
@@ -161,6 +164,11 @@ func (e *Engine) listRoot(ctx stdctx.Context, opts *ListOptions) (*Result, error
for _, node := range filesResult.Nodes {
// Only add folders (directories), not files
if node.Type == NodeTypeDirectory {
// Skip if already added by a provider
if seen[node.Name] {
continue
}
seen[node.Name] = true
// Ensure path doesn't have /files/ prefix for display
node.Path = strings.TrimPrefix(node.Path, "files/")
node.Path = strings.TrimPrefix(node.Path, "/")
@@ -186,6 +194,16 @@ func (e *Engine) getFileProvider() Provider {
return nil
}
// GetProvider returns a provider by name
func (e *Engine) GetProvider(name string) Provider {
for _, p := range e.providers {
if p.Name() == name {
return p
}
}
return nil
}
// Search searches for nodes matching the query
func (e *Engine) Search(ctx stdctx.Context, path string, opts *SearchOptions) (*Result, error) {
provider, subPath, err := e.resolveProvider(path)

88
internal/cli/contextengine/file_provider.go → internal/cli/filesystem/file.go
View File

@@ -14,7 +14,7 @@
// limitations under the License.
//
package contextengine
package filesystem
import (
stdctx "context"
@@ -542,6 +542,92 @@ func (p *FileProvider) downloadFile(ctx stdctx.Context, fileID string) ([]byte,
return resp.Body, nil
}
// DeleteFile deletes a file or folder by its ID
func (p *FileProvider) DeleteFile(ctx stdctx.Context, fileID string) error {
// Use JSON body format expected by Python backend: {"ids": ["file_id"]}
payload := map[string]interface{}{
"ids": []string{fileID},
}
resp, err := p.httpClient.Request("DELETE", "/files", true, "api", nil, payload)
if err != nil {
return fmt.Errorf("delete request failed: %w", err)
}
// Handle empty response (e.g., 204 No Content)
if len(resp.Body) == 0 {
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return nil
}
return fmt.Errorf("delete failed with status code: %d", resp.StatusCode)
}
var apiResp struct {
Code int `json:"code"`
Data interface{} `json:"data"`
Message string `json:"message"`
}
if err := json.Unmarshal(resp.Body, &apiResp); err != nil {
return fmt.Errorf("failed to parse response: %w", err)
}
if apiResp.Code != 0 {
return fmt.Errorf("delete failed: %s", apiResp.Message)
}
return nil
}
// DeleteFolderByPath deletes a folder by its path (e.g., "skills/hub11/skill-name")
func (p *FileProvider) DeleteFolderByPath(ctx stdctx.Context, folderPath string) error {
parts := SplitPath(folderPath)
if len(parts) == 0 {
return fmt.Errorf("empty folder path")
}
// Find the folder ID by traversing the path
var folderID string
currentPath := ""
for i, part := range parts {
if i == 0 {
// First part - find in root
id, err := p.getFolderIDByName(ctx, part)
if err != nil {
return fmt.Errorf("folder not found: %s", part)
}
folderID = id
currentPath = part
} else {
// Subsequent parts - find in parent folder
result, err := p.listFilesByParentID(ctx, folderID, currentPath, nil)
if err != nil {
return fmt.Errorf("failed to list folder contents: %w", err)
}
found := false
for _, node := range result.Nodes {
if node.Name == part && node.Type == NodeTypeDirectory {
folderID = getString(node.Metadata["id"])
if folderID == "" {
return fmt.Errorf("folder ID not found for: %s", part)
}
currentPath = currentPath + "/" + part
found = true
break
}
}
if !found {
return fmt.Errorf("folder not found: %s in %s", part, currentPath)
}
}
}
// Delete the folder
return p.DeleteFile(ctx, folderID)
}
// ==================== Conversion Functions ====================
// fileToNode converts a file map to a Node

2154
internal/cli/filesystem/skill.go Normal file
View File

File diff suppressed because it is too large Load Diff

164
internal/cli/filesystem/skill_hub/security/guard.go Normal file
View File

@@ -0,0 +1,164 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package security
import (
"fmt"
"strings"
)
// Guard provides security policy enforcement
type Guard struct {
trustedRepos map[string]bool
policy map[string][3]string
}
// NewGuard creates a new security guard
func NewGuard() *Guard {
return &Guard{
trustedRepos: TrustedRepos,
policy: InstallPolicy,
}
}
// extractCanonicalRepo extracts the canonical owner/repo from an identifier
// Supports formats: "owner/repo", "github.com/owner/repo/path", "owner/repo/path"
func extractCanonicalRepo(identifier string) string {
// Normalize the identifier
identifier = strings.TrimSpace(identifier)
identifier = strings.ToLower(identifier)
// Remove protocol prefix if present
if idx := strings.Index(identifier, "://"); idx != -1 {
identifier = identifier[idx+3:]
}
// Remove github.com prefix if present
if strings.HasPrefix(identifier, "github.com/") {
identifier = strings.TrimPrefix(identifier, "github.com/")
}
// Split into parts
parts := strings.Split(identifier, "/")
if len(parts) < 2 {
return ""
}
// Extract owner and repo (first two components)
owner := strings.TrimSpace(parts[0])
repo := strings.TrimSpace(parts[1])
if owner == "" || repo == "" {
return ""
}
return owner + "/" + repo
}
// ResolveTrustLevel determines the trust level based on source and identifier
func (g *Guard) ResolveTrustLevel(source, identifier string) string {
// Official/builtin source
if source == "official" || source == "builtin" {
return "builtin"
}
// Extract canonical repo key and check against trusted repositories
canonicalRepo := extractCanonicalRepo(identifier)
if canonicalRepo != "" && g.trustedRepos[canonicalRepo] {
return "trusted"
}
// Default to community
return "community"
}
// ShouldAllowInstall determines if installation should be allowed based on scan results
// Returns (allowed bool, reason string)
func (g *Guard) ShouldAllowInstall(result *ScanResult, force bool) (bool, string) {
policy, ok := g.policy[result.TrustLevel]
if !ok {
policy = g.policy["community"]
}
vi, ok := VerdictIndex[result.Verdict]
if !ok {
vi = 2 // dangerous
}
decision := policy[vi]
switch decision {
case "allow":
return true, fmt.Sprintf("Allowed (%s source, %s verdict)", result.TrustLevel, result.Verdict)
case "ask":
return false, fmt.Sprintf("Requires confirmation (%s source + %s verdict, %d findings)",
result.TrustLevel, result.Verdict, len(result.Findings))
case "block":
if force {
return true, fmt.Sprintf("Force-installed despite %s verdict (%d findings)",
result.Verdict, len(result.Findings))
}
return false, fmt.Sprintf("Blocked (%s source + %s verdict, %d findings). Use --force to override.",
result.TrustLevel, result.Verdict, len(result.Findings))
}
return false, "Unknown policy decision"
}
// FormatScanReport formats a scan result for display
func (g *Guard) FormatScanReport(result *ScanResult) string {
var sb strings.Builder
sb.WriteString("╔════════════════════════════════════════════════════════════════╗\n")
sb.WriteString(fmt.Sprintf("║ Security Scan Report: %-40s ║\n", result.SkillName))
sb.WriteString("╚════════════════════════════════════════════════════════════════╝\n")
sb.WriteString(fmt.Sprintf("Source: %s\n", result.Source))
sb.WriteString(fmt.Sprintf("Trust Level: %s\n", result.TrustLevel))
sb.WriteString(fmt.Sprintf("Verdict: %s\n", result.Verdict))
sb.WriteString(fmt.Sprintf("Findings: %d\n", len(result.Findings)))
if len(result.Findings) > 0 {
sb.WriteString("\n─── Findings ───\n")
// Group by severity
severityOrder := []string{"critical", "high", "medium", "low"}
for _, sev := range severityOrder {
for _, f := range result.Findings {
if f.Severity == sev {
sb.WriteString(fmt.Sprintf("\n[%s] %s\n", strings.ToUpper(sev), f.PatternID))
sb.WriteString(fmt.Sprintf(" Category: %s\n", f.Category))
sb.WriteString(fmt.Sprintf(" File: %s:%d\n", f.File, f.Line))
sb.WriteString(fmt.Sprintf(" Match: %s\n", f.Match))
sb.WriteString(fmt.Sprintf(" Description: %s\n", f.Description))
}
}
}
}
sb.WriteString("\n")
return sb.String()
}
// AddTrustedRepo adds a repository to the trusted list
func (g *Guard) AddTrustedRepo(repo string) {
g.trustedRepos[repo] = true
}
// IsTrustedRepo checks if a repository is trusted
func (g *Guard) IsTrustedRepo(repo string) bool {
return g.trustedRepos[repo]
}

284
internal/cli/filesystem/skill_hub/security/patterns.go Normal file
View File

@@ -0,0 +1,284 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package security
// ThreatPattern represents a security threat detection pattern
// Inspired by hermes-agent's skills_guard.py
type ThreatPattern struct {
Pattern string // Regular expression pattern
PatternID string // Unique identifier for this pattern
Severity string // critical | high | medium | low
Category string // exfiltration | injection | destructive | persistence | network | obfuscation
Description string // Human-readable description
}
// ThreatPatterns contains all security threat detection rules
var ThreatPatterns = []ThreatPattern{
// ========== Data Exfiltration ==========
{
Pattern: `curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)`,
PatternID: "env_exfil_curl",
Severity: "critical",
Category: "exfiltration",
Description: "curl command interpolating secret environment variable",
},
{
Pattern: `wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)`,
PatternID: "env_exfil_wget",
Severity: "critical",
Category: "exfiltration",
Description: "wget command interpolating secret environment variable",
},
{
Pattern: `\$HOME/\.ssh|\~/\.ssh`,
PatternID: "ssh_dir_access",
Severity: "high",
Category: "exfiltration",
Description: "references user SSH directory",
},
{
Pattern: `os\.environ\b`,
PatternID: "python_os_environ",
Severity: "high",
Category: "exfiltration",
Description: "accesses os.environ (potential env dump)",
},
{
Pattern: `printenv|env\s*\|`,
PatternID: "dump_all_env",
Severity: "high",
Category: "exfiltration",
Description: "dumps all environment variables",
},
// ========== Prompt Injection ==========
{
Pattern: `(?i)ignore\s+(?:\w+\s+)*(previous|all|above|prior)\s+instructions`,
PatternID: "prompt_injection_ignore",
Severity: "critical",
Category: "injection",
Description: "prompt injection: ignore previous instructions",
},
{
Pattern: `(?i)\bDAN\s+mode\b|Do\s+Anything\s+Now`,
PatternID: "jailbreak_dan",
Severity: "critical",
Category: "injection",
Description: "DAN (Do Anything Now) jailbreak attempt",
},
{
Pattern: `(?i)you\s+are\s+(?:\w+\s+)*now\s+`,
PatternID: "role_hijack",
Severity: "high",
Category: "injection",
Description: "attempts to override the agent's role",
},
{
Pattern: `(?i)system\s+prompt\s+override`,
PatternID: "sys_prompt_override",
Severity: "critical",
Category: "injection",
Description: "attempts to override the system prompt",
},
{
Pattern: `(?i)disregard\s+(?:\w+\s+)*(your|all|any)\s+(?:\w+\s+)*(instructions|rules|guidelines)`,
PatternID: "disregard_rules",
Severity: "critical",
Category: "injection",
Description: "instructs agent to disregard its rules",
},
// ========== Destructive Operations ==========
{
Pattern: `rm\s+-rf\s+/`,
PatternID: "destructive_root_rm",
Severity: "critical",
Category: "destructive",
Description: "recursive delete from root",
},
{
Pattern: `rm\s+(-[^\s]*)?r.*\$HOME|\brmdir\s+.*\$HOME`,
PatternID: "destructive_home_rm",
Severity: "critical",
Category: "destructive",
Description: "recursive delete targeting home directory",
},
{
Pattern: `\bmkfs\b`,
PatternID: "format_filesystem",
Severity: "critical",
Category: "destructive",
Description: "formats a filesystem",
},
{
Pattern: `\bdd\s+.*if=.*of=/dev/`,
PatternID: "disk_overwrite",
Severity: "critical",
Category: "destructive",
Description: "raw disk write operation",
},
{
Pattern: `shutil\.rmtree\s*\(\s*["\'/]`,
PatternID: "python_rmtree",
Severity: "high",
Category: "destructive",
Description: "Python rmtree on absolute or root-relative path",
},
{
Pattern: `rm\s+(-[a-zA-Z]*r[a-zA-Z]*\s+|--)recursive\s+).*\$`,
PatternID: "rm_recursive_dangerous",
Severity: "high",
Category: "destructive",
Description: "recursive rm with suspicious target",
},
// ========== Persistence ==========
{
Pattern: `\bcrontab\b`,
PatternID: "persistence_cron",
Severity: "medium",
Category: "persistence",
Description: "modifies cron jobs",
},
{
Pattern: `\.(bashrc|zshrc|profile|bash_profile|bash_login|zprofile|zlogin)\b`,
PatternID: "shell_rc_mod",
Severity: "medium",
Category: "persistence",
Description: "references shell startup file",
},
{
Pattern: `authorized_keys`,
PatternID: "ssh_backdoor",
Severity: "critical",
Category: "persistence",
Description: "modifies SSH authorized keys",
},
{
Pattern: `AGENTS\.md|CLAUDE\.md|\.cursorrules|\.clinerules`,
PatternID: "agent_config_mod",
Severity: "critical",
Category: "persistence",
Description: "references agent config files (could persist malicious instructions)",
},
{
Pattern: `\.ssh/config`,
PatternID: "ssh_config_mod",
Severity: "high",
Category: "persistence",
Description: "modifies SSH configuration",
},
// ========== Network Threats ==========
{
Pattern: `\bnc\s+-[lp]|ncat\s+-[lp]|\bsocat\b`,
PatternID: "reverse_shell",
Severity: "critical",
Category: "network",
Description: "potential reverse shell listener",
},
{
Pattern: `/bin/(ba)?sh\s+-i\s+.*>/dev/tcp/`,
PatternID: "bash_reverse_shell",
Severity: "critical",
Category: "network",
Description: "bash interactive reverse shell via /dev/tcp",
},
{
Pattern: `\bngrok\b|\blocaltunnel\b|\bserveo\b|\bcloudflared\b`,
PatternID: "tunnel_service",
Severity: "high",
Category: "network",
Description: "uses tunneling service for external access",
},
{
Pattern: `webhook\.site|requestbin\.com|pipedream\.net|hookbin\.com`,
PatternID: "exfil_service",
Severity: "high",
Category: "network",
Description: "references known data exfiltration/webhook testing service",
},
{
Pattern: `python\s+-c\s+.*socket.*subprocess`,
PatternID: "python_reverse_shell",
Severity: "critical",
Category: "network",
Description: "Python reverse shell pattern",
},
// ========== Obfuscation ==========
{
Pattern: `base64\s+(-d|--decode)\s*\|`,
PatternID: "base64_decode_pipe",
Severity: "high",
Category: "obfuscation",
Description: "base64 decodes and pipes to execution",
},
{
Pattern: `\beval\s*\(\s*["\']`,
PatternID: "eval_string",
Severity: "high",
Category: "obfuscation",
Description: "eval() with string argument",
},
{
Pattern: `echo\s+[^\n]*\|\s*(bash|sh|python|perl|ruby|node)`,
PatternID: "echo_pipe_exec",
Severity: "critical",
Category: "obfuscation",
Description: "echo piped to interpreter for execution",
},
{
Pattern: `curl\s+[^\n]*\|\s*(ba)?sh`,
PatternID: "curl_pipe_shell",
Severity: "critical",
Category: "supply_chain",
Description: "curl piped to shell (download-and-execute)",
},
{
Pattern: `\bexec\s*\(\s*(base64|decode|unescape)`,
PatternID: "exec_encoded",
Severity: "high",
Category: "obfuscation",
Description: "executes encoded content",
},
}
// TrustedRepos contains the list of trusted repositories
// These repos have a higher trust level
var TrustedRepos = map[string]bool{
"openai/skills": true,
"anthropics/skills": true,
"microsoft/skills": true,
"google/skills": true,
}
// InstallPolicy defines the installation policy for each trust level
// Format: [safe, caution, dangerous] -> action
// Actions: allow, block, ask
var InstallPolicy = map[string][3]string{
"builtin": {"allow", "allow", "allow"}, // Official skills: always allow
"trusted": {"allow", "allow", "block"}, // Trusted repos: caution allowed, dangerous blocked
"community": {"allow", "block", "block"}, // Community: only safe allowed
}
// VerdictIndex maps verdict to array index
var VerdictIndex = map[string]int{
"safe": 0,
"caution": 1,
"dangerous": 2,
}

150
internal/cli/filesystem/skill_hub/security/scanner.go Normal file
View File

@@ -0,0 +1,150 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package security
import (
"regexp"
"strings"
)
// Finding represents a security issue found during scanning
type Finding struct {
PatternID string // Rule ID
Severity string // critical | high | medium | low
Category string // exfiltration | injection | destructive | persistence | network | obfuscation
File string // File path where found
Line int // Line number
Match string // The matched text
Description string // Human-readable description
}
// ScanResult represents the result of a security scan
type ScanResult struct {
SkillName string
Source string
TrustLevel string // builtin | trusted | community
Verdict string // safe | caution | dangerous
Findings []Finding
}
// Scanner performs security scans on skill content
type Scanner struct {
patterns []ThreatPattern
}
// NewScanner creates a new security scanner
func NewScanner() *Scanner {
return &Scanner{
patterns: ThreatPatterns,
}
}
// ScanSkill scans skill files for security threats
func (s *Scanner) ScanSkill(skillName, source, trustLevel string, files map[string][]byte) *ScanResult {
var allFindings []Finding
for filename, content := range files {
findings := s.scanFile(filename, string(content))
allFindings = append(allFindings, findings...)
}
verdict := s.determineVerdict(allFindings)
return &ScanResult{
SkillName: skillName,
Source: source,
TrustLevel: trustLevel,
Verdict: verdict,
Findings: allFindings,
}
}
// scanFile scans a single file for threats
func (s *Scanner) scanFile(filename, content string) []Finding {
var findings []Finding
lines := strings.Split(content, "\n")
for _, pattern := range s.patterns {
re, err := regexp.Compile("(?i:" + pattern.Pattern + ")")
if err != nil {
continue
}
for i, line := range lines {
if matches := re.FindString(line); matches != "" {
findings = append(findings, Finding{
PatternID: pattern.PatternID,
Severity: pattern.Severity,
Category: pattern.Category,
File: filename,
Line: i + 1,
Match: strings.TrimSpace(matches),
Description: pattern.Description,
})
}
}
}
return findings
}
// determineVerdict determines the overall verdict based on findings
func (s *Scanner) determineVerdict(findings []Finding) string {
if len(findings) == 0 {
return "safe"
}
hasCritical := false
hasHigh := false
for _, f := range findings {
if f.Severity == "critical" {
hasCritical = true
} else if f.Severity == "high" {
hasHigh = true
}
}
if hasCritical {
return "dangerous"
}
if hasHigh {
return "caution"
}
return "caution"
}
// HasCriticalChecks if any finding is critical severity
func (r *ScanResult) HasCritical() bool {
for _, f := range r.Findings {
if f.Severity == "critical" {
return true
}
}
return false
}
// CountBySeverity counts findings by severity level
func (r *ScanResult) CountBySeverity(severity string) int {
count := 0
for _, f := range r.Findings {
if f.Severity == severity {
count++
}
}
return count
}

933
internal/cli/filesystem/skill_hub/source/clawhub.go Normal file
View File

@@ -0,0 +1,933 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
// progressLogger is a simple logger for user-facing progress messages
type progressLogger struct {
enabled bool
}
func (l *progressLogger) log(format string, args ...interface{}) {
if l.enabled {
fmt.Printf(" → "+format+"\n", args...)
}
}
func (l *progressLogger) error(format string, args ...interface{}) {
fmt.Printf(" ✗ "+format+"\n", args...)
}
func (l *progressLogger) success(format string, args ...interface{}) {
fmt.Printf(" ✓ "+format+"\n", args...)
}
const (
clawHubBaseURL = "https://clawhub.ai/api/v1"
)
// ClawHubSource handles ClawHub registry skills
// Reference implementation: hermes-agent/tools/skills_hub.py ClawHubSource
// All skills are treated as community trust — ClawHavoc incident showed
// their vetting is insufficient (341 malicious skills found Feb 2026).
type ClawHubSource struct {
client HTTPClientInterface
logger progressLogger
}
// NewClawHubSource creates a new ClawHub source adapter
func NewClawHubSource(client HTTPClientInterface) *ClawHubSource {
return &ClawHubSource{client: client, logger: progressLogger{enabled: true}}
}
// SourceID returns the source identifier
func (s *ClawHubSource) SourceID() string {
return "clawhub"
}
// TrustLevel returns the trust level for ClawHub
func (s *ClawHubSource) TrustLevel(identifier string) string {
// ClawHub has community verification
return "community"
}
// Search searches for skills on ClawHub matching the query
func (s *ClawHubSource) Search(query string, limit int) ([]*SkillMetadata, error) {
if limit <= 0 {
limit = 10
}
// Try direct slug match first for exact queries
if query != "" && len(query) >= 2 {
meta, err := s.exactSlugMeta(query)
if err == nil && meta != nil {
return []*SkillMetadata{meta}, nil
}
}
// Use the lightweight listing API
url := fmt.Sprintf("%s/skills", clawHubBaseURL)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
q := req.URL.Query()
if query != "" {
q.Add("search", query)
}
q.Add("limit", strconv.Itoa(limit))
req.URL.RawQuery = q.Encode()
resp, err := s.client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to search ClawHub: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("ClawHub API returned %d", resp.StatusCode)
}
var data struct {
Items []struct {
Slug string `json:"slug"`
DisplayName string `json:"displayName"`
Name string `json:"name"`
Summary string `json:"summary"`
Description string `json:"description"`
Tags interface{} `json:"tags"`
} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return nil, err
}
results := make([]*SkillMetadata, 0, len(data.Items))
for _, item := range data.Items {
slug := item.Slug
if slug == "" {
continue
}
displayName := item.DisplayName
if displayName == "" {
displayName = item.Name
}
if displayName == "" {
displayName = slug
}
summary := item.Summary
if summary == "" {
summary = item.Description
}
results = append(results, &SkillMetadata{
Name: displayName,
Description: summary,
Version: "",
Author: "",
Tags: normalizeTags(item.Tags),
})
}
// Apply search scoring and filtering
results = s.finalizeSearchResults(query, results, limit)
return results, nil
}
// Fetch retrieves a skill from ClawHub
// Downloads the skill as a ZIP bundle and extracts text files
// Supports identifier with version: "slug@version" or just "slug" (uses latest)
func (s *ClawHubSource) Fetch(identifier string) (*SkillBundle, error) {
slug, specifiedVersion := extractSlugAndVersion(identifier)
s.logger.log("Looking up skill '%s' on ClawHub...", slug)
// Fetch skill metadata
skillData, err := s.getSkillData(slug)
if err != nil {
s.logger.error("Cannot find skill '%s' on ClawHub: %v", slug, err)
return nil, fmt.Errorf("skill '%s' not found on ClawHub: %w", slug, err)
}
s.logger.success("Found skill: %s", skillData.DisplayName)
// Determine version to download
var version string
if specifiedVersion != "" {
version = specifiedVersion
s.logger.log("Using specified version: %s", version)
} else {
// Resolve the latest version
s.logger.log("Resolving latest version...")
version, err = s.resolveLatestVersion(slug, skillData)
if err != nil {
s.logger.error("Cannot determine version for '%s': %v", slug, err)
return nil, fmt.Errorf("could not resolve latest version for %s: %w", slug, err)
}
if version == "" {
s.logger.error("No versions available for skill '%s'", slug)
return nil, fmt.Errorf("no version found for skill %s", slug)
}
s.logger.success("Latest version: %s", version)
}
// Try to get files from version metadata endpoint first (avoids rate-limited /download)
var files map[string][]byte
s.logger.log("Fetching skill files (version %s)...", version)
versionData, err := s.getVersionData(slug, version)
if err == nil {
files = s.extractFiles(versionData)
if len(files) > 0 {
s.logger.success("Fetched %d files from metadata", len(files))
}
}
// Fallback to ZIP download if metadata method didn't return files
if len(files) == 0 {
s.logger.log("Trying ZIP download...")
// Add delay before download to avoid rate limit
time.Sleep(3 * time.Second)
zipFiles, err2 := s.downloadZip(slug, version)
if err2 != nil {
s.logger.error("Failed to download skill bundle: %v", err2)
return nil, fmt.Errorf("failed to download skill '%s': %w", slug, err2)
}
files = zipFiles
s.logger.success("Downloaded %d files via ZIP", len(files))
}
// Validate: must have SKILL.md
if _, ok := files["SKILL.md"]; !ok {
s.logger.error("Downloaded bundle is missing SKILL.md (required file)")
return nil, fmt.Errorf("SKILL.md not found in skill %s (version %s)", slug, version)
}
return &SkillBundle{
Name: slug,
Files: files,
Source: "clawhub",
Identifier: slug,
TrustLevel: s.TrustLevel(identifier),
Metadata: &SkillMetadata{
Name: skillData.DisplayName,
Description: skillData.Summary,
Version: version,
},
}, nil
}
// Inspect retrieves metadata from ClawHub without downloading full content
func (s *ClawHubSource) Inspect(identifier string) (*SkillMetadata, error) {
slug := extractSlug(identifier)
skillData, err := s.getSkillData(slug)
if err != nil {
return nil, err
}
return &SkillMetadata{
Name: skillData.DisplayName,
Description: skillData.Summary,
Version: "",
Author: "",
Tags: normalizeTags(skillData.Tags),
}, nil
}
// getSkillData fetches skill metadata from ClawHub API with retry logic
func (s *ClawHubSource) getSkillData(slug string) (*clawHubSkillData, error) {
url := fmt.Sprintf("%s/skills/%s", clawHubBaseURL, slug)
body, err := s.doRequestWithRetry("GET", url, nil)
if err != nil {
return nil, err
}
// ClawHub API may return nested structure: {"skill": {...}, "latestVersion": ...}
var rawData map[string]interface{}
if err := json.Unmarshal(body, &rawData); err != nil {
return nil, err
}
return coerceSkillPayload(rawData), nil
}
// getVersionData fetches version-specific metadata with retry logic
func (s *ClawHubSource) getVersionData(slug, version string) (map[string]interface{}, error) {
url := fmt.Sprintf("%s/skills/%s/versions/%s", clawHubBaseURL, slug, version)
body, err := s.doRequestWithRetry("GET", url, nil)
if err != nil {
return nil, err
}
var data map[string]interface{}
if err := json.Unmarshal(body, &data); err != nil {
return nil, err
}
return data, nil
}
// resolveLatestVersion extracts the latest version from skill data with retry logic
func (s *ClawHubSource) resolveLatestVersion(slug string, skillData *clawHubSkillData) (string, error) {
// Try latestVersion field first
if skillData.LatestVersion != "" {
return skillData.LatestVersion, nil
}
// Try tags.latest
if skillData.TagsLatest != "" {
return skillData.TagsLatest, nil
}
// Fallback: fetch versions list and take first
url := fmt.Sprintf("%s/skills/%s/versions", clawHubBaseURL, slug)
body, err := s.doRequestWithRetry("GET", url, nil)
if err != nil {
return "", err
}
var versions []struct {
Version string `json:"version"`
}
if err := json.Unmarshal(body, &versions); err != nil {
return "", err
}
if len(versions) > 0 && versions[0].Version != "" {
return versions[0].Version, nil
}
return "", nil
}
// downloadZip downloads skill as ZIP bundle and extracts text files
func (s *ClawHubSource) downloadZip(slug, version string) (map[string][]byte, error) {
// Use the correct endpoint with slug parameter (matching hermes-agent)
url := fmt.Sprintf("%s/download?slug=%s&version=%s", clawHubBaseURL, slug, version)
s.logger.log("Downloading ZIP from: %s", url)
body, err := s.doRequestWithRetry("GET", url, nil)
if err != nil {
return nil, fmt.Errorf("download failed: %w", err)
}
s.logger.log("Downloaded %d bytes, extracting files...", len(body))
// Extract ZIP
zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
s.logger.error("Downloaded file is not a valid ZIP archive: %v", err)
return nil, fmt.Errorf("invalid ZIP file: %w", err)
}
files := make(map[string][]byte)
skippedCount := 0
for _, file := range zipReader.File {
if file.FileInfo().IsDir() {
continue
}
// Validate path for safety
name := file.Name
if !isSafePath(name) {
skippedCount++
continue
}
// Skip large files (>500KB)
if file.UncompressedSize64 > 500_000 {
skippedCount++
s.logger.log("Skipping large file: %s (%.1f MB)", name, float64(file.UncompressedSize64)/1024/1024)
continue
}
// Read file content
rc, err := file.Open()
if err != nil {
skippedCount++
continue
}
content, err := io.ReadAll(rc)
rc.Close()
if err != nil {
skippedCount++
continue
}
// Only include text files (check for null bytes indicating binary)
if isTextContent(content) {
files[name] = content
} else {
skippedCount++
s.logger.log("Skipping binary file: %s", name)
}
}
if skippedCount > 0 {
s.logger.log("Skipped %d files (unsafe paths, large files, or binary content)", skippedCount)
}
if len(files) == 0 {
s.logger.error("No valid files found in the ZIP archive")
return nil, fmt.Errorf("no valid files extracted from ZIP")
}
return files, nil
}
// extractFiles extracts files from version data structure
func (s *ClawHubSource) extractFiles(versionData map[string]interface{}) map[string][]byte {
files := make(map[string][]byte)
// Check for nested version -> files structure
if nested, ok := versionData["version"].(map[string]interface{}); ok {
versionData = nested
}
fileList, ok := versionData["files"]
if !ok {
return files
}
// Handle map structure: {"filename": "content"}
if fileMap, ok := fileList.(map[string]interface{}); ok {
for name, content := range fileMap {
if s, ok := content.(string); ok && isSafePath(name) {
files[name] = []byte(s)
}
}
return files
}
// Handle array structure with file metadata
if fileArray, ok := fileList.([]interface{}); ok {
for _, item := range fileArray {
fileMeta, ok := item.(map[string]interface{})
if !ok {
continue
}
name := ""
if n, ok := fileMeta["path"].(string); ok && n != "" {
name = n
} else if n, ok := fileMeta["name"].(string); ok && n != "" {
name = n
}
if name == "" || !isSafePath(name) {
continue
}
// Try inline content first
if content, ok := fileMeta["content"].(string); ok {
files[name] = []byte(content)
continue
}
// Try rawUrl/downloadUrl
var url string
if u, ok := fileMeta["rawUrl"].(string); ok && u != "" {
url = u
} else if u, ok := fileMeta["downloadUrl"].(string); ok && u != "" {
url = u
} else if u, ok := fileMeta["url"].(string); ok && u != "" {
url = u
}
if url != "" && strings.HasPrefix(url, "http") {
content, err := s.fetchText(url)
if err == nil {
files[name] = []byte(content)
}
}
}
}
return files
}
// fetchText fetches text content from URL
func (s *ClawHubSource) fetchText(url string) (string, error) {
resp, err := s.client.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(body), nil
}
// doRequestWithRetry performs HTTP request with retry logic for 429 rate limiting
func (s *ClawHubSource) doRequestWithRetry(method, url string, body []byte) ([]byte, error) {
maxRetries := 5
var lastErr error
isDownload := strings.Contains(url, "/download")
for attempt := 0; attempt < maxRetries; attempt++ {
// Initial delay for download requests to avoid triggering rate limit
if attempt == 0 && isDownload {
s.logger.log("Adding initial delay for download request...")
time.Sleep(5 * time.Second)
}
var bodyReader io.Reader
if body != nil {
bodyReader = bytes.NewReader(body)
}
req, err := http.NewRequest(method, url, bodyReader)
if err != nil {
lastErr = fmt.Errorf("failed to create request: %w", err)
s.logger.error("Request setup failed: %v", lastErr)
continue
}
// Simple headers like hermes-agent
req.Header.Set("User-Agent", "RAGFlow-CLI/1.0")
req.Header.Set("Accept", "application/json")
resp, err := s.client.Do(req)
if err != nil {
lastErr = err
if attempt < maxRetries-1 {
s.logger.error("Request failed (attempt %d/%d): %v", attempt+1, maxRetries, err)
}
continue
}
// Read response body immediately
respBody, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
lastErr = fmt.Errorf("failed to read response: %w", err)
if attempt < maxRetries-1 {
s.logger.error("Response read failed (attempt %d/%d): %v", attempt+1, maxRetries, err)
}
continue
}
// Handle rate limiting - ClawHub has strict limits, wait 30-60s to reset window
if resp.StatusCode == http.StatusTooManyRequests {
retryAfter := resp.Header.Get("Retry-After")
waitSeconds := 30 // Default: wait 30 seconds
if retryAfter != "" {
if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds > 0 {
waitSeconds = seconds
}
}
// Ensure minimum 30s wait to reset rate limit window
if waitSeconds < 30 {
waitSeconds = 30
}
// Cap at 60 seconds
if waitSeconds > 60 {
waitSeconds = 60
}
s.logger.log("Rate limited by ClawHub, waiting %d seconds...", waitSeconds)
time.Sleep(time.Duration(waitSeconds) * time.Second)
lastErr = fmt.Errorf("rate limited (429)")
continue
}
if resp.StatusCode == http.StatusNotFound {
lastErr = fmt.Errorf("skill not found (HTTP 404)")
s.logger.error("%v", lastErr)
return nil, lastErr // Don't retry 404
}
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
lastErr = fmt.Errorf("access denied (HTTP %d) - check your credentials", resp.StatusCode)
s.logger.error("%v", lastErr)
return nil, lastErr // Don't retry auth errors
}
if resp.StatusCode != http.StatusOK {
lastErr = fmt.Errorf("ClawHub API returned HTTP %d", resp.StatusCode)
if attempt < maxRetries-1 {
s.logger.error("Server error (attempt %d/%d): HTTP %d", attempt+1, maxRetries, resp.StatusCode)
}
continue
}
return respBody, nil
}
// Provide helpful error message based on the last error
var userMsg string
if lastErr != nil {
errStr := lastErr.Error()
switch {
case strings.Contains(errStr, "connection refused"):
userMsg = "Cannot connect to ClawHub - the service may be down or your network is blocking the connection"
case strings.Contains(errStr, "timeout") || strings.Contains(errStr, "deadline exceeded"):
userMsg = "Connection to ClawHub timed out - your network may be slow or the service is unresponsive"
case strings.Contains(errStr, "no such host") || strings.Contains(errStr, "DNS"):
userMsg = "Cannot resolve ClawHub hostname - check your internet connection or DNS settings"
case strings.Contains(errStr, "certificate"):
userMsg = "SSL certificate error - your system may have outdated certificates or someone is intercepting the connection"
default:
userMsg = fmt.Sprintf("Network error after %d attempts: %v", maxRetries, lastErr)
}
} else {
userMsg = fmt.Sprintf("Failed after %d attempts - unknown error", maxRetries)
}
return nil, fmt.Errorf("%s", userMsg)
}
// exactSlugMeta tries to find skill by exact slug match
func (s *ClawHubSource) exactSlugMeta(query string) (*SkillMetadata, error) {
slug := extractSlug(query)
queryTermList := extractQueryTerms(query)
candidates := []string{}
// If slug looks valid, add it
if slug != "" && regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`).MatchString(slug) {
candidates = append(candidates, slug)
}
// Generate variations from query terms
if len(queryTermList) > 0 {
baseSlug := strings.Join(queryTermList, "-")
if len(queryTermList) >= 2 {
candidates = append(candidates,
baseSlug+"-agent",
baseSlug+"-skill",
baseSlug+"-tool",
baseSlug+"-assistant",
baseSlug+"-playbook",
baseSlug,
)
} else {
candidates = append(candidates, baseSlug)
}
}
seen := make(map[string]bool)
for _, candidate := range candidates {
if seen[candidate] {
continue
}
seen[candidate] = true
meta, err := s.Inspect(candidate)
if err == nil && meta != nil && meta.Name != "" {
return meta, nil
}
}
return nil, fmt.Errorf("no exact match found")
}
// finalizeSearchResults applies scoring and filtering to search results
func (s *ClawHubSource) finalizeSearchResults(query string, results []*SkillMetadata, limit int) []*SkillMetadata {
if query == "" {
deduped := dedupeResults(results)
if len(deduped) > limit {
return deduped[:limit]
}
return deduped
}
// Score and filter
filtered := make([]*SkillMetadata, 0)
for _, meta := range results {
if s.searchScore(query, meta) > 0 {
filtered = append(filtered, meta)
}
}
// Sort by score
sort.Slice(filtered, func(i, j int) bool {
scoreI := s.searchScore(query, filtered[i])
scoreJ := s.searchScore(query, filtered[j])
if scoreI != scoreJ {
return scoreI > scoreJ
}
if filtered[i].Name != filtered[j].Name {
return strings.ToLower(filtered[i].Name) < strings.ToLower(filtered[j].Name)
}
return strings.ToLower(filtered[i].Description) < strings.ToLower(filtered[j].Description)
})
deduped := dedupeResults(filtered)
if len(deduped) > limit {
return deduped[:limit]
}
return deduped
}
// searchScore calculates relevance score for a skill against query
func (s *ClawHubSource) searchScore(query string, meta *SkillMetadata) int {
queryNorm := strings.ToLower(strings.TrimSpace(query))
if queryNorm == "" {
return 1
}
nameLower := strings.ToLower(meta.Name)
descLower := strings.ToLower(meta.Description)
queryTermList := extractQueryTerms(queryNorm)
nameTermList := extractQueryTerms(nameLower)
score := 0
// Exact matches (high scores)
if queryNorm == nameLower {
score += 130
}
if strings.ReplaceAll(nameLower, " ", "-") == queryNorm {
score += 120
}
if strings.HasPrefix(nameLower, queryNorm) {
score += 90
}
// Query terms match name terms
if len(queryTermList) > 0 && len(nameTermList) >= len(queryTermList) {
match := true
for i, term := range queryTermList {
if i >= len(nameTermList) || nameTermList[i] != term {
match = false
break
}
}
if match {
score += 65
}
}
// Substring matches
if strings.Contains(nameLower, queryNorm) {
score += 35
}
if strings.Contains(descLower, queryNorm) {
score += 10
}
// Individual term matches
for _, term := range queryTermList {
if strings.Contains(nameLower, term) {
score += 12
}
if strings.Contains(descLower, term) {
score += 3
}
}
return score
}
// Helper types and functions
// clawHubSkillData represents ClawHub skill API response
type clawHubSkillData struct {
Slug string `json:"slug"`
DisplayName string `json:"displayName"`
Name string `json:"name"`
Summary string `json:"summary"`
Description string `json:"description"`
Tags interface{} `json:"tags"`
LatestVersion string `json:"latestVersion"`
TagsLatest string `json:"tags_latest"` // Extracted from tags dict
}
// coerceSkillPayload handles nested ClawHub API response structures
// ClawHub API may return: {"skill": {...}, "latestVersion": ...} or flat structure
func coerceSkillPayload(data map[string]interface{}) *clawHubSkillData {
result := &clawHubSkillData{}
// Check for nested skill structure
nested, hasNested := data["skill"].(map[string]interface{})
if hasNested {
// Merge nested skill data
for k, v := range nested {
data[k] = v
}
// Keep latestVersion from outer if present
if lv, ok := data["latestVersion"].(string); ok && lv != "" {
result.LatestVersion = lv
}
}
// Extract fields
if v, ok := data["slug"].(string); ok {
result.Slug = v
}
if v, ok := data["displayName"].(string); ok {
result.DisplayName = v
}
if v, ok := data["name"].(string); ok && result.DisplayName == "" {
result.DisplayName = v
}
if v, ok := data["summary"].(string); ok {
result.Summary = v
}
if v, ok := data["description"].(string); ok && result.Summary == "" {
result.Summary = v
}
if v, ok := data["tags"]; ok {
result.Tags = v
// Extract latest from tags dict
if tagMap, ok := v.(map[string]interface{}); ok {
if latest, ok := tagMap["latest"].(string); ok {
result.TagsLatest = latest
}
}
}
return result
}
// extractSlug extracts the skill slug from identifier
func extractSlug(identifier string) string {
parts := strings.Split(identifier, "/")
return parts[len(parts)-1]
}
// extractSlugAndVersion extracts the skill slug and optional version from identifier
// Supports formats: "slug", "slug@version", "owner/slug", "owner/slug@version"
func extractSlugAndVersion(identifier string) (slug, version string) {
// First get the last part (handles owner/slug format)
parts := strings.Split(identifier, "/")
lastPart := parts[len(parts)-1]
// Check for version separator @
if idx := strings.LastIndex(lastPart, "@"); idx > 0 {
return lastPart[:idx], lastPart[idx+1:]
}
return lastPart, ""
}
// normalizeTags normalizes tags from various formats
func normalizeTags(tags interface{}) []string {
result := []string{}
switch v := tags.(type) {
case []interface{}:
for _, t := range v {
if s, ok := t.(string); ok && s != "" && s != "latest" {
result = append(result, s)
}
}
case []string:
for _, s := range v {
if s != "" && s != "latest" {
result = append(result, s)
}
}
case map[string]interface{}:
for k := range v {
if k != "" && k != "latest" {
result = append(result, k)
}
}
}
return result
}
// dedupeResults removes duplicate skills by name, keeping first occurrence
func dedupeResults(results []*SkillMetadata) []*SkillMetadata {
seen := make(map[string]bool)
unique := []*SkillMetadata{}
for _, r := range results {
key := strings.ToLower(r.Name)
if !seen[key] {
seen[key] = true
unique = append(unique, r)
}
}
return unique
}
// extractQueryTerms splits query into normalized terms
func extractQueryTerms(query string) []string {
re := regexp.MustCompile(`[^a-z0-9]+`)
parts := re.Split(strings.ToLower(query), -1)
result := []string{}
for _, p := range parts {
if p != "" {
result = append(result, p)
}
}
return result
}
// isSafePath validates that a path is safe (no directory traversal)
func isSafePath(path string) bool {
// Clean the path
clean := filepath.Clean(path)
// Check for absolute paths
if filepath.IsAbs(clean) {
return false
}
// Check for parent directory references
parts := strings.Split(clean, string(filepath.Separator))
for _, part := range parts {
if part == ".." {
return false
}
}
return true
}
// isTextContent checks if content appears to be text (not binary)
func isTextContent(data []byte) bool {
// Check for null bytes (indicates binary)
for _, b := range data {
if b == 0 {
return false
}
}
return true
}
func min(a, b int) int {
if a < b {
return a
}
return b
}

260
internal/cli/filesystem/skill_hub/source/github.go Normal file
View File

@@ -0,0 +1,260 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import (
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"path"
"strings"
)
// GitHubSource handles GitHub repository skills
type GitHubSource struct {
client HTTPClientInterface
}
// NewGitHubSource creates a new GitHub source adapter
func NewGitHubSource(client HTTPClientInterface) *GitHubSource {
return &GitHubSource{client: client}
}
// SourceID returns the source identifier
func (s *GitHubSource) SourceID() string {
return "github"
}
// TrustLevel returns the trust level based on repository
func (s *GitHubSource) TrustLevel(identifier string) string {
owner, repo, _, err := parseGitHubURL(identifier)
if err != nil {
return "community"
}
if isTrustedGitHubRepo(owner, repo) {
return "trusted"
}
return "community"
}
// Fetch retrieves a skill from GitHub
func (s *GitHubSource) Fetch(identifier string) (*SkillBundle, error) {
owner, repo, pathStr, err := parseGitHubURL(identifier)
if err != nil {
return nil, err
}
// Default to repo root if no path specified
if pathStr == "" {
pathStr = "."
}
// Try to get SKILL.md first to determine skill name
skillName := repo
meta := &SkillMetadata{Version: "1.0.0"}
skillMdContent, err := s.fetchFileContent(owner, repo, path.Join(pathStr, "SKILL.md"))
if err == nil {
parsedMeta, parseErr := parseSkillFrontmatter(skillMdContent)
if parseErr == nil {
meta = parsedMeta
if meta.Name != "" {
skillName = meta.Name
}
}
// If parsing fails, use default meta and skillName
}
// Fetch all files in the directory
files, err := s.fetchDirectoryContents(owner, repo, pathStr)
if err != nil {
return nil, fmt.Errorf("failed to fetch directory contents: %w", err)
}
return &SkillBundle{
Name: skillName,
Files: files,
Source: "github",
Identifier: identifier,
TrustLevel: s.TrustLevel(identifier),
Metadata: meta,
}, nil
}
// Inspect retrieves metadata from GitHub
func (s *GitHubSource) Inspect(identifier string) (*SkillMetadata, error) {
owner, repo, pathStr, err := parseGitHubURL(identifier)
if err != nil {
return nil, err
}
skillMdPath := path.Join(pathStr, "SKILL.md")
content, err := s.fetchFileContent(owner, repo, skillMdPath)
if err != nil {
// Return basic metadata if SKILL.md not found
return &SkillMetadata{
Name: repo,
Description: fmt.Sprintf("Skill from %s/%s", owner, repo),
Version: "1.0.0",
}, nil
}
meta, err := parseSkillFrontmatter(content)
if err != nil {
return nil, fmt.Errorf("invalid SKILL.md frontmatter in %s: %w", identifier, err)
}
return meta, nil
}
// fetchFileContent fetches a single file from GitHub
func (s *GitHubSource) fetchFileContent(owner, repo, filePath string) (string, error) {
var url string
if filePath == "" || filePath == "." {
url = fmt.Sprintf("https://api.github.com/repos/%s/%s/contents", owner, repo)
} else {
url = fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s", owner, repo, filePath)
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
req.Header.Set("User-Agent", "ragflow-cli")
resp, err := s.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GitHub API returned %d", resp.StatusCode)
}
var result struct {
Content string `json:"content"`
Encoding string `json:"encoding"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", err
}
if result.Encoding == "base64" {
decoded, err := base64.StdEncoding.DecodeString(result.Content)
if err != nil {
return "", err
}
return string(decoded), nil
}
return result.Content, nil
}
// fetchDirectoryContents recursively fetches directory contents from GitHub
func (s *GitHubSource) fetchDirectoryContents(owner, repo, dirPath string) (map[string][]byte, error) {
var url string
if dirPath == "" || dirPath == "." {
url = fmt.Sprintf("https://api.github.com/repos/%s/%s/contents", owner, repo)
} else {
url = fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s", owner, repo, dirPath)
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
req.Header.Set("User-Agent", "ragflow-cli")
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GitHub API returned %d", resp.StatusCode)
}
var items []struct {
Name string `json:"name"`
Path string `json:"path"`
Type string `json:"type"`
DownloadURL string `json:"download_url"`
}
if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
return nil, err
}
files := make(map[string][]byte)
for _, item := range items {
// Skip hidden files and common ignore patterns
if strings.HasPrefix(item.Name, ".") {
continue
}
if item.Name == "node_modules" || item.Name == "__pycache__" {
continue
}
if item.Type == "file" {
// Calculate relative path
relPath := item.Path
if dirPath != "" && dirPath != "." {
relPath = strings.TrimPrefix(item.Path, dirPath+"/")
}
content, err := s.downloadFile(item.DownloadURL)
if err != nil {
continue // Skip files we can't download
}
files[relPath] = content
} else if item.Type == "dir" {
// Recursively fetch subdirectory
subFiles, err := s.fetchDirectoryContents(owner, repo, item.Path)
if err != nil {
continue
}
for subPath, content := range subFiles {
relPath := subPath
if dirPath != "" && dirPath != "." {
relPath = strings.TrimPrefix(subPath, dirPath+"/")
}
files[relPath] = content
}
}
}
return files, nil
}
// downloadFile downloads a file from the given URL
func (s *GitHubSource) downloadFile(url string) ([]byte, error) {
resp, err := s.client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
return io.ReadAll(resp.Body)
}

177
internal/cli/filesystem/skill_hub/source/interface.go Normal file
View File

@@ -0,0 +1,177 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import (
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
)
// SkillSource is the interface for skill sources
type SkillSource interface {
// SourceID returns the source identifier (local, github, clawhub, skillssh)
SourceID() string
// Fetch downloads and returns the skill bundle
Fetch(identifier string) (*SkillBundle, error)
// Inspect retrieves metadata without downloading full content
Inspect(identifier string) (*SkillMetadata, error)
// TrustLevel returns the trust level for this source (builtin/trusted/community)
TrustLevel(identifier string) string
}
// SourceResolver resolves source references to appropriate adapters
type SourceResolver struct {
sources map[string]SkillSource
}
// NewSourceResolver creates a new source resolver
func NewSourceResolver(client HTTPClientInterface) *SourceResolver {
return &SourceResolver{
sources: map[string]SkillSource{
"local": NewLocalSource(),
"github": NewGitHubSource(client),
"clawhub": NewClawHubSource(client),
"skillssh": NewSkillsShSource(client),
},
}
}
// Resolve parses a source reference and returns the appropriate source adapter
// Supported formats:
// - ./path, /absolute/path -> local
// - github.com/owner/repo/path -> github
// - clawhub://owner/skill-name, clawhub.ai/owner/skill-name -> clawhub
// - skill://skill-name, skills.sh/skill/name -> skillssh
func (r *SourceResolver) Resolve(ref string) (SkillSource, string, error) {
ref = strings.TrimSpace(ref)
if ref == "" {
return nil, "", fmt.Errorf("empty source reference")
}
// Check for URI schemes
if strings.HasPrefix(ref, "clawhub://") {
identifier := strings.TrimPrefix(ref, "clawhub://")
return r.sources["clawhub"], identifier, nil
}
if strings.HasPrefix(ref, "skill://") {
identifier := strings.TrimPrefix(ref, "skill://")
return r.sources["skillssh"], identifier, nil
}
// Check for local path (starts with ./ or / or ~)
if strings.HasPrefix(ref, "./") || strings.HasPrefix(ref, "/") || strings.HasPrefix(ref, "~/") {
// Expand ~ to home directory
if strings.HasPrefix(ref, "~/") {
home, err := getHomeDir()
if err != nil {
return nil, "", fmt.Errorf("cannot resolve home directory: %w", err)
}
ref = filepath.Join(home, ref[2:])
}
return r.sources["local"], ref, nil
}
// Check for github.com domain
if strings.HasPrefix(ref, "github.com/") || strings.HasPrefix(ref, "https://github.com/") {
identifier := strings.TrimPrefix(ref, "https://")
return r.sources["github"], identifier, nil
}
// Check for clawhub.ai domain
if strings.HasPrefix(ref, "clawhub.ai/") || strings.HasPrefix(ref, "https://clawhub.ai/") {
identifier := strings.TrimPrefix(ref, "https://")
identifier = strings.TrimPrefix(identifier, "clawhub.ai/")
return r.sources["clawhub"], identifier, nil
}
// Check for skills.sh domain
if strings.HasPrefix(ref, "skills.sh/") || strings.HasPrefix(ref, "https://skills.sh/") {
identifier := strings.TrimPrefix(ref, "https://")
identifier = strings.TrimPrefix(identifier, "skills.sh/")
return r.sources["skillssh"], identifier, nil
}
// Default: treat as local path if it exists, otherwise error
return r.sources["local"], ref, nil
}
// getHomeDir returns the user's home directory
func getHomeDir() (string, error) {
home := os.Getenv("HOME")
if home == "" {
home = os.Getenv("USERPROFILE")
}
if home == "" {
return "", fmt.Errorf("cannot determine home directory")
}
return home, nil
}
// parseGitHubURL parses a GitHub URL and returns owner, repo, and path
func parseGitHubURL(urlStr string) (owner, repo, path string, err error) {
// Remove protocol prefix if present
urlStr = strings.TrimPrefix(urlStr, "https://")
urlStr = strings.TrimPrefix(urlStr, "http://")
// Remove github.com/ prefix
urlStr = strings.TrimPrefix(urlStr, "github.com/")
parts := strings.Split(urlStr, "/")
if len(parts) < 2 {
return "", "", "", fmt.Errorf("invalid GitHub URL format")
}
owner = parts[0]
repo = parts[1]
if len(parts) > 2 {
path = strings.Join(parts[2:], "/")
}
return owner, repo, path, nil
}
// extractSkillNameFromPath extracts the skill name from a path
func extractSkillNameFromPath(path string) string {
base := filepath.Base(path)
// Remove common suffixes
base = strings.TrimSuffix(base, ".git")
return base
}
// isTrustedGitHubRepo checks if a GitHub repo is trusted
func isTrustedGitHubRepo(owner, repo string) bool {
fullName := owner + "/" + repo
trusted := map[string]bool{
"openai/skills": true,
"anthropics/skills": true,
"microsoft/skills": true,
"google/skills": true,
}
return trusted[fullName]
}
// Helper to check if URL is valid
func isValidURL(str string) bool {
u, err := url.Parse(str)
return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
}

206
internal/cli/filesystem/skill_hub/source/local.go Normal file
View File

@@ -0,0 +1,206 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import (
"fmt"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
// LocalSource handles local filesystem skills
type LocalSource struct{}
// NewLocalSource creates a new local source adapter
func NewLocalSource() *LocalSource {
return &LocalSource{}
}
// SourceID returns the source identifier
func (s *LocalSource) SourceID() string {
return "local"
}
// TrustLevel returns the trust level for local sources
func (s *LocalSource) TrustLevel(identifier string) string {
return "community" // Local skills default to community trust level
}
// Fetch retrieves a skill from the local filesystem
func (s *LocalSource) Fetch(identifier string) (*SkillBundle, error) {
// Validate path exists
info, err := os.Stat(identifier)
if err != nil {
return nil, fmt.Errorf("cannot access path %s: %w", identifier, err)
}
if !info.IsDir() {
return nil, fmt.Errorf("%s is not a directory", identifier)
}
// Read SKILL.md
skillMdPath := filepath.Join(identifier, "SKILL.md")
content, err := os.ReadFile(skillMdPath)
if err != nil {
return nil, fmt.Errorf("SKILL.md not found in %s: %w", identifier, err)
}
// Parse frontmatter
meta, err := parseSkillFrontmatter(string(content))
if err != nil {
return nil, fmt.Errorf("invalid SKILL.md frontmatter in %s: %w", identifier, err)
}
skillName := meta.Name
if skillName == "" {
skillName = filepath.Base(identifier)
}
// Collect all files
files := make(map[string][]byte)
ignorePatterns := []string{
".git/", ".svn/", ".hg/", "node_modules/", "__MACOSX/",
".DS_Store", "._*", "*.log", "*.tmp", "*.temp", "*.swp", "*.swo", "*~",
".env", ".env.*", ".vscode/", ".idea/", "Thumbs.db", "desktop.ini",
}
err = filepath.Walk(identifier, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
// Skip non-regular files (symlinks, devices, pipes, etc.)
if !info.Mode().IsRegular() {
return nil
}
relPath, err := filepath.Rel(identifier, path)
if err != nil {
return err
}
// Check ignore patterns
for _, pattern := range ignorePatterns {
if matched, _ := filepath.Match(pattern, relPath); matched {
return nil
}
if strings.Contains(relPath, pattern) {
return nil
}
}
// Only include text files based on extension
if !isTextFile(path) {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
files[relPath] = data
return nil
})
if err != nil {
return nil, err
}
return &SkillBundle{
Name: skillName,
Files: files,
Source: "local",
Identifier: identifier,
TrustLevel: s.TrustLevel(identifier),
Metadata: meta,
}, nil
}
// Inspect retrieves metadata without reading all files
func (s *LocalSource) Inspect(identifier string) (*SkillMetadata, error) {
info, err := os.Stat(identifier)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("not a directory")
}
skillMdPath := filepath.Join(identifier, "SKILL.md")
content, err := os.ReadFile(skillMdPath)
if err != nil {
return nil, err
}
meta, err := parseSkillFrontmatter(string(content))
if err != nil {
return nil, fmt.Errorf("invalid SKILL.md frontmatter in %s: %w", identifier, err)
}
if meta.Name == "" {
meta.Name = filepath.Base(identifier)
}
return meta, nil
}
// parseSkillFrontmatter extracts YAML frontmatter from SKILL.md content
// Returns an error if frontmatter delimiters are missing or YAML is invalid
func parseSkillFrontmatter(content string) (*SkillMetadata, error) {
meta := &SkillMetadata{}
// Look for YAML frontmatter
content = strings.TrimSpace(content)
if !strings.HasPrefix(content, "---") {
return nil, fmt.Errorf("missing opening frontmatter delimiter '---'")
}
// Find end of frontmatter
endIdx := strings.Index(content[3:], "---")
if endIdx == -1 {
return nil, fmt.Errorf("missing closing frontmatter delimiter '---'")
}
frontmatter := content[3 : endIdx+3]
if err := yaml.Unmarshal([]byte(frontmatter), meta); err != nil {
return nil, fmt.Errorf("invalid YAML frontmatter: %w", err)
}
return meta, nil
}
// isTextFile checks if a file is a text file based on extension
func isTextFile(filename string) bool {
ext := strings.ToLower(filepath.Ext(filename))
if ext != "" && ext[0] == '.' {
ext = ext[1:]
}
textExts := map[string]bool{
"md": true, "mdx": true, "txt": true, "json": true, "json5": true,
"yaml": true, "yml": true, "toml": true, "js": true, "cjs": true, "mjs": true,
"ts": true, "tsx": true, "jsx": true, "py": true, "sh": true, "rb": true,
"go": true, "rs": true, "swift": true, "kt": true, "java": true, "cs": true,
"cpp": true, "c": true, "h": true, "hpp": true, "sql": true, "csv": true,
"ini": true, "cfg": true, "env": true, "xml": true, "html": true,
"css": true, "scss": true, "sass": true, "svg": true,
}
return textExts[ext]
}

574
internal/cli/filesystem/skill_hub/source/skillssh.go Normal file
View File

@@ -0,0 +1,574 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import (
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
)
const (
skillsShBaseURL = "https://skills.sh"
)
var (
// Regex patterns for parsing skills.sh detail page
skillsShInstallCmdRe = regexp.MustCompile(`(?i)npx\s+skills\s+add\s+(?P<repo>https?://github\.com/[^\s<]+|[^\s<]+)(?:\s+--skill\s+(?P<skill>[^\s<]+))?`)
skillsShPageH1Re = regexp.MustCompile(`(?i)<h1[^>]*>(?P<title>.*?)</h1>`)
skillsShProseH1Re = regexp.MustCompile(`(?i)<div[^>]*class=["'][^"']*prose[^"']*["'][^>]*>.*?<h1[^>]*>(?P<title>.*?)</h1>`)
skillsShProsePRe = regexp.MustCompile(`(?i)<div[^>]*class=["'][^"']*prose[^"']*["'][^>]*>.*?<p[^>]*>(?P<body>.*?)</p>`)
skillsShWeeklyRe = regexp.MustCompile(`Weekly Installs.*?children\\":\\"(?P<count>[0-9.,Kk]+)\\"`)
)
// SkillsShDetail holds parsed information from skills.sh detail page
type SkillsShDetail struct {
Repo string `json:"repo"`
InstallSkill string `json:"install_skill"`
PageTitle string `json:"page_title"`
BodyTitle string `json:"body_title"`
BodySummary string `json:"body_summary"`
WeeklyInstalls string `json:"weekly_installs"`
InstallCommand string `json:"install_command"`
RepoURL string `json:"repo_url"`
DetailURL string `json:"detail_url"`
}
// SkillsShSource handles skills.sh registry skills
type SkillsShSource struct {
client HTTPClientInterface
github *GitHubSource
}
// NewSkillsShSource creates a new skills.sh source adapter
func NewSkillsShSource(client HTTPClientInterface) *SkillsShSource {
return &SkillsShSource{
client: client,
github: NewGitHubSource(client),
}
}
// SourceID returns the source identifier
func (s *SkillsShSource) SourceID() string {
return "skills-sh"
}
// TrustLevel returns the trust level for skills.sh
func (s *SkillsShSource) TrustLevel(identifier string) string {
canonical := s.normalizeIdentifier(identifier)
// Delegate to github trust level based on the repo
for _, candidate := range s.candidateIdentifiers(canonical) {
if level := s.github.TrustLevel(candidate); level != "community" {
return level
}
}
return "community"
}
// Fetch retrieves a skill from skills.sh
func (s *SkillsShSource) Fetch(identifier string) (*SkillBundle, error) {
canonical := s.normalizeIdentifier(identifier)
// Fetch detail page from skills.sh
detail, err := s.fetchDetailPage(canonical)
if err != nil {
// Continue without detail info
detail = nil
}
// Try candidate identifiers
for _, candidate := range s.candidateIdentifiers(canonical) {
bundle, err := s.github.Fetch(candidate)
if err == nil && bundle != nil {
// Validate SKILL.md exists
if _, ok := bundle.Files["SKILL.md"]; !ok {
continue
}
// Update bundle with skills.sh info
bundle.Source = "skills-sh"
bundle.Identifier = s.wrapIdentifier(canonical)
bundle.TrustLevel = s.TrustLevel(identifier)
if detail != nil {
bundle.Metadata = s.mergeDetailMetadata(bundle.Metadata, detail, canonical)
}
return bundle, nil
}
}
// Try to discover identifier
resolved, err := s.discoverIdentifier(canonical, detail)
if err == nil && resolved != "" {
bundle, err := s.github.Fetch(resolved)
if err == nil && bundle != nil {
// Validate SKILL.md exists
if _, ok := bundle.Files["SKILL.md"]; !ok {
return nil, fmt.Errorf("skill missing required SKILL.md file")
}
bundle.Source = "skills-sh"
bundle.Identifier = s.wrapIdentifier(canonical)
bundle.TrustLevel = s.TrustLevel(identifier)
if detail != nil {
bundle.Metadata = s.mergeDetailMetadata(bundle.Metadata, detail, canonical)
}
return bundle, nil
}
}
return nil, fmt.Errorf("skill not found: %s", identifier)
}
// Inspect retrieves metadata from skills.sh
func (s *SkillsShSource) Inspect(identifier string) (*SkillMetadata, error) {
canonical := s.normalizeIdentifier(identifier)
// Fetch detail page
detail, err := s.fetchDetailPage(canonical)
if err != nil {
detail = nil
}
// Try to get metadata from github
meta, err := s.resolveGitHubMeta(canonical, detail)
if err != nil {
return nil, err
}
// Update with skills.sh info
meta = s.finalizeInspectMeta(meta, canonical, detail)
return meta, nil
}
// normalizeIdentifier removes skills.sh prefixes
func (s *SkillsShSource) normalizeIdentifier(identifier string) string {
prefixes := []string{
"skills-sh/",
"skills.sh/",
"skils-sh/",
"skils.sh/",
}
for _, prefix := range prefixes {
if strings.HasPrefix(identifier, prefix) {
return identifier[len(prefix):]
}
}
return identifier
}
// wrapIdentifier adds skills-sh prefix
func (s *SkillsShSource) wrapIdentifier(identifier string) string {
return "skills-sh/" + identifier
}
// candidateIdentifiers generates possible GitHub paths for a skill
func (s *SkillsShSource) candidateIdentifiers(identifier string) []string {
parts := strings.SplitN(identifier, "/", 3)
if len(parts) < 3 {
return []string{identifier}
}
repo := parts[0] + "/" + parts[1]
skillPath := strings.TrimPrefix(parts[2], "/")
candidates := []string{
fmt.Sprintf("github.com/%s/%s", repo, skillPath),
fmt.Sprintf("github.com/%s/skills/%s", repo, skillPath),
fmt.Sprintf("github.com/%s/.agents/skills/%s", repo, skillPath),
fmt.Sprintf("github.com/%s/.claude/skills/%s", repo, skillPath),
}
// Deduplicate
seen := make(map[string]bool)
result := []string{}
for _, c := range candidates {
if !seen[c] {
seen[c] = true
result = append(result, c)
}
}
return result
}
// fetchDetailPage fetches and parses skills.sh detail page
func (s *SkillsShSource) fetchDetailPage(identifier string) (*SkillsShDetail, error) {
url := fmt.Sprintf("%s/%s", skillsShBaseURL, identifier)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
resp, err := s.client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch detail page: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("skills.sh returned %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return s.parseDetailPage(identifier, string(body)), nil
}
// parseDetailPage extracts information from skills.sh HTML
func (s *SkillsShSource) parseDetailPage(identifier, html string) *SkillsShDetail {
parts := strings.SplitN(identifier, "/", 3)
if len(parts) < 3 {
return nil
}
defaultRepo := parts[0] + "/" + parts[1]
skillToken := parts[2]
repo := defaultRepo
installSkill := skillToken
// Extract install command
installCmd := ""
if match := skillsShInstallCmdRe.FindStringSubmatch(html); match != nil {
installCmd = strings.TrimSpace(match[0])
repoValue := strings.TrimSpace(s.extractGroup(skillsShInstallCmdRe, match, "repo"))
skillValue := strings.TrimSpace(s.extractGroup(skillsShInstallCmdRe, match, "skill"))
if skillValue != "" {
installSkill = skillValue
}
if extracted := s.extractRepoSlug(repoValue); extracted != "" {
repo = extracted
}
}
return &SkillsShDetail{
Repo: repo,
InstallSkill: installSkill,
PageTitle: s.extractFirstMatch(skillsShPageH1Re, html),
BodyTitle: s.extractFirstMatch(skillsShProseH1Re, html),
BodySummary: s.extractFirstMatch(skillsShProsePRe, html),
WeeklyInstalls: s.extractWeeklyInstalls(html),
InstallCommand: installCmd,
RepoURL: fmt.Sprintf("https://github.com/%s", repo),
DetailURL: fmt.Sprintf("%s/%s", skillsShBaseURL, identifier),
}
}
// discoverIdentifier tries to find the skill in non-standard locations
func (s *SkillsShSource) discoverIdentifier(identifier string, detail *SkillsShDetail) (string, error) {
parts := strings.SplitN(identifier, "/", 3)
if len(parts) < 3 {
return "", fmt.Errorf("invalid identifier format")
}
defaultRepo := parts[0] + "/" + parts[1]
repo := defaultRepo
if detail != nil && detail.Repo != "" {
repo = detail.Repo
}
skillToken := parts[2]
tokens := []string{skillToken}
if detail != nil {
tokens = append(tokens, detail.InstallSkill, detail.PageTitle, detail.BodyTitle)
}
// Try standard skill paths
basePaths := []string{"skills/", ".agents/skills/", ".claude/skills/"}
for _, basePath := range basePaths {
candidate := fmt.Sprintf("github.com/%s/%s%s", repo, basePath, skillToken)
meta, err := s.github.Inspect(candidate)
if err == nil && meta != nil {
return candidate, nil
}
}
// Try tree lookup for nested skills
treeResult, err := s.findSkillInRepoTree(repo, skillToken)
if err == nil && treeResult != "" {
return treeResult, nil
}
// Scan repo root directories
rootURL := fmt.Sprintf("https://api.github.com/repos/%s/contents/", repo)
req, err := http.NewRequest("GET", rootURL, nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
req.Header.Set("User-Agent", "ragflow-cli")
resp, err := s.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("github API returned %d", resp.StatusCode)
}
var entries []struct {
Name string `json:"name"`
Type string `json:"type"`
}
if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
return "", err
}
for _, entry := range entries {
if entry.Type != "dir" {
continue
}
if strings.HasPrefix(entry.Name, ".") || strings.HasPrefix(entry.Name, "_") {
continue
}
if entry.Name == "skills" || entry.Name == ".agents" || entry.Name == ".claude" {
continue // Already tried
}
// Try direct match
directID := fmt.Sprintf("github.com/%s/%s/%s", repo, entry.Name, skillToken)
meta, err := s.github.Inspect(directID)
if err == nil && meta != nil {
return directID, nil
}
}
return "", fmt.Errorf("skill not found in repo")
}
// findSkillInRepoTree searches for skill in repo tree
func (s *SkillsShSource) findSkillInRepoTree(repo, skillToken string) (string, error) {
// Get repo tree
url := fmt.Sprintf("https://api.github.com/repos/%s/git/trees/HEAD?recursive=1", repo)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
req.Header.Set("User-Agent", "ragflow-cli")
resp, err := s.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("github API returned %d", resp.StatusCode)
}
var result struct {
Tree []struct {
Path string `json:"path"`
Type string `json:"type"`
} `json:"tree"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", err
}
// Look for skill directories matching the token
for _, item := range result.Tree {
if item.Type != "tree" {
continue
}
parts := strings.Split(item.Path, "/")
if len(parts) == 0 {
continue
}
dirName := parts[len(parts)-1]
if s.matchesSkillToken(dirName, skillToken) {
return fmt.Sprintf("github.com/%s/%s", repo, item.Path), nil
}
}
return "", fmt.Errorf("skill not found in tree")
}
// matchesSkillToken checks if a directory name matches skill token
func (s *SkillsShSource) matchesSkillToken(dirName, skillToken string) bool {
variants := s.tokenVariants(dirName)
tokenVariants := s.tokenVariants(skillToken)
for v := range tokenVariants {
if variants[v] {
return true
}
}
return false
}
// tokenVariants generates normalized token variants
func (s *SkillsShSource) tokenVariants(value string) map[string]bool {
variants := make(map[string]bool)
if value == "" {
return variants
}
value = strings.ToLower(strings.TrimSpace(value))
if value == "" {
return variants
}
// Base name (last path component)
parts := strings.Split(value, "/")
base := parts[len(parts)-1]
// Clean variant
clean := strings.TrimPrefix(base, "@")
variants[value] = true
variants[strings.ReplaceAll(value, "_", "-")] = true
variants[strings.ReplaceAll(value, "/", "-")] = true
variants[base] = true
variants[strings.ReplaceAll(base, "_", "-")] = true
variants[clean] = true
variants[strings.ReplaceAll(clean, "_", "-")] = true
return variants
}
// resolveGitHubMeta tries to get metadata from GitHub
func (s *SkillsShSource) resolveGitHubMeta(identifier string, detail *SkillsShDetail) (*SkillMetadata, error) {
for _, candidate := range s.candidateIdentifiers(identifier) {
meta, err := s.github.Inspect(candidate)
if err == nil && meta != nil {
return meta, nil
}
}
resolved, err := s.discoverIdentifier(identifier, detail)
if err == nil && resolved != "" {
return s.github.Inspect(resolved)
}
return nil, fmt.Errorf("skill metadata not found")
}
// finalizeInspectMeta updates metadata with skills.sh info
func (s *SkillsShSource) finalizeInspectMeta(meta *SkillMetadata, canonical string, detail *SkillsShDetail) *SkillMetadata {
if meta == nil {
meta = &SkillMetadata{}
}
meta = &SkillMetadata{
Name: meta.Name,
Description: meta.Description,
Version: meta.Version,
Author: meta.Author,
Tags: meta.Tags,
Tools: meta.Tools,
}
// Use body summary as description if available
if detail != nil && detail.BodySummary != "" {
meta.Description = s.stripHTML(detail.BodySummary)
} else if detail != nil && detail.WeeklyInstalls != "" && meta.Description != "" {
meta.Description = fmt.Sprintf("%s · %s weekly installs on skills.sh", meta.Description, detail.WeeklyInstalls)
}
return meta
}
// mergeDetailMetadata merges skills.sh detail into bundle metadata
func (s *SkillsShSource) mergeDetailMetadata(meta *SkillMetadata, detail *SkillsShDetail, canonical string) *SkillMetadata {
if meta == nil {
meta = &SkillMetadata{}
}
// Create new metadata to avoid modifying the original
merged := &SkillMetadata{
Name: meta.Name,
Description: meta.Description,
Version: meta.Version,
Author: meta.Author,
Tags: meta.Tags,
Tools: meta.Tools,
}
if detail.BodySummary != "" {
merged.Description = s.stripHTML(detail.BodySummary)
}
return merged
}
// extractFirstMatch extracts first matching group from regex
func (s *SkillsShSource) extractFirstMatch(re *regexp.Regexp, text string) string {
match := re.FindStringSubmatch(text)
if match == nil {
return ""
}
for i, name := range re.SubexpNames() {
if i > 0 && i < len(match) && name != "" {
return s.stripHTML(strings.TrimSpace(match[i]))
}
}
return ""
}
// extractGroup extracts a named group from regex match
// The regex must be passed to map group names to capture indices
func (s *SkillsShSource) extractGroup(re *regexp.Regexp, match []string, name string) string {
if re == nil || match == nil || name == "" {
return ""
}
for i, groupName := range re.SubexpNames() {
if i >= 0 && i < len(match) && groupName == name {
return match[i]
}
}
return ""
}
// extractWeeklyInstalls extracts weekly install count
func (s *SkillsShSource) extractWeeklyInstalls(html string) string {
match := skillsShWeeklyRe.FindStringSubmatch(html)
if match == nil {
return ""
}
for i, name := range skillsShWeeklyRe.SubexpNames() {
if i > 0 && i < len(match) && name == "count" {
return match[i]
}
}
return ""
}
// extractRepoSlug extracts owner/repo from URL or string
func (s *SkillsShSource) extractRepoSlug(value string) string {
value = strings.TrimSpace(value)
value = strings.TrimPrefix(value, "https://github.com/")
value = strings.Trim(value, "/")
parts := strings.Split(value, "/")
if len(parts) >= 2 {
return parts[0] + "/" + parts[1]
}
return ""
}
// stripHTML removes HTML tags
func (s *SkillsShSource) stripHTML(value string) string {
// Simple HTML tag removal
re := regexp.MustCompile(`<[^>]+>`)
return strings.TrimSpace(re.ReplaceAllString(value, ""))
}

47
internal/cli/filesystem/skill_hub/source/types.go Normal file
View File

@@ -0,0 +1,47 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package source
import "net/http"
// HTTPClientInterface defines the interface for HTTP operations
// This is duplicated here to avoid circular imports
type HTTPClientInterface interface {
Do(req *http.Request) (*http.Response, error)
Get(url string) (*http.Response, error)
}
// SkillMetadata represents the metadata from SKILL.md frontmatter
// This is duplicated here to avoid circular imports
type SkillMetadata struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
Version string `yaml:"version"`
Author string `yaml:"author"`
Tags []string `yaml:"tags"`
Tools interface{} `yaml:"tools"`
}
// SkillBundle represents a downloaded skill package
type SkillBundle struct {
Name string
Files map[string][]byte
Source string
Identifier string
TrustLevel string
Metadata *SkillMetadata
}

437
internal/cli/filesystem/skill_install.go Normal file
View File

@@ -0,0 +1,437 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package filesystem
import (
stdctx "context"
"fmt"
"net/http"
"net/http/cookiejar"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/net/http2"
"golang.org/x/net/publicsuffix"
"ragflow/internal/cli/filesystem/skill_hub/security"
"ragflow/internal/cli/filesystem/skill_hub/source"
)
// InstallSkillArgs holds the parsed arguments for install-skill command
type InstallSkillArgs struct {
SpaceID string // Target skills space ID
SourceRef string // Source reference (path or identifier)
Version string // Skill version
SkillName string // Optional: override skill name
Force bool // Force reinstall
SkipVerify bool // Skip security verification
ShowHelp bool
}
// SkillInstallCommand handles the install-skill command
type SkillInstallCommand struct {
client HTTPClientInterface
fileProvider *FileProvider
skillProvider Provider
scanner *security.Scanner
guard *security.Guard
sourceResolver *source.SourceResolver
}
// sourceHTTPClientAdapter adapts filesystem.HTTPClientInterface to source.HTTPClientInterface
// This allows us to use the existing HTTP client infrastructure with the source package
type sourceHTTPClientAdapter struct {
client HTTPClientInterface
httpClient *http.Client
}
func (a *sourceHTTPClientAdapter) Do(req *http.Request) (*http.Response, error) {
// Use standard http.Client for direct requests (e.g., GitHub API)
// This bypasses the RAGFlow API client which adds its own base URL
return a.httpClient.Do(req)
}
func (a *sourceHTTPClientAdapter) Get(url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return a.Do(req)
}
// NewInstallSkillCommand creates a new install-skill command handler
func NewInstallSkillCommand(client HTTPClientInterface, fileProvider *FileProvider, skillProvider Provider) *SkillInstallCommand {
// Log proxy settings
if httpProxy := os.Getenv("http_proxy"); httpProxy != "" {
fmt.Printf("Using HTTP proxy: %s\n", httpProxy)
}
if httpsProxy := os.Getenv("https_proxy"); httpsProxy != "" {
fmt.Printf("Using HTTPS proxy: %s\n", httpsProxy)
}
// Create transport with HTTP/2 support and connection reuse
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
// Enable connection pooling
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
// Enable keep-alive
DisableKeepAlives: false,
ForceAttemptHTTP2: true,
}
// Enable HTTP/2
http2.ConfigureTransport(transport)
// Check what proxy will be used
testURL, _ := url.Parse("https://github.com")
if proxy, err := transport.Proxy(&http.Request{URL: testURL}); err == nil && proxy != nil {
fmt.Printf("Proxy enabled for GitHub: %s\n", proxy.String())
} else if err != nil {
fmt.Printf("Warning: proxy detection error: %v\n", err)
}
// Create cookie jar for session persistence
jar, err := cookiejar.New(&cookiejar.Options{
PublicSuffixList: publicsuffix.List,
})
if err != nil {
fmt.Printf("Warning: failed to create cookie jar: %v\n", err)
jar = nil
}
// Wrap client with adapter - use standard http.Client with timeout for direct external requests
adaptedClient := &sourceHTTPClientAdapter{
client: client,
httpClient: &http.Client{
Timeout: 60 * time.Second,
Transport: transport,
Jar: jar,
},
}
return &SkillInstallCommand{
client: client,
fileProvider: fileProvider,
skillProvider: skillProvider,
scanner: security.NewScanner(),
guard: security.NewGuard(),
sourceResolver: source.NewSourceResolver(adaptedClient),
}
}
// Execute runs the install-skill command
func (c *SkillInstallCommand) Execute(args []string) error {
parsedArgs, err := c.parseArgs(args)
if err != nil {
return err
}
if parsedArgs.ShowHelp {
c.PrintHelp()
return nil
}
ctx := stdctx.Background()
// 1. Resolve source
fmt.Printf("Resolving source reference: %s\n", parsedArgs.SourceRef)
src, identifier, err := c.sourceResolver.Resolve(parsedArgs.SourceRef)
if err != nil {
return fmt.Errorf("invalid source reference: %w", err)
}
// 2. Fetch skill bundle
// If version specified, append to identifier for sources that support it
fetchIdentifier := identifier
if parsedArgs.Version != "" {
fetchIdentifier = fmt.Sprintf("%s@%s", identifier, parsedArgs.Version)
fmt.Printf("Fetching skill from %s (version %s)...\n", src.SourceID(), parsedArgs.Version)
} else {
fmt.Printf("Fetching skill from %s...\n", src.SourceID())
}
bundle, err := src.Fetch(fetchIdentifier)
if err != nil {
return fmt.Errorf("failed to fetch skill: %w", err)
}
fmt.Printf("Found skill '%s' (v%s) with %d files\n",
bundle.Name, bundle.Metadata.Version, len(bundle.Files))
// Override skill name if specified
if parsedArgs.SkillName != "" {
bundle.Name = parsedArgs.SkillName
}
// 3. Check if skill already exists
exists, err := c.skillExists(ctx, parsedArgs.SpaceID, bundle.Name)
if err != nil {
return fmt.Errorf("failed to check existing skill: %w", err)
}
if exists && !parsedArgs.Force {
return fmt.Errorf("skill '%s' already exists in space '%s'. Use --force to reinstall", bundle.Name, parsedArgs.SpaceID)
}
// 4. Security scan (unless skipped)
if !parsedArgs.SkipVerify {
fmt.Println("Running security scan...")
trustLevel := src.TrustLevel(identifier)
scanResult := c.scanner.ScanSkill(bundle.Name, src.SourceID(), trustLevel, bundle.Files)
allowed, reason := c.guard.ShouldAllowInstall(scanResult, parsedArgs.Force)
if !allowed {
fmt.Println(c.guard.FormatScanReport(scanResult))
return fmt.Errorf("installation blocked: %s", reason)
}
fmt.Println(c.guard.FormatScanReport(scanResult))
fmt.Printf("✓ Security check passed: %s\n\n", reason)
}
// 5. Force mode: delete existing skill first
if parsedArgs.Force && exists {
fmt.Printf("Force mode: removing existing skill '%s'...\n", bundle.Name)
if err := c.uninstallSkill(ctx, parsedArgs.SpaceID, bundle.Name); err != nil {
return fmt.Errorf("failed to remove existing skill: %w", err)
}
fmt.Println()
}
// 6. Install skill
fmt.Printf("Installing skill '%s' to space '%s'...\n", bundle.Name, parsedArgs.SpaceID)
if err := c.installSkill(ctx, parsedArgs.SpaceID, bundle, parsedArgs.Force); err != nil {
return fmt.Errorf("failed to install skill: %w", err)
}
// 7. Update index
fmt.Printf("Updating search index for skill '%s'...\n", bundle.Name)
if err := c.updateIndex(ctx, parsedArgs.SpaceID, bundle.Name); err != nil {
fmt.Printf("⚠ Warning: failed to update index: %v\n", err)
}
fmt.Printf("✓ Successfully installed skill '%s' (version: %s)\n", bundle.Name, bundle.Metadata.Version)
return nil
}
// uninstallSkill removes an existing skill (for --force mode)
func (c *SkillInstallCommand) uninstallSkill(ctx stdctx.Context, spaceID, skillName string) error {
var indexErr, folderErr error
// Delete index
if skillProv, ok := c.skillProvider.(*SkillProvider); ok {
if err := skillProv.DeleteSkill(ctx, spaceID, skillName); err != nil {
indexErr = fmt.Errorf("failed to delete search index: %w", err)
fmt.Printf("⚠ Warning: %v\n", indexErr)
} else {
fmt.Printf("✓ Search index deleted\n")
}
}
// Delete folder
if c.fileProvider != nil {
folderPath := fmt.Sprintf("skills/%s/%s", spaceID, skillName)
if err := c.fileProvider.DeleteFolderByPath(ctx, folderPath); err != nil {
folderErr = fmt.Errorf("failed to delete skill folder: %w", err)
fmt.Printf("⚠ Warning: %v\n", folderErr)
} else {
fmt.Printf("✓ Skill folder deleted\n")
}
}
// Return error if both failed
if indexErr != nil && folderErr != nil {
return fmt.Errorf("failed to uninstall: index (%v), folder (%v)", indexErr, folderErr)
}
return nil
}
// installSkill installs a skill bundle using existing SkillUploader
func (c *SkillInstallCommand) installSkill(ctx stdctx.Context, spaceID string, bundle *source.SkillBundle, force bool) error {
// Create a temporary directory to hold the skill files
tempDir, err := os.MkdirTemp("", "skill-install-*")
if err != nil {
return fmt.Errorf("failed to create temp directory: %w", err)
}
defer os.RemoveAll(tempDir)
// Write files to temp directory
skillDir := filepath.Join(tempDir, bundle.Name)
if err := os.MkdirAll(skillDir, 0755); err != nil {
return fmt.Errorf("failed to create skill directory: %w", err)
}
for relPath, content := range bundle.Files {
filePath := filepath.Join(skillDir, relPath)
dir := filepath.Dir(filePath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err)
}
if err := os.WriteFile(filePath, content, 0644); err != nil {
return fmt.Errorf("failed to write file %s: %w", relPath, err)
}
}
// Use existing SkillUploader to upload the skill
uploader := NewSkillUploader(c.client, c.fileProvider)
uploader.SetSkillProvider(c.skillProvider)
uploader.SetForce(force)
version := bundle.Metadata.Version
if version == "" {
version = "1.0.0"
}
return uploader.UploadSkill(ctx, skillDir, version, fmt.Sprintf("skills/%s", spaceID), bundle.Name)
}
// skillExists checks if a skill already exists
func (c *SkillInstallCommand) skillExists(ctx stdctx.Context, spaceID, skillName string) (bool, error) {
folderPath := fmt.Sprintf("skills/%s/%s", spaceID, skillName)
_, err := c.fileProvider.List(ctx, folderPath, nil)
if err != nil {
// If error, likely doesn't exist
return false, nil
}
return true, nil
}
// updateIndex updates the search index for a skill
// Note: Indexing is now handled by SkillUploader during upload
func (c *SkillInstallCommand) updateIndex(ctx stdctx.Context, spaceID, skillName string) error {
// Indexing is automatically performed by SkillUploader.UploadSkill
// This method is kept for potential future use
return nil
}
// parseArgs parses command arguments
func (c *SkillInstallCommand) parseArgs(args []string) (*InstallSkillArgs, error) {
result := &InstallSkillArgs{}
var nonFlagArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch arg {
case "-h", "--help":
result.ShowHelp = true
return result, nil
case "-v", "--version":
if i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
result.Version = args[i+1]
i++
} else {
return nil, fmt.Errorf("version flag requires a value")
}
case "-n", "--name":
if i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
result.SkillName = args[i+1]
i++
} else {
return nil, fmt.Errorf("name flag requires a value")
}
case "-f", "--force":
result.Force = true
case "--skip-verify":
result.SkipVerify = true
default:
if !strings.HasPrefix(arg, "-") {
nonFlagArgs = append(nonFlagArgs, arg)
}
}
}
// Parse space and source ref
if len(nonFlagArgs) < 1 {
return nil, fmt.Errorf("space ID is required")
}
if len(nonFlagArgs) < 2 {
return nil, fmt.Errorf("source reference is required (local path or remote identifier)")
}
result.SpaceID = nonFlagArgs[0]
result.SourceRef = nonFlagArgs[1]
return result, nil
}
// PrintHelp prints the help message
func (c *SkillInstallCommand) PrintHelp() {
fmt.Println(`Usage: install-skill <space> <source> [options]
Install a skill from multiple sources into a RAGFlow space.
Arguments:
<space> Target skills space ID (required)
<source> Skill source reference (required):
- Local: ./path/to/skill or /absolute/path
- GitHub: github.com/owner/repo/path/to/skill
- ClawHub: clawhub://owner/skill-name or clawhub.ai/owner/skill-name
- skills.sh: skill://skill-name or skills.sh/skill/name
Options:
-v, --version string Specify skill version (default: from SKILL.md or 1.0.0)
-n, --name string Override skill name (default: from SKILL.md)
-f, --force Force reinstall if skill exists (deletes existing first)
--skip-verify Skip security verification (use with caution)
-h, --help Show this help message
Security:
By default, all skills are scanned for potential security threats before
installation. The scan checks for:
- Data exfiltration patterns (curl $SECRET, .ssh access, etc.)
- Prompt injection attempts (DAN mode, ignore instructions, etc.)
- Destructive commands (rm -rf /, mkfs, etc.)
- Persistence mechanisms (cron, .bashrc, authorized_keys, etc.)
- Network threats (reverse shells, tunneling, etc.)
- Obfuscation (base64 | bash, eval(), etc.)
Trust levels:
- builtin: Official RAGFlow skills (always allowed)
- trusted: openai/skills, anthropics/skills (caution allowed)
- community: All other sources (findings blocked unless --force)
Examples:
# Install from local path
install-skill my-space ./my-local-skill
# Install from GitHub
install-skill my-space github.com/openai/skills/skill-creator
# Force reinstall (delete existing and reinstall)
install-skill my-space ./my-skill --force
# Force install with custom name, skip security check
install-skill my-space claw://unknown-skill --force --name my-skill --skip-verify
# Install specific version
install-skill my-space skill://kubernetes --version 2.1.0
Note: 'add-skill' command is deprecated. Use 'install-skill' instead.`)
}
// getDir extracts directory from file path
func getDir(path string) string {
idx := strings.LastIndex(path, "/")
if idx == -1 {
return ""
}
return path[:idx]
}

166
internal/cli/filesystem/skill_uninstall.go Normal file
View File

@@ -0,0 +1,166 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package filesystem
import (
stdctx "context"
"fmt"
"strings"
)
// UninstallSkillArgs holds the parsed arguments for uninstall-skill command
type UninstallSkillArgs struct {
SkillName string
SpaceID string
ShowHelp bool
}
// SkillUninstallCommand handles the uninstall-skill command
type SkillUninstallCommand struct {
client HTTPClientInterface
skillProvider Provider
fileProvider *FileProvider
}
// NewUninstallSkillCommand creates a new uninstall-skill command handler
func NewUninstallSkillCommand(client HTTPClientInterface, skillProvider Provider, fileProvider *FileProvider) *SkillUninstallCommand {
return &SkillUninstallCommand{
client: client,
skillProvider: skillProvider,
fileProvider: fileProvider,
}
}
// Execute runs the uninstall-skill command
func (c *SkillUninstallCommand) Execute(args []string) error {
parsedArgs, err := c.parseArgs(args)
if err != nil {
return err
}
if parsedArgs.ShowHelp {
c.PrintHelp()
return nil
}
return c.uninstallSkill(stdctx.Background(), parsedArgs.SpaceID, parsedArgs.SkillName)
}
// uninstallSkill deletes a skill and its index
func (c *SkillUninstallCommand) uninstallSkill(ctx stdctx.Context, spaceID, skillName string) error {
if c.skillProvider == nil {
return fmt.Errorf("skill provider not available")
}
fmt.Printf("Uninstalling skill '%s' from space '%s'...\n\n", skillName, spaceID)
var indexErr, folderErr error
// 1. Delete search index
skillProvider, ok := c.skillProvider.(*SkillProvider)
if ok {
fmt.Printf("Deleting search index for skill '%s'...\n", skillName)
if err := skillProvider.DeleteSkill(ctx, spaceID, skillName); err != nil {
indexErr = fmt.Errorf("failed to delete search index: %w", err)
fmt.Printf("⚠ %v\n", indexErr)
} else {
fmt.Printf("✓ Search index deleted\n")
}
}
// 2. Delete file system folder
if c.fileProvider != nil {
fmt.Printf("Deleting skill folder '%s/%s'...\n", spaceID, skillName)
folderPath := fmt.Sprintf("skills/%s/%s", spaceID, skillName)
if err := c.fileProvider.DeleteFolderByPath(ctx, folderPath); err != nil {
folderErr = fmt.Errorf("failed to delete skill folder: %w", err)
fmt.Printf("⚠ %v\n", folderErr)
} else {
fmt.Printf("✓ Skill folder deleted\n")
}
}
// 3. Report results
fmt.Println()
if indexErr != nil && folderErr != nil {
return fmt.Errorf("failed to completely uninstall skill '%s': index deletion failed (%v), folder deletion failed (%v)",
skillName, indexErr, folderErr)
}
if indexErr != nil {
return fmt.Errorf("failed to uninstall skill '%s': %w", skillName, indexErr)
}
if folderErr != nil {
return fmt.Errorf("failed to uninstall skill '%s': %w", skillName, folderErr)
}
fmt.Printf("✓ Successfully uninstalled skill '%s'\n", skillName)
return nil
}
// parseArgs parses command arguments
func (c *SkillUninstallCommand) parseArgs(args []string) (*UninstallSkillArgs, error) {
result := &UninstallSkillArgs{}
var nonFlagArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch arg {
case "-h", "--help":
result.ShowHelp = true
return result, nil
default:
if !strings.HasPrefix(arg, "-") {
nonFlagArgs = append(nonFlagArgs, arg)
}
}
}
// Parse space and skill name
if len(nonFlagArgs) < 1 {
return nil, fmt.Errorf("space ID is required")
}
if len(nonFlagArgs) < 2 {
return nil, fmt.Errorf("skill name is required")
}
result.SpaceID = nonFlagArgs[0]
result.SkillName = nonFlagArgs[1]
return result, nil
}
// PrintHelp prints the help message
func (c *SkillUninstallCommand) PrintHelp() {
fmt.Println(`Usage: uninstall-skill <space> <skill-name>
Remove a skill from RAGFlow and delete its search index.
Arguments:
<space> Skills space ID (required)
<skill-name> Name of the skill to uninstall (required)
Options:
-h, --help Show this help message
Examples:
uninstall-skill my-space my-skill
uninstall-skill production document-analyzer
Note: 'delete-skill' command is deprecated. Use 'uninstall-skill' instead.`)
}

6
internal/cli/contextengine/types.go → internal/cli/filesystem/types.go
View File

@@ -14,11 +14,11 @@
// limitations under the License.
//
package contextengine
package filesystem
import "time"
// NodeType represents the type of a node in the context filesystem
// NodeType represents the type of a node in the virtual filesystem
type NodeType string
const (
@@ -52,7 +52,7 @@ const (
CommandCat CommandType = "cat"
)
// Command represents a context engine command
// Command represents a filesystem command
type Command struct {
Type CommandType `json:"type"`
Path string `json:"path"`

2
internal/cli/contextengine/utils.go → internal/cli/filesystem/utils.go
View File

@@ -14,7 +14,7 @@
// limitations under the License.
//
package contextengine
package filesystem
import (
"encoding/json"

44
internal/cli/http_client.go
View File

@@ -336,6 +336,50 @@ func (c *HTTPClient) RequestJSON(method, path string, useAPIBase bool, authKind
return resp.JSON()
}
// UploadMultipart uploads data using multipart/form-data
func (c *HTTPClient) UploadMultipart(path string, contentType string, body io.Reader) error {
url := c.BuildURL(path, true)
req, err := http.NewRequest("POST", url, body)
if err != nil {
return err
}
// Set headers
req.Header.Set("Content-Type", contentType)
if c.APIToken != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.APIToken))
} else if c.LoginToken != "" {
req.Header.Set("Authorization", c.LoginToken)
}
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return fmt.Errorf("upload failed: HTTP %d - %s", resp.StatusCode, string(respBody))
}
// Check response code
var result struct {
Code int `json:"code"`
Message string `json:"message"`
}
if err := json.Unmarshal(respBody, &result); err == nil && result.Code != 0 {
return fmt.Errorf("upload failed: %s", result.Message)
}
return nil
}
// RequestStream makes an HTTP request for SSE streaming and returns the response body reader
func (c *HTTPClient) RequestStream(method, path string, useAPIBase bool, authKind string, headers map[string]string, jsonBody map[string]interface{}) (io.ReadCloser, error) {
url := c.BuildURL(path, useAPIBase)

2
internal/cli/lexer.go
View File

@@ -108,7 +108,7 @@ func (l *Lexer) NextToken() Token {
tok.Type = TokenEOF
tok.Value = ""
default:
if isLetter(l.ch) {
if isLetter(l.ch) || l.ch == '_' {
ident := l.readIdentifier()
return l.lookupIdent(ident)
} else if isDigit(l.ch) {

174
internal/cli/parser.go
View File

@@ -57,9 +57,10 @@ func (p *Parser) Parse(adminCommand bool) (*Command, error) {
}
// Check for ContextEngine commands (ls, cat, search)
//if p.curToken.Type == TokenIdentifier && isCECommand(p.curToken.Value) {
// return p.parseCECommand()
//}
// Note: These are now handled in parseUserCommand to support both SQL-style and CE-style syntax
// if p.curToken.Type == TokenIdentifier && isCECommand(p.curToken.Value) {
// return p.parseCECommand()
// }
return p.parseCommand(adminCommand)
}
@@ -199,9 +200,9 @@ func (p *Parser) parseUserCommand() (*Command, error) {
case TokenCheck:
return p.parseCheckCommand()
case TokenLS:
return p.parseContextListCommand()
return p.parseCEListCommand()
case TokenCat:
return p.parseContextCatCommand()
return p.parseCECatCommand()
case TokenUse:
return p.parseUseCommand()
case TokenUpdate:
@@ -248,7 +249,7 @@ func isKeyword(tokenType int) bool {
return tokenType >= TokenLogin && tokenType <= TokenTag
}
// isCECommand checks if the given string is a ContextEngine command
// isCECommand checks if the given string is a Filesystem command
func isCECommand(s string) bool {
upper := strings.ToUpper(s)
switch upper {
@@ -304,6 +305,8 @@ func (p *Parser) parseCECommand() (*Command, error) {
switch cmdName {
case "LS", "LIST":
return p.parseCEListCommand()
case "CAT":
return p.parseCECatCommand()
case "SEARCH":
return p.parseCESearchCommand()
default:
@@ -327,8 +330,49 @@ func (p *Parser) parseCEListCommand() (*Command, error) {
if p.curToken.Type == TokenQuotedString {
path = strings.Trim(path, "\"'")
}
cmd.Params["path"] = path
p.nextToken()
// Handle path components separated by slashes (e.g., "skills/hub1")
for p.curToken.Type == TokenSlash {
p.nextToken() // consume slash
if p.curToken.Type == TokenIdentifier || p.curToken.Type == TokenDatasets ||
p.curToken.Type == TokenAgents || p.curToken.Type == TokenChats {
path = path + "/" + p.curToken.Value
p.nextToken()
} else if p.curToken.Type == TokenNumber {
// Handle version numbers like 1.0.0 (parsed as number . number . number)
// OR filenames starting with numbers like 3_list_compressors.pdf
numberPart := p.curToken.Value
p.nextToken()
// Continue reading .number parts (version number format)
if p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
versionPart := numberPart
for p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
p.nextToken() // consume .
if p.curToken.Type == TokenNumber {
versionPart = versionPart + "." + p.curToken.Value
p.nextToken()
} else {
break
}
}
path = path + "/" + versionPart
} else if p.curToken.Type == TokenIdentifier {
// Filename starting with number: 3_list_compressors.pdf
path = path + "/" + numberPart + p.curToken.Value
p.nextToken()
} else {
// Just a number
path = path + "/" + numberPart
}
} else {
// Trailing slash, just append it
path = path + "/"
break
}
}
cmd.Params["path"] = path
} else {
// Default to "datasets" root
cmd.Params["path"] = "datasets"
@@ -342,6 +386,76 @@ func (p *Parser) parseCEListCommand() (*Command, error) {
return cmd, nil
}
// parseCECatCommand parses the cat command
// Syntax: cat <path>
func (p *Parser) parseCECatCommand() (*Command, error) {
p.nextToken() // consume CAT
cmd := NewCommand("ce_cat")
if p.curToken.Type != TokenIdentifier && p.curToken.Type != TokenQuotedString {
return nil, fmt.Errorf("expected path after CAT")
}
path := p.curToken.Value
if p.curToken.Type == TokenQuotedString {
path = strings.Trim(path, "\"'")
}
p.nextToken()
// Handle path components separated by slashes (e.g., "skills/hub1/skill/README.md")
for p.curToken.Type == TokenSlash {
p.nextToken() // consume slash
if p.curToken.Type == TokenIdentifier || p.curToken.Type == TokenAgents ||
p.curToken.Type == TokenChats || p.curToken.Type == TokenDatasets {
path = path + "/" + p.curToken.Value
p.nextToken()
} else if p.curToken.Type == TokenNumber {
// Handle version numbers like 1.0.0 (parsed as number . number . number)
// OR filenames starting with numbers like 3_list_compressors.pdf
numberPart := p.curToken.Value
p.nextToken()
// Continue reading .number parts (version number format)
if p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
versionPart := numberPart
for p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
p.nextToken() // consume .
if p.curToken.Type == TokenNumber {
versionPart = versionPart + "." + p.curToken.Value
p.nextToken()
} else {
break
}
}
path = path + "/" + versionPart
} else if p.curToken.Type == TokenIdentifier {
// Filename starting with number: 3_list_compressors.pdf
path = path + "/" + numberPart + p.curToken.Value
p.nextToken()
} else {
// Just a number
path = path + "/" + numberPart
}
} else if p.curToken.Type == TokenQuotedString {
path = path + "/" + strings.Trim(p.curToken.Value, "\"'")
p.nextToken()
} else {
// Trailing slash, just append it
path = path + "/"
break
}
}
cmd.Params["path"] = path
// Optional semicolon
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseCESearchCommand parses the search command
// Syntax: search <query> or search <query> in <path>
func (p *Parser) parseCESearchCommand() (*Command, error) {
@@ -372,8 +486,52 @@ func (p *Parser) parseCESearchCommand() (*Command, error) {
if p.curToken.Type == TokenQuotedString {
path = strings.Trim(path, "\"'")
}
cmd.Params["path"] = path
p.nextToken()
// Handle path components separated by slashes (e.g., "skills/hub1")
for p.curToken.Type == TokenSlash {
p.nextToken() // consume slash
if p.curToken.Type == TokenIdentifier || p.curToken.Type == TokenAgents ||
p.curToken.Type == TokenChats || p.curToken.Type == TokenDatasets {
path = path + "/" + p.curToken.Value
p.nextToken()
} else if p.curToken.Type == TokenNumber {
// Handle version numbers like 1.0.0 (parsed as number . number . number)
// OR filenames starting with numbers like 3_list_compressors.pdf
numberPart := p.curToken.Value
p.nextToken()
// Continue reading .number parts (version number format)
if p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
versionPart := numberPart
for p.curToken.Type == TokenIllegal && p.curToken.Value == "." {
p.nextToken() // consume .
if p.curToken.Type == TokenNumber {
versionPart = versionPart + "." + p.curToken.Value
p.nextToken()
} else {
break
}
}
path = path + "/" + versionPart
} else if p.curToken.Type == TokenIdentifier {
// Filename starting with number: 3_list_compressors.pdf
path = path + "/" + numberPart + p.curToken.Value
p.nextToken()
} else {
// Just a number
path = path + "/" + numberPart
}
} else if p.curToken.Type == TokenQuotedString {
path = path + "/" + strings.Trim(p.curToken.Value, "\"'")
p.nextToken()
} else {
// Trailing slash, just append it
path = path + "/"
break
}
}
cmd.Params["path"] = path
} else {
cmd.Params["path"] = "."
}

23
internal/cli/response.go
View File

@@ -322,3 +322,26 @@ func (r *ContextSearchResponse) PrintOut() {
fmt.Printf("%d, %s\n", r.Code, r.Message)
}
}
// ContextCatResponse represents the response for cat command
type ContextCatResponse struct {
Code int `json:"code"`
Content string `json:"content"`
Message string `json:"message"`
Duration float64
OutputFormat OutputFormat
}
func (r *ContextCatResponse) Type() string { return "ce_cat" }
func (r *ContextCatResponse) TimeCost() float64 { return r.Duration }
func (r *ContextCatResponse) SetOutputFormat(format OutputFormat) { r.OutputFormat = format }
func (r *ContextCatResponse) PrintOut() {
if r.Code == 0 {
fmt.Println(r.Content)
} else {
fmt.Println("ERROR")
fmt.Printf("%d, %s\n", r.Code, r.Message)
}
}

2
internal/cli/types.go
View File

@@ -109,6 +109,7 @@ const (
TokenVector
TokenSize
TokenName // For ALTER PROVIDER <name> NAME <new_name>
TokenPool
TokenBalance
TokenInstance
TokenInstances
@@ -152,6 +153,7 @@ const (
TokenQuotedString
TokenInteger
TokenFloat
TokenNumber = TokenInteger // Alias for integer tokens in path parsing (e.g., version numbers like 1.0.0)
// Special
TokenSemicolon

36
internal/cli/user_command.go
View File

@@ -22,7 +22,7 @@ import (
"encoding/json"
"fmt"
"os"
ce "ragflow/internal/cli/contextengine"
ce "ragflow/internal/cli/filesystem"
"strings"
"time"
)
@@ -1818,6 +1818,36 @@ func (c *RAGFlowClient) AddCustomModel(cmd *Command) (ResponseIf, error) {
// Context related commands
// CECat handles the cat command - shows content using Context Engine
func (c *RAGFlowClient) CECat(cmd *Command) (ResponseIf, error) {
if c.HTTPClient.APIToken == "" && c.HTTPClient.LoginToken == "" {
return nil, fmt.Errorf("API token not set. Please login first")
}
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
path, ok := cmd.Params["path"].(string)
if !ok {
return nil, fmt.Errorf("fail to convert 'path' to string")
}
// Execute cat command through Filesystem Engine
ctx := context.Background()
content, err := c.ContextEngine.Cat(ctx, path)
if err != nil {
return nil, err
}
// Convert to response
var response ContextCatResponse
response.OutputFormat = c.OutputFormat
response.Code = 0
response.Content = string(content)
return &response, nil
}
// CEList handles the ls command - lists nodes using Context Engine
func (c *RAGFlowClient) CEList(cmd *Command) (ResponseIf, error) {
// Get path from command params, default to "datasets"
@@ -1838,7 +1868,7 @@ func (c *RAGFlowClient) CEList(cmd *Command) (ResponseIf, error) {
opts.Offset = offset
}
// Execute list command through Context Engine
// Execute list command through Filesystem Engine
ctx := context.Background()
result, err := c.ContextEngine.List(ctx, path, opts)
if err != nil {
@@ -1877,7 +1907,7 @@ func (c *RAGFlowClient) CESearch(cmd *Command) (ResponseIf, error) {
opts.Recursive = recursive
}
// Execute search command through Context Engine
// Execute search command through Filesystem Engine
ctx := context.Background()
result, err := c.ContextEngine.Search(ctx, path, opts)
if err != nil {

2
internal/cli/user_parser.go
View File

@@ -2164,7 +2164,7 @@ func (p *Parser) parseSearchCommand() (*Command, error) {
return cmd, nil
}
cmd := NewCommand("context_search")
cmd := NewCommand("ce_search")
cmd.Params["query"] = question