Refine handling of POST /api/v1/datasets/search in GO (#15583)

### What problem does this PR solve?

Refine handling of POST /api/v1/datasets/search in GO

### Type of change

- [x] Refactoring

internal/cli/README.md

@@ -103,6 +103,9 @@ search "machine learning"                    # Search all datasets (JSON output)
 search "neural networks" datasets/kb1        # Search in kb1
 search "AI" datasets/kb1  --output plain     # Plain text output
 search "RAG" -n 20                           # Return 20 results
+SEARCH 'machine learning' ON DATASETS 'kb1' 'kb2'
+SEARCH 'AI' ON DATASETS 'kb1' WITH top_k 1024 similarity_threshold 0.0 vector_similarity_weight 0.3 keyword true
+SEARCH 'AI' ON DATASETS 'kb1' WITH cross_languages ['Chinese']
 ```
 
 #### `cat <path>` - Display content

internal/cli/cli.go

@@ -421,13 +421,13 @@ const historyFileName = ".ragflow_cli_history"
 
 // CLI represents the command line interface
 type CLI struct {
-	client         *RAGFlowClient
-	contextEngine  *filesystem.Engine
-	prompt         string
-	running        bool
-	line           *liner.State
-	args           *ConnectionArgs
-	outputFormat   OutputFormat // Output format
+	client        *RAGFlowClient
+	contextEngine *filesystem.Engine
+	prompt        string
+	running       bool
+	line          *liner.State
+	args          *ConnectionArgs
+	outputFormat  OutputFormat // Output format
 }
 
 // NewCLI creates a new CLI instance
@@ -1072,13 +1072,13 @@ func (c *CLI) printSkillSearchResults(result *filesystem.Result, format OutputFo
 
 	// Skill search result structure
 	type skillSearchResult struct {
-		SkillID     string   `json:"skill_id"`
-		Name        string   `json:"name"`
-		Description string   `json:"description"`
-		Tags        string   `json:"tags"`
-		Score       float64  `json:"score"`
-		BM25Score   float64  `json:"bm25_score"`
-		VectorScore float64  `json:"vector_score"`
+		SkillID     string  `json:"skill_id"`
+		Name        string  `json:"name"`
+		Description string  `json:"description"`
+		Tags        string  `json:"tags"`
+		Score       float64 `json:"score"`
+		BM25Score   float64 `json:"bm25_score"`
+		VectorScore float64 `json:"vector_score"`
 	}
 
 	results := make([]skillSearchResult, 0, len(result.Nodes))
@@ -1457,6 +1457,41 @@ Examples:
   search "AI" datasets/kb1                          # Search in kb1
   search "RAG" skills/space1 -n 20                    # Search skills in hub1, return 20 results
   search "data processing" skills                   # Search skills (default space)
+
+Datasets syntax (full filter set):
+  search 'query' on datasets 'kb_names' [with <option> <value> ...] [;]
+
+  'kb_names' is a single quoted string. Pass one name for a single
+  dataset, or a comma-separated list (no spaces) to search across
+  multiple datasets in one call:
+    'kb1'                  # one dataset
+    'kb1,kb2'              # two datasets
+    'kb1,kb2,kb3'          # three datasets
+
+  When 'on datasets' is given, the search runs against the named
+  dataset(s) and accepts the full WITH-option set below.
+
+  WITH options (space-separated, not comma-separated):
+    top_k                   <int>     Number of results (default 5)
+    page_size               <int>     Page size for pagination
+    page                    <int>     Page number (1-based)
+    similarity_threshold    <float>   Minimum similarity score (0.0-1.0)
+    vector_similarity_weight <float>  Weight given to vector vs text score
+    keyword                 true|false  Enable keyword extraction via LLM
+    use_kg                  true|false  Enable knowledge-graph augmentation
+    rerank_id               'id'      Rerank model to apply
+    tenant_rerank_id        'id'      Tenant-scoped rerank model
+    search_id               'id'      Idempotency / search-session id
+    meta_data_filter        '<json>'  Metadata filter (must be valid JSON)
+    cross_languages         ['a','b'] Source languages to translate from
+    doc_ids                 ['d1',...] Restrict to specific document ids
+
+  Examples:
+    search 'AI' on datasets 'kb_chinese' with top_k 10;
+    search 'AI' on datasets 'kb1' 'kb2' with top_k 20 similarity_threshold 0.3 cross_languages ['Chinese']
+        doc_ids ['d1', 'd2'];
+    search 'manual' on datasets 'kb1' with
+        meta_data_filter '{"method":"manual","conditions":[{"key":"author","op":"eq","value":"Luo"}]}';
 `
 	fmt.Println(help)
 }

internal/cli/client.go

@@ -223,6 +223,9 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.ListDatasetDocumentUserCommand(cmd)
 	case "search_on_datasets":
 		return c.SearchOnDatasets(cmd)
+	case "search_help":
+		printSearchHelp()
+		return nil, nil
 	case "create_token":
 		return c.CreateToken(cmd)
 	case "list_tokens":
@@ -335,8 +338,8 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.RmTags(cmd)
 	case "remove_chunks":
 		return c.RemoveChunks(cmd)
-	case "list_metadata":
-		return c.ListMetadata(cmd)
+	case "get_metadata":
+		return c.GetMetadata(cmd)
 	case "parse_documents_user_command":
 		return c.ParseDocumentsUserCommand(cmd)
 	// ContextEngine commands

internal/cli/lexer.go

@@ -94,6 +94,12 @@ func (l *Lexer) NextToken() Token {
 	case '-':
 		tok = newToken(TokenDash, l.ch)
 		l.readChar()
+	case '[':
+		tok = newToken(TokenLBracket, l.ch)
+		l.readChar()
+	case ']':
+		tok = newToken(TokenRBracket, l.ch)
+		l.readChar()
 	case '\'':
 		tok.Type = TokenQuotedString
 		tok.Value = l.readQuotedString('\'')

internal/cli/parser.go

@@ -308,7 +308,10 @@ func (p *Parser) parseNumber() (int, error) {
 }
 
 func (p *Parser) parseFloat() (float64, error) {
-	if p.curToken.Type != TokenInteger {
+	// Accept either TokenInteger or TokenFloat so that literals like
+	// `0.3` (which the lexer tags as TokenFloat) and `10` (TokenInteger)
+	// both parse cleanly.
+	if p.curToken.Type != TokenInteger && p.curToken.Type != TokenFloat {
 		return math.NaN(), fmt.Errorf("expected number, got %s", p.curToken.Value)
 	}
 	result, err := strconv.ParseFloat(p.curToken.Value, 64)
@@ -319,8 +322,66 @@ func (p *Parser) parseFloat() (float64, error) {
 	return result, nil
 }
 
+// parseQuotedStringList consumes a bracket-delimited list of quoted strings:
+//   [ 'a', 'b', 'c' ]
+// Empty list [] is allowed. The cursor must be positioned on '[' when called;
+// on return, the cursor is positioned just past the closing ']'.
+func (p *Parser) parseQuotedStringList() ([]string, error) {
+	if p.curToken.Type != TokenLBracket {
+		return nil, fmt.Errorf("expected '[', got %s", p.curToken.Value)
+	}
+	p.nextToken() // skip '['
+
+	// Always return a non-nil slice so callers (and json.Marshal) see []
+	// instead of null for the empty-list case.
+	list := make([]string, 0)
+	// Allow empty list []
+	if p.curToken.Type == TokenRBracket {
+		p.nextToken() // skip ']'
+		return list, nil
+	}
+
+	for {
+		s, err := p.parseQuotedString()
+		if err != nil {
+			return nil, fmt.Errorf("expected quoted string in list: %w", err)
+		}
+		list = append(list, s)
+		p.nextToken() // step past the closing quote
+
+		if p.curToken.Type == TokenComma {
+			p.nextToken() // step past ','
+			continue
+		}
+		if p.curToken.Type == TokenRBracket {
+			p.nextToken() // step past ']'
+			return list, nil
+		}
+		return nil, fmt.Errorf("expected ',' or ']' in list, got %s", p.curToken.Value)
+	}
+}
+
 func tokenTypeToString(t int) string {
-	// Simplified for error messages
+	switch t {
+	case TokenEOF:
+		return "end of input"
+	case TokenIdentifier:
+		return "identifier"
+	case TokenInteger:
+		return "integer"
+	case TokenFloat:
+		return "float"
+	case TokenQuotedString:
+		return "quoted string"
+	case TokenLBracket:
+		return "'['"
+	case TokenRBracket:
+		return "']'"
+	case TokenComma:
+		return "','"
+	case TokenSemicolon:
+		return "';'"
+	}
 	return fmt.Sprintf("token(%d)", t)
 }
 

internal/cli/types.go

@@ -187,6 +187,8 @@ const (
 	TokenEOF
 	TokenDash
 	TokenIllegal
+	TokenLBracket // '['
+	TokenRBracket // ']'
 )
 
 // Token represents a lexical token

internal/cli/user_command.go

@@ -498,8 +498,8 @@ func (c *RAGFlowClient) getDatasetID(datasetName string) (string, error) {
 	return "", fmt.Errorf("dataset '%s' not found", datasetName)
 }
 
-// ListMetadata lists metadata for datasets
-func (c *RAGFlowClient) ListMetadata(cmd *Command) (ResponseIf, error) {
+// GetMetadata gets metadata for one or more datasets
+func (c *RAGFlowClient) GetMetadata(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
@@ -608,6 +608,68 @@ func (c *RAGFlowClient) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
 		"vector_similarity_weight": 0.3,
 	}
 
+	// Add optional parameters from command
+	if val, ok := cmd.Params["top_k"]; ok {
+		payload["top_k"] = val
+	}
+	if val, ok := cmd.Params["similarity_threshold"]; ok {
+		payload["similarity_threshold"] = val
+	}
+	if val, ok := cmd.Params["vector_similarity_weight"]; ok {
+		payload["vector_similarity_weight"] = val
+	}
+	if val, ok := cmd.Params["keyword"]; ok {
+		payload["keyword"] = val
+	}
+	if val, ok := cmd.Params["use_kg"]; ok {
+		payload["use_kg"] = val
+	}
+	if val, ok := cmd.Params["rerank_id"]; ok {
+		payload["rerank_id"] = val
+	}
+	if val, ok := cmd.Params["tenant_rerank_id"]; ok {
+		payload["tenant_rerank_id"] = val
+	}
+	if val, ok := cmd.Params["page_size"]; ok {
+		payload["page_size"] = val
+	}
+	if val, ok := cmd.Params["page"]; ok {
+		payload["page"] = val
+	}
+	if val, ok := cmd.Params["search_id"]; ok {
+		if s, ok := val.(string); ok {
+			payload["search_id"] = s
+		}
+	}
+	if val, ok := cmd.Params["cross_languages"]; ok {
+		if list, ok := val.([]string); ok {
+			payload["cross_languages"] = list
+		}
+	}
+	if val, ok := cmd.Params["doc_ids"]; ok {
+		if list, ok := val.([]string); ok {
+			payload["doc_ids"] = list
+		}
+	}
+	if val, ok := cmd.Params["meta_data_filter"]; ok {
+		// Accept either a raw JSON string from the CLI or a pre-decoded
+		// map[string]interface{} (future-proofing for callers that
+		// construct the command programmatically). The string form is
+		// the public CLI surface; the map form is for unit tests.
+		switch v := val.(type) {
+		case string:
+			var decoded map[string]interface{}
+			if err := json.Unmarshal([]byte(v), &decoded); err != nil {
+				return nil, fmt.Errorf("invalid meta_data_filter JSON: %w", err)
+			}
+			payload["meta_data_filter"] = decoded
+		case map[string]interface{}:
+			payload["meta_data_filter"] = v
+		default:
+			return nil, fmt.Errorf("meta_data_filter must be JSON string or object")
+		}
+	}
+
 	if iterations > 1 {
 		// Benchmark mode - return raw result for benchmark stats
 		return c.HTTPClient.RequestWithIterations("POST", "/datasets/search", "web", nil, payload, iterations)

internal/cli/user_parser.go

@@ -6,6 +6,13 @@ import (
 	"strings"
 )
 
+func tokenTypeDescription(t int, tok Token) string {
+	if tok.Type == t && tok.Value != "" {
+		return fmt.Sprintf("%s %q", tokenTypeToString(t), tok.Value)
+	}
+	return tokenTypeToString(t)
+}
+
 // Command parsers
 func (p *Parser) parseLogout() (*Command, error) {
 	cmd := NewCommand("logout")
@@ -138,8 +145,6 @@ func (p *Parser) parseListCommand() (*Command, error) {
 		return p.parseListDatasets()
 	case TokenDocuments:
 		return p.parseListDatasetDocuments()
-	case TokenMetadata:
-		return p.parseListMetadata()
 	case TokenAgents:
 		return p.parseListAgents()
 	case TokenTokens:
@@ -210,7 +215,7 @@ func (p *Parser) parseListDatasetDocuments() (*Command, error) {
 	return cmd, nil
 }
 
-func (p *Parser) parseListMetadata() (*Command, error) {
+func (p *Parser) parseGetMetadata() (*Command, error) {
 	p.nextToken() // consume METADATA
 
 	if p.curToken.Type != TokenOf {
@@ -233,6 +238,10 @@ func (p *Parser) parseListMetadata() (*Command, error) {
 		datasetNames = append(datasetNames, name)
 
 		p.nextToken()
+
+		if p.curToken.Type == TokenComma {
+			return nil, fmt.Errorf("syntax error: dataset names must be space-separated, not comma-separated (got %q after %q)", "'", name)
+		}
 		// Stop at semicolon or non-quoted (dataset name must be quoted)
 		if p.curToken.Type == TokenSemicolon {
 			break
@@ -243,7 +252,7 @@ func (p *Parser) parseListMetadata() (*Command, error) {
 		}
 	}
 
-	cmd := NewCommand("list_metadata")
+	cmd := NewCommand("get_metadata")
 	cmd.Params["dataset_names"] = datasetNames
 
 	// Semicolon is optional
@@ -2379,6 +2388,27 @@ func (p *Parser) parseInsertMetadataFromFile() (*Command, error) {
 func (p *Parser) parseSearchCommand() (*Command, error) {
 	p.nextToken() // consume SEARCH
 
+	// Handle help flag: -h / --help. The lexer tokenizes each leading
+	// `-` as a separate `TokenDash` and then the rest of the flag name
+	// (e.g. "help") as a `TokenIdentifier`. We collect any leading
+	// dashes before checking the identifier value. Short-circuit with
+	// a dedicated command type so the dispatcher can print the search
+	// usage instead of erroring out on the missing question.
+	if p.curToken.Type == TokenDash {
+		dashCount := 0
+		for p.curToken.Type == TokenDash {
+			dashCount++
+			p.nextToken()
+		}
+		if dashCount > 0 && p.curToken.Type == TokenIdentifier {
+			switch strings.ToLower(p.curToken.Value) {
+			case "h", "help":
+				return NewCommand("search_help"), nil
+			}
+		}
+		return nil, fmt.Errorf("expected quoted string or identifier")
+	}
+
 	var err error
 	var question string
 	if p.curToken.Type == TokenQuotedString {
@@ -2415,7 +2445,119 @@ func (p *Parser) parseSearchCommand() (*Command, error) {
 		cmd.Params["datasets"] = datasets
 
 		p.nextToken()
-		// Semicolon is optional for UNSET TOKEN
+
+		// Parse optional WITH clause for additional parameters
+		if p.curToken.Type == TokenWith || (p.curToken.Type == TokenIdentifier && strings.ToLower(p.curToken.Value) == "with") {
+			if p.curToken.Type == TokenWith {
+				p.nextToken()
+			} else {
+				p.nextToken() // skip "with" identifier
+			}
+
+			for p.curToken.Type != TokenEOF && p.curToken.Type != TokenSemicolon {
+				if p.curToken.Type == TokenComma {
+					return nil, fmt.Errorf("syntax error: WITH options must be space-separated, not comma-separated")
+				}
+				// Parse parameter name
+				if p.curToken.Type != TokenIdentifier {
+					break
+				}
+				paramName := strings.ToLower(p.curToken.Value)
+				p.nextToken()
+
+				// Parse parameter value
+				var paramValue interface{}
+				valueToken := p.curToken.Type
+				var valueErr error
+				switch p.curToken.Type {
+				case TokenInteger:
+					paramValue, valueErr = p.parseNumber()
+					if valueErr != nil {
+						return nil, valueErr
+					}
+					p.nextToken() // step past the integer
+				case TokenFloat:
+					paramValue, valueErr = p.parseFloat()
+					if valueErr != nil {
+						return nil, valueErr
+					}
+					p.nextToken() // step past the float
+				case TokenQuotedString:
+					paramValue, valueErr = p.parseQuotedString()
+					if valueErr != nil {
+						return nil, valueErr
+					}
+					p.nextToken() // step past the closing quote
+				case TokenIdentifier:
+					// Bare identifiers are only meaningful for the
+					// boolean keys (keyword / use_kg = true|false);
+					// everything else rejects them.
+					paramValue = p.curToken.Value
+					p.nextToken()
+				case TokenLBracket:
+					// List value: parsed inside the switch below by
+					// cross_languages / doc_ids. No value is captured here.
+					paramValue = nil
+				default:
+					// EOF, ';', or any other non-value token: the option
+					// is missing a value, which is a hard error rather
+					// than a silent drop.
+					return nil, fmt.Errorf("WITH option %q is missing a value", paramName)
+				}
+
+				switch paramName {
+				case "top_k", "page_size", "page":
+					if valueToken != TokenInteger {
+						return nil, fmt.Errorf("WITH option %q must be an integer, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
+					}
+					cmd.Params[paramName] = paramValue
+				case "similarity_threshold", "vector_similarity_weight":
+					switch n := paramValue.(type) {
+					case int:
+						cmd.Params[paramName] = float64(n)
+					case float64:
+						cmd.Params[paramName] = n
+					default:
+						return nil, fmt.Errorf("WITH option %q must be a number, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
+					}
+				case "keyword", "use_kg":
+					s, ok := paramValue.(string)
+					if !ok {
+						return nil, fmt.Errorf("WITH option %q must be true or false, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
+					}
+					switch strings.ToLower(s) {
+					case "true":
+						cmd.Params[paramName] = true
+					case "false":
+						cmd.Params[paramName] = false
+					default:
+						return nil, fmt.Errorf("WITH option %q must be true or false, got %q", paramName, s)
+					}
+				case "rerank_id", "tenant_rerank_id", "search_id", "meta_data_filter":
+					if valueToken != TokenQuotedString {
+						return nil, fmt.Errorf("WITH option %q must be a quoted string, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
+					}
+					// meta_data_filter JSON string is decoded into a map in
+					// the SearchOnDatasets handler; parser stores the raw
+					// string so the handler can surface a clean error on
+					// invalid JSON.
+					cmd.Params[paramName] = paramValue
+				case "cross_languages", "doc_ids":
+					if p.curToken.Type != TokenLBracket {
+						return nil, fmt.Errorf("WITH option %q must be a list, e.g. %q ['a', 'b']", paramName, paramName)
+					}
+					list, err := p.parseQuotedStringList()
+					if err != nil {
+						return nil, err
+					}
+					cmd.Params[paramName] = list
+				default:
+					return nil, fmt.Errorf("unknown WITH option %q", paramName)
+				}
+			}
+		}
+
+		// Semicolon is optional
 		if p.curToken.Type == TokenSemicolon {
 			p.nextToken()
 		}
@@ -3599,13 +3741,16 @@ func (p *Parser) parseUnsetCommand() (*Command, error) {
 	return NewCommand("unset_token"), nil
 }
 
-// parseGetCommand parses: GET CHUNK 'chunk_id'
+// parseGetCommand parses: GET CHUNK or GET METADATA
 func (p *Parser) parseGetCommand() (*Command, error) {
 	p.nextToken() // consume GET
 
 	if p.curToken.Type == TokenChunk {
 		return p.parseGetChunk()
 	}
+	if p.curToken.Type == TokenMetadata {
+		return p.parseGetMetadata()
+	}
 
 	return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value)
 }