Update chunk/metadata cli (#15055)

### What problem does this PR solve?

Update chunk/metadata cli

### Type of change

- [ ] Refactoring

internal/cli/benchmark.go

@@ -237,12 +237,12 @@ func (c *RAGFlowClient) executeBenchmarkSilent(cmd *Command, iterations int) []*
 			question, _ := cmd.Params["question"].(string)
 			datasetIDs, _ := cmd.Params["dataset_ids"].([]string)
 			payload := map[string]interface{}{
-				"kb_id":                    datasetIDs,
+				"dataset_ids":              datasetIDs,
 				"question":                 question,
 				"similarity_threshold":     0.2,
 				"vector_similarity_weight": 0.3,
 			}
-			resp, err = c.HTTPClient.Request("POST", "/chunk/retrieval_test", "web", nil, payload)
+			resp, err = c.HTTPClient.Request("POST", "/datasets/search", "web", nil, payload)
 		default:
 			// For other commands, we would need to add specific handling
 			// For now, mark as failed

internal/cli/client.go

@@ -314,12 +314,16 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.InsertMetadataFromFile(cmd)
 	case "update_chunk":
 		return c.UpdateChunk(cmd)
+	case "get_chunk":
+		return c.GetChunk(cmd)
 	case "set_meta":
 		return c.SetMeta(cmd)
 	case "rm_tags":
 		return c.RmTags(cmd)
 	case "remove_chunks":
 		return c.RemoveChunks(cmd)
+	case "list_metadata":
+		return c.ListMetadata(cmd)
 	// ContextEngine commands
 	case "ce_ls":
 		return c.CEList(cmd)

internal/cli/filesystem/dataset.go

@@ -305,8 +305,8 @@ func (p *DatasetProvider) searchWithRetrieval(ctx stdctx.Context, opts *SearchOp
 
 	// Build retrieval request
 	payload := map[string]interface{}{
-		"kb_id":    kbIDs,
-		"question": opts.Query,
+		"dataset_ids": kbIDs,
+		"question":    opts.Query,
 	}
 
 	// Set top_k (default to 10 if not specified)
@@ -323,8 +323,8 @@ func (p *DatasetProvider) searchWithRetrieval(ctx stdctx.Context, opts *SearchOp
 	}
 	payload["similarity_threshold"] = threshold
 
-	// Call retrieval API (useAPIBase=false because the route is /v1/chunk/retrieval_test, not /api/v1/...)
-	resp, err := p.httpClient.Request("POST", "/chunk/retrieval_test", "auto", nil, payload)
+	// Call retrieval API
+	resp, err := p.httpClient.Request("POST", "/datasets/search", "auto", nil, payload)
 	if err != nil {
 		return nil, fmt.Errorf("retrieval request failed: %w", err)
 	}
@@ -589,8 +589,8 @@ func (p *DatasetProvider) searchDocuments(ctx stdctx.Context, datasetName string
 
 	// Build retrieval request for specific dataset
 	payload := map[string]interface{}{
-		"kb_id":    []string{kbID},
-		"question": opts.Query,
+		"dataset_ids": []string{kbID},
+		"question":    opts.Query,
 	}
 
 	// Set top_k (default to 10 if not specified)
@@ -607,8 +607,8 @@ func (p *DatasetProvider) searchDocuments(ctx stdctx.Context, datasetName string
 	}
 	payload["similarity_threshold"] = threshold
 
-	// Call retrieval API (useAPIBase=false because the route is /v1/chunk/retrieval_test, not /api/v1/...)
-	resp, err := p.httpClient.Request("POST", "/chunk/retrieval_test", "auto", nil, payload)
+	// Call retrieval API
+	resp, err := p.httpClient.Request("POST", "/datasets/search", "auto", nil, payload)
 	if err != nil {
 		return nil, fmt.Errorf("retrieval request failed: %w", err)
 	}

internal/cli/lexer.go

@@ -349,6 +349,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenParser, Value: ident}
 	case "PIPELINE":
 		return Token{Type: TokenPipeline, Value: ident}
+	case "GET":
+		return Token{Type: TokenGet, Value: ident}
 	case "SEARCH":
 		return Token{Type: TokenSearch, Value: ident}
 	case "CURRENT":

internal/cli/parser.go

@@ -219,6 +219,8 @@ func (p *Parser) parseUserCommand() (*Command, error) {
 		return p.parseUpdateCommand()
 	case TokenRemove:
 		return p.parseRemoveCommand()
+	case TokenGet:
+		return p.parseGetCommand()
 
 	default:
 		return nil, fmt.Errorf("unknown command: %s", p.curToken.Value)

internal/cli/response.go

@@ -16,7 +16,10 @@
 
 package cli
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 type ResponseIf interface {
 	Type() string
@@ -121,6 +124,104 @@ func (r *ListDocumentsResponse) PrintOut() {
 	}
 }
 
+type ChunkResponse struct {
+	Code         int                    `json:"code"`
+	Data         map[string]interface{} `json:"data"`
+	Message      string                 `json:"message"`
+	Duration     float64
+	OutputFormat OutputFormat
+}
+
+func (r *ChunkResponse) Type() string {
+	return "chunk"
+}
+
+func (r *ChunkResponse) TimeCost() float64 {
+	return r.Duration
+}
+
+func (r *ChunkResponse) SetOutputFormat(format OutputFormat) {
+	r.OutputFormat = format
+}
+
+func (r *ChunkResponse) PrintOut() {
+	if r.Code == 0 {
+		for k, v := range r.Data {
+			fmt.Printf("%s: %v\n", k, v)
+		}
+	} else {
+		fmt.Println("ERROR")
+		fmt.Printf("%d, %s\n", r.Code, r.Message)
+	}
+}
+
+type MetadataResponse struct {
+	Code         int                     `json:"code"`
+	Data         map[string]interface{}  `json:"data"`
+	Message      string                  `json:"message"`
+	Duration     float64
+	OutputFormat OutputFormat
+}
+
+func (r *MetadataResponse) Type() string {
+	return "metadata"
+}
+
+func (r *MetadataResponse) TimeCost() float64 {
+	return r.Duration
+}
+
+func (r *MetadataResponse) SetOutputFormat(format OutputFormat) {
+	r.OutputFormat = format
+}
+
+func (r *MetadataResponse) PrintOut() {
+	if r.Code == 0 {
+		// Data is map[field]map[value][]doc_id - print flattened metadata
+		if r.Data != nil {
+			printFlattenedMetadata(r.Data, r.OutputFormat)
+		}
+	} else {
+		fmt.Println("ERROR")
+		fmt.Printf("%d, %s\n", r.Code, r.Message)
+	}
+}
+
+func printFlattenedMetadata(data map[string]interface{}, format OutputFormat) {
+	// Convert flattened metadata to table format
+	// {field: {value: [doc_ids]}} -> [{field, value, document_ids}, ...]
+	tableData := make([]map[string]interface{}, 0)
+	for field, values := range data {
+		valueMap, ok := values.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		for value, docIDs := range valueMap {
+			var docIDStr string
+			switch v := docIDs.(type) {
+			case []string:
+				docIDStr = strings.Join(v, ", ")
+			case []interface{}:
+				docStrs := make([]string, 0, len(v))
+				for _, d := range v {
+					if s, ok := d.(string); ok {
+						docStrs = append(docStrs, s)
+					}
+				}
+				docIDStr = strings.Join(docStrs, ", ")
+			default:
+				docIDStr = fmt.Sprintf("%v", docIDs)
+			}
+			tableData = append(tableData, map[string]interface{}{
+				"field":         field,
+				"value":         value,
+				"document_ids": docIDStr,
+			})
+		}
+	}
+	PrintTableSimpleByFormat(tableData, format)
+}
+
 type SimpleResponse struct {
 	Code         int    `json:"code"`
 	Message      string `json:"message"`

internal/cli/types.go

@@ -145,6 +145,7 @@ const (
 	TokenFile
 	TokenMetadata
 	TokenTable
+	TokenGet
 	TokenUpdate
 	TokenRemove
 	TokenChunk
@@ -172,6 +173,7 @@ const (
 	TokenNumber = TokenInteger // Alias for integer tokens in path parsing (e.g., version numbers like 1.0.0)
 
 	// Special
+	_ = iota
 	TokenSemicolon
 	TokenComma
 	TokenSlash

internal/cli/user_command.go

@@ -498,6 +498,52 @@ func (c *RAGFlowClient) getDatasetID(datasetName string) (string, error) {
 	return "", fmt.Errorf("dataset '%s' not found", datasetName)
 }
 
+// ListMetadata lists metadata for datasets
+func (c *RAGFlowClient) ListMetadata(cmd *Command) (ResponseIf, error) {
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	datasetNames, ok := cmd.Params["dataset_names"].([]string)
+	if !ok || len(datasetNames) == 0 {
+		return nil, fmt.Errorf("dataset_names not provided")
+	}
+
+	// Convert dataset names to IDs
+	datasetIDs := make([]string, 0, len(datasetNames))
+	for _, name := range datasetNames {
+		id, err := c.getDatasetID(name)
+		if err != nil {
+			return nil, err
+		}
+		datasetIDs = append(datasetIDs, id)
+	}
+
+	// Build comma-separated dataset_ids for query param
+	datasetIDsStr := strings.Join(datasetIDs, ",")
+
+	resp, err := c.HTTPClient.Request("GET", "/datasets/metadata/flattened?dataset_ids="+datasetIDsStr, "web", nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list metadata: %w", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to list metadata: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+
+	var result MetadataResponse
+	if err = json.Unmarshal(resp.Body, &result); err != nil {
+		return nil, fmt.Errorf("list metadata failed: invalid JSON (%w)", err)
+	}
+
+	if result.Code != 0 {
+		return nil, fmt.Errorf("%s", result.Message)
+	}
+	result.Duration = resp.Duration
+
+	return &result, nil
+}
+
 // formatEmptyArray converts empty arrays to "[]" string
 func formatEmptyArray(v interface{}) string {
 	if v == nil {
@@ -2971,6 +3017,54 @@ func (c *RAGFlowClient) UpdateChunk(cmd *Command) (ResponseIf, error) {
 	return &result, nil
 }
 
+// GetChunk retrieves a chunk by ID
+func (c *RAGFlowClient) GetChunk(cmd *Command) (ResponseIf, error) {
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	chunkID, ok := cmd.Params["chunk_id"].(string)
+	if !ok {
+		return nil, fmt.Errorf("chunk_id not provided")
+	}
+
+	datasetName, ok := cmd.Params["dataset_name"].(string)
+	if !ok {
+		return nil, fmt.Errorf("dataset_name not provided")
+	}
+
+	datasetID, err := c.getDatasetID(datasetName)
+	if err != nil {
+		return nil, err
+	}
+
+	docID, ok := cmd.Params["doc_id"].(string)
+	if !ok {
+		return nil, fmt.Errorf("doc_id not provided")
+	}
+
+	resp, err := c.HTTPClient.Request("GET", fmt.Sprintf("/datasets/%s/documents/%s/chunks/%s", datasetID, docID, chunkID), "web", nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get chunk: %w", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to get chunk: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+
+	var result ChunkResponse
+	if err = json.Unmarshal(resp.Body, &result); err != nil {
+		return nil, fmt.Errorf("get chunk failed: invalid JSON (%w)", err)
+	}
+
+	if result.Code != 0 {
+		return nil, fmt.Errorf("%s", result.Message)
+	}
+	result.Duration = resp.Duration
+
+	return &result, nil
+}
+
 // SetMeta sets metadata for a document
 func (c *RAGFlowClient) SetMeta(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
@@ -3047,7 +3141,7 @@ func (c *RAGFlowClient) RmTags(cmd *Command) (ResponseIf, error) {
 		"tags": tags,
 	}
 
-	resp, err := c.HTTPClient.Request("POST", "/kb/"+kbID+"/rm_tags", "web", nil, payload)
+	resp, err := c.HTTPClient.Request("DELETE", "/datasets/"+kbID+"/tags", "web", nil, payload)
 	if err != nil {
 		return nil, fmt.Errorf("failed to remove tags: %w", err)
 	}

internal/cli/user_parser.go

@@ -138,6 +138,8 @@ func (p *Parser) parseListCommand() (*Command, error) {
 		return p.parseListDatasets()
 	case TokenDocuments:
 		return p.parseListDatasetDocuments()
+	case TokenMetadata:
+		return p.parseListMetadata()
 	case TokenAgents:
 		return p.parseListAgents()
 	case TokenTokens:
@@ -208,6 +210,50 @@ func (p *Parser) parseListDatasetDocuments() (*Command, error) {
 	return cmd, nil
 }
 
+func (p *Parser) parseListMetadata() (*Command, error) {
+	p.nextToken() // consume METADATA
+
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after METADATA")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after OF")
+	}
+	p.nextToken()
+
+	// Parse dataset names (space-separated)
+	var datasetNames []string
+	for {
+		name, err := p.parseQuotedString()
+		if err != nil {
+			return nil, fmt.Errorf("expected dataset name: %w", err)
+		}
+		datasetNames = append(datasetNames, name)
+
+		p.nextToken()
+		// Stop at semicolon or non-quoted (dataset name must be quoted)
+		if p.curToken.Type == TokenSemicolon {
+			break
+		}
+		// If next token is not a quoted string, stop parsing dataset names
+		if p.curToken.Type != TokenQuotedString {
+			break
+		}
+	}
+
+	cmd := NewCommand("list_metadata")
+	cmd.Params["dataset_names"] = datasetNames
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
 func (p *Parser) parseListAgents() (*Command, error) {
 	p.nextToken() // consume AGENTS
 
@@ -3234,6 +3280,8 @@ func (p *Parser) parseUserStatement() (*Command, error) {
 		return p.parseInsertCommand()
 	case TokenSearch:
 		return p.parseSearchCommand()
+	case TokenGet:
+		return p.parseGetCommand()
 	case TokenUpdate:
 		return p.parseUpdateCommand()
 	case TokenRemove:
@@ -3327,6 +3375,70 @@ func (p *Parser) parseUnsetCommand() (*Command, error) {
 	return NewCommand("unset_token"), nil
 }
 
+// parseGetCommand parses: GET CHUNK 'chunk_id'
+func (p *Parser) parseGetCommand() (*Command, error) {
+	p.nextToken() // consume GET
+
+	if p.curToken.Type == TokenChunk {
+		return p.parseGetChunk()
+	}
+
+	return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value)
+}
+
+// parseGetChunk parses: GET CHUNK 'chunk_id' OF DATASET 'dataset_name' DOCUMENT 'doc_id'
+func (p *Parser) parseGetChunk() (*Command, error) {
+	p.nextToken() // consume CHUNK
+
+	// Parse chunk_id
+	chunkID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected chunk_id: %w", err)
+	}
+
+	cmd := NewCommand("get_chunk")
+	cmd.Params["chunk_id"] = chunkID
+
+	p.nextToken()
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after chunk_id")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after OF")
+	}
+	p.nextToken()
+
+	// Parse dataset_name
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset_name: %w", err)
+	}
+	cmd.Params["dataset_name"] = datasetName
+
+	p.nextToken()
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after dataset_name")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
 // Internal
 // parseUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
 func (p *Parser) parseUpdateCommand() (*Command, error) {
@@ -3512,7 +3624,7 @@ func (p *Parser) parseRemoveChunk() (*Command, error) {
 	} else {
 		// curToken is TokenChunks, consume it first
 		p.nextToken()
-		// Multiple chunks: REMOVE CHUNKS 'id1', 'id2' FROM DOCUMENT 'doc_id'
+		// Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' (space-separated)
 		// Parse first chunk ID
 		chunkID, err := p.parseQuotedString()
 		if err != nil {
@@ -3520,19 +3632,18 @@ func (p *Parser) parseRemoveChunk() (*Command, error) {
 		}
 		chunkIDs := []string{chunkID}
 
-		// Parse additional chunk IDs separated by commas
+		// Parse additional chunk IDs separated by spaces (each quoted)
 		for {
 			p.nextToken()
-			if p.curToken.Type == TokenComma {
-				p.nextToken()
-				chunkID, err := p.parseQuotedString()
-				if err != nil {
-					return nil, fmt.Errorf("expected chunk_id after comma: %w", err)
-				}
-				chunkIDs = append(chunkIDs, chunkID)
-			} else {
+			// Stop if we hit FROM or non-quoted token
+			if p.curToken.Type == TokenFrom || p.curToken.Type != TokenQuotedString {
 				break
 			}
+			chunkID, err := p.parseQuotedString()
+			if err != nil {
+				return nil, fmt.Errorf("expected chunk_id: %w", err)
+			}
+			chunkIDs = append(chunkIDs, chunkID)
 		}
 		cmd.Params["chunk_ids"] = chunkIDs
 	}