Implement Delete in GO and refactor functions (#13974)

### What problem does this PR solve?

Implement Delete in GO and refactor functions

### Type of change

- [x] Refactoring

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a remove_chunks command to delete specific or all chunks from a
document.
  * Added new endpoints for chunk removal and chunk update.

* **Refactor**
* Renamed index commands to dataset/metadata table terminology and
updated REST routes accordingly.
* Updated chunk update flow to a JSON POST style and improved metadata
error messages.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

internal/cli/client.go

@@ -208,14 +208,6 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.UnsetToken(cmd)
 	case "show_version":
 		return c.ShowServerVersion(cmd)
-	case "create_index":
-		return c.CreateIndex(cmd)
-	case "drop_index":
-		return c.DropIndex(cmd)
-	case "create_doc_meta_index":
-		return c.CreateDocMetaIndex(cmd)
-	case "drop_doc_meta_index":
-		return c.DropDocMetaIndex(cmd)
 	case "list_available_providers":
 		return c.ListAvailableProviders(cmd)
 	case "show_provider":
@@ -256,6 +248,27 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.UseModel(cmd)
 	case "show_current_model":
 		return c.ShowCurrentModel(cmd)
+	// Dataset, metadata commands
+	case "create_dataset_table":
+		return c.CreateDatasetInDocEngine(cmd)
+	case "drop_dataset_table":
+		return c.DropDatasetInDocEngine(cmd)
+	case "create_metadata_table":
+		return c.CreateMetadataInDocEngine(cmd)
+	case "drop_metadata_table":
+		return c.DropMetadataInDocEngine(cmd)
+	case "insert_dataset_from_file":
+		return c.InsertDatasetFromFile(cmd)
+	case "insert_metadata_from_file":
+		return c.InsertMetadataFromFile(cmd)
+	case "update_chunk":
+		return c.UpdateChunk(cmd)
+	case "set_meta":
+		return c.SetMeta(cmd)
+	case "rm_tags":
+		return c.RmTags(cmd)
+	case "remove_chunks":
+		return c.RemoveChunks(cmd)
 	// ContextEngine commands
 	case "context_list":
 		return c.ContextList(cmd)
@@ -267,16 +280,6 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
 		return c.CEList(cmd)
 	case "ce_search":
 		return c.CESearch(cmd)
-	case "insert_dataset_from_file":
-		return c.InsertDatasetFromFile(cmd)
-	case "insert_metadata_from_file":
-		return c.InsertMetadataFromFile(cmd)
-	case "update_chunk":
-		return c.UpdateChunk(cmd)
-	case "set_meta":
-		return c.SetMeta(cmd)
-	case "rm_tags":
-		return c.RmTags(cmd)
 	// TODO: Implement other commands
 	default:
 		return nil, fmt.Errorf("command '%s' would be executed with API", cmd.Type)

internal/cli/lexer.go

@@ -53,6 +53,22 @@ func (l *Lexer) peekChar() byte {
 	return l.input[l.readPos]
 }
 
+func (l *Lexer) peekToken() string {
+	// Skip whitespace starting from readPos
+	skipPos := l.readPos
+	for skipPos < len(l.input) && (l.input[skipPos] == ' ' || l.input[skipPos] == '\t' || l.input[skipPos] == '\n' || l.input[skipPos] == '\r') {
+		skipPos++
+	}
+
+	// Read identifier starting from skipPos
+	start := skipPos
+	for skipPos < len(l.input) && (isLetter(l.input[skipPos]) || isDigit(l.input[skipPos]) || l.input[skipPos] == '_' || l.input[skipPos] == '-' || l.input[skipPos] == '.') {
+		skipPos++
+	}
+
+	return l.input[start:skipPos]
+}
+
 func (l *Lexer) skipWhitespace() {
 	for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
 		l.readChar()
@@ -206,6 +222,18 @@ func (l *Lexer) lookupIdent(ident string) Token {
 	case "PASSWORD":
 		return Token{Type: TokenPassword, Value: ident}
 	case "DATASET":
+		// Check if followed by TABLE for compound token
+		if strings.ToUpper(l.peekToken()) == "TABLE" {
+			// Skip whitespace to TABLE
+			for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+				l.readChar()
+			}
+			// Skip past TABLE
+			for isLetter(l.ch) || isDigit(l.ch) || l.ch == '_' || l.ch == '-' || l.ch == '.' {
+				l.readChar()
+			}
+			return Token{Type: TokenDatasetTable, Value: "DATASET TABLE"}
+		}
 		return Token{Type: TokenDataset, Value: ident}
 	case "DATASETS":
 		return Token{Type: TokenDatasets, Value: ident}
@@ -325,10 +353,14 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenTokens, Value: ident}
 	case "INDEX":
 		return Token{Type: TokenIndex, Value: ident}
-	case "VECTOR_SIZE":
-		return Token{Type: TokenVectorSize, Value: ident}
-	case "DOC_META":
-		return Token{Type: TokenDocMeta, Value: ident}
+	case "VECTOR":
+		return Token{Type: TokenVector, Value: ident}
+	case "SIZE":
+		return Token{Type: TokenSize, Value: ident}
+	case "METADATA":
+		return Token{Type: TokenMetadata, Value: ident}
+	case "TABLE":
+		return Token{Type: TokenTable, Value: ident}
 	case "AVAILABLE":
 		return Token{Type: TokenAvailable, Value: ident}
 	case "NAME":
@@ -345,8 +377,6 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenInsert, Value: ident}
 	case "FILE":
 		return Token{Type: TokenFile, Value: ident}
-	case "METADATA":
-		return Token{Type: TokenMetadata, Value: ident}
 	case "USE":
 		return Token{Type: TokenUse, Value: ident}
 	case "UPDATE":
@@ -355,6 +385,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenRemove, Value: ident}
 	case "CHUNK":
 		return Token{Type: TokenChunk, Value: ident}
+	case "CHUNKS":
+		return Token{Type: TokenChunks, Value: ident}
 	case "DOCUMENT":
 		return Token{Type: TokenDocument, Value: ident}
 	case "TAGS":

internal/cli/types.go

@@ -47,6 +47,7 @@ const (
 	TokenPassword
 	TokenDataset
 	TokenDatasets
+	TokenDatasetTable
 	TokenOf
 	TokenAgents
 	TokenRole
@@ -103,7 +104,8 @@ const (
 	TokenTokens
 	TokenUnset
 	TokenIndex
-	TokenVectorSize
+	TokenVector
+	TokenSize
 	TokenDocMeta
 	TokenName // For ALTER PROVIDER <name> NAME <new_name>
 	TokenInstance
@@ -117,9 +119,11 @@ const (
 	TokenInsert
 	TokenFile
 	TokenMetadata
+	TokenTable
 	TokenUpdate
 	TokenRemove
 	TokenChunk
+	TokenChunks
 	TokenDocument
 	TokenTag
 	TokenLog

internal/cli/user_command.go

@@ -759,8 +759,8 @@ func (c *RAGFlowClient) UnsetToken(cmd *Command) (ResponseIf, error) {
 	return &result, nil
 }
 
-// CreateIndex creates an index for a dataset
-func (c *RAGFlowClient) CreateIndex(cmd *Command) (ResponseIf, error) {
+// CreateDataset creates a table for a dataset
+func (c *RAGFlowClient) CreateDataset(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
@@ -786,13 +786,13 @@ func (c *RAGFlowClient) CreateIndex(cmd *Command) (ResponseIf, error) {
 		"vector_size": vectorSize,
 	}
 
-	resp, err := c.HTTPClient.Request("POST", "/kb/index", false, "web", nil, payload)
+	resp, err := c.HTTPClient.Request("POST", "/kb/doc_engine_table", false, "web", nil, payload)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create index: %w", err)
+		return nil, fmt.Errorf("failed to create table: %w", err)
 	}
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to create index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+		return nil, fmt.Errorf("failed to create table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
 	}
 
 	resJSON, err := resp.JSON()
@@ -808,16 +808,73 @@ func (c *RAGFlowClient) CreateIndex(cmd *Command) (ResponseIf, error) {
 	var result SimpleResponse
 	result.Code = int(code)
 	if result.Code == 0 {
-		result.Message = fmt.Sprintf("Success to create index for dataset: %s", datasetName)
+		result.Message = fmt.Sprintf("Success to create table for dataset: %s", datasetName)
 	} else {
-		result.Message = fmt.Sprintf("Failed to create index: %v", resJSON)
+		result.Message = fmt.Sprintf("Failed to create table: %v", resJSON)
 	}
 	result.Duration = 0
 	return &result, nil
 }
 
-// DropIndex drops an index for a dataset
-func (c *RAGFlowClient) DropIndex(cmd *Command) (ResponseIf, error) {
+// CreateDatasetInDocEngine creates a table for a dataset in doc engine
+func (c *RAGFlowClient) CreateDatasetInDocEngine(cmd *Command) (ResponseIf, error) {
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	datasetName, ok := cmd.Params["dataset_name"].(string)
+	if !ok {
+		return nil, fmt.Errorf("dataset_name not provided")
+	}
+
+	vectorSize, ok := cmd.Params["vector_size"].(int)
+	if !ok {
+		return nil, fmt.Errorf("vector_size not provided")
+	}
+
+	// Get dataset ID by name
+	datasetID, err := c.getDatasetID(datasetName)
+	if err != nil {
+		return nil, err
+	}
+
+	payload := map[string]interface{}{
+		"kb_id":       datasetID,
+		"vector_size": vectorSize,
+	}
+
+	resp, err := c.HTTPClient.Request("POST", "/kb/doc_engine_table", false, "web", nil, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create table: %w", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to create table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+
+	resJSON, err := resp.JSON()
+	if err != nil {
+		return nil, fmt.Errorf("invalid JSON response: %w", err)
+	}
+
+	code, ok := resJSON["code"].(float64)
+	if !ok {
+		return nil, fmt.Errorf("invalid response format: code is not a number")
+	}
+
+	var result SimpleResponse
+	result.Code = int(code)
+	if result.Code == 0 {
+		result.Message = fmt.Sprintf("Success to create table for dataset: %s", datasetName)
+	} else {
+		result.Message = fmt.Sprintf("Failed to create table: %v", resJSON)
+	}
+	result.Duration = 0
+	return &result, nil
+}
+
+// DropDatasetInDocEngine drops a table for a dataset in doc engine
+func (c *RAGFlowClient) DropDatasetInDocEngine(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
@@ -837,13 +894,13 @@ func (c *RAGFlowClient) DropIndex(cmd *Command) (ResponseIf, error) {
 		"kb_id": datasetID,
 	}
 
-	resp, err := c.HTTPClient.Request("DELETE", "/kb/index", false, "web", nil, payload)
+	resp, err := c.HTTPClient.Request("DELETE", "/kb/doc_engine_table", false, "web", nil, payload)
 	if err != nil {
-		return nil, fmt.Errorf("failed to drop index: %w", err)
+		return nil, fmt.Errorf("failed to drop dataset: %w", err)
 	}
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to drop index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+		return nil, fmt.Errorf("failed to drop dataset: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
 	}
 
 	resJSON, err := resp.JSON()
@@ -859,27 +916,27 @@ func (c *RAGFlowClient) DropIndex(cmd *Command) (ResponseIf, error) {
 	var result SimpleResponse
 	result.Code = int(code)
 	if result.Code == 0 {
-		result.Message = fmt.Sprintf("Success to drop index for dataset: %s", datasetName)
+		result.Message = fmt.Sprintf("Success to drop table for dataset: %s", datasetName)
 	} else {
-		result.Message = fmt.Sprintf("Failed to drop index: %v", resJSON)
+		result.Message = fmt.Sprintf("Failed to drop table for dataset: %s: %v", datasetName, resJSON)
 	}
 	result.Duration = 0
 	return &result, nil
 }
 
-// CreateDocMetaIndex creates the document metadata index for the tenant
-func (c *RAGFlowClient) CreateDocMetaIndex(cmd *Command) (ResponseIf, error) {
+// CreateMetadataInDocEngine creates the document metadata table for the tenant
+func (c *RAGFlowClient) CreateMetadataInDocEngine(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
 
-	resp, err := c.HTTPClient.Request("POST", "/tenant/doc_meta_index", false, "web", nil, nil)
+	resp, err := c.HTTPClient.Request("POST", "/tenant/doc_engine_metadata_table", false, "web", nil, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create doc meta index: %w", err)
+		return nil, fmt.Errorf("failed to create metadata table: %w", err)
 	}
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to create doc meta index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+		return nil, fmt.Errorf("failed to create metadata table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
 	}
 
 	resJSON, err := resp.JSON()
@@ -895,27 +952,27 @@ func (c *RAGFlowClient) CreateDocMetaIndex(cmd *Command) (ResponseIf, error) {
 	var result SimpleResponse
 	result.Code = int(code)
 	if result.Code == 0 {
-		result.Message = "Success to create doc meta index"
+		result.Message = "Success to create metadata table"
 	} else {
-		result.Message = fmt.Sprintf("Failed to create doc meta index: %v", resJSON)
+		result.Message = fmt.Sprintf("Failed to create metadata table: %v", resJSON)
 	}
 	result.Duration = 0
 	return &result, nil
 }
 
-// DropDocMetaIndex drops the document metadata index for the tenant
-func (c *RAGFlowClient) DropDocMetaIndex(cmd *Command) (ResponseIf, error) {
+// DropMetadataInDocEngine drops the document metadata table for the tenant
+func (c *RAGFlowClient) DropMetadataInDocEngine(cmd *Command) (ResponseIf, error) {
 	if c.ServerType != "user" {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
 
-	resp, err := c.HTTPClient.Request("DELETE", "/tenant/doc_meta_index", false, "web", nil, nil)
+	resp, err := c.HTTPClient.Request("DELETE", "/tenant/doc_engine_metadata_table", false, "web", nil, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to drop doc meta index: %w", err)
+		return nil, fmt.Errorf("failed to drop metadata table: %w", err)
 	}
 
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to drop doc meta index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+		return nil, fmt.Errorf("failed to drop metadata table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
 	}
 
 	resJSON, err := resp.JSON()
@@ -931,9 +988,9 @@ func (c *RAGFlowClient) DropDocMetaIndex(cmd *Command) (ResponseIf, error) {
 	var result SimpleResponse
 	result.Code = int(code)
 	if result.Code == 0 {
-		result.Message = "Success to drop doc meta index"
+		result.Message = "Success to drop metadata table"
 	} else {
-		result.Message = fmt.Sprintf("Failed to drop doc meta index: %v", resJSON)
+		result.Message = fmt.Sprintf("Failed to drop metadata table: %v", resJSON)
 	}
 	result.Duration = 0
 	return &result, nil
@@ -1729,8 +1786,12 @@ func (c *RAGFlowClient) UpdateChunk(cmd *Command) (ResponseIf, error) {
 		return nil, fmt.Errorf("invalid JSON body: %w", err)
 	}
 
-	path := fmt.Sprintf("/datasets/%s/documents/%s/chunks/%s", datasetID, docID, chunkID)
-	resp, err := c.HTTPClient.Request("PUT", path, true, "api", nil, payload)
+	// Add IDs to payload
+	payload["dataset_id"] = datasetID
+	payload["document_id"] = docID
+	payload["chunk_id"] = chunkID
+
+	resp, err := c.HTTPClient.Request("POST", "/chunk/update", false, "web", nil, payload)
 	if err != nil {
 		return nil, fmt.Errorf("failed to update chunk: %w", err)
 	}
@@ -1865,3 +1926,64 @@ func (c *RAGFlowClient) RmTags(cmd *Command) (ResponseIf, error) {
 	result.Duration = 0
 	return &result, nil
 }
+
+// RemoveChunks removes chunks from a document
+func (c *RAGFlowClient) RemoveChunks(cmd *Command) (ResponseIf, error) {
+	if c.ServerType != "user" {
+		return nil, fmt.Errorf("this command is only allowed in USER mode")
+	}
+
+	docID, ok := cmd.Params["doc_id"].(string)
+	if !ok {
+		return nil, fmt.Errorf("doc_id not provided")
+	}
+
+	payload := map[string]interface{}{
+		"doc_id": docID,
+	}
+
+	// Check if delete_all is set
+	if deleteAll, ok := cmd.Params["delete_all"].(bool); ok && deleteAll {
+		payload["delete_all"] = true
+	} else if chunkIDs, ok := cmd.Params["chunk_ids"].([]string); ok {
+		payload["chunk_ids"] = chunkIDs
+	}
+
+	resp, err := c.HTTPClient.Request("POST", "/chunk/rm", false, "web", nil, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to remove chunks: %w", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to remove chunks: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
+	}
+
+	resJSON, err := resp.JSON()
+	if err != nil {
+		return nil, fmt.Errorf("invalid JSON response: %w", err)
+	}
+
+	code, ok := resJSON["code"].(float64)
+	if !ok {
+		return nil, fmt.Errorf("invalid response format: code is not a number")
+	}
+
+	var result SimpleResponse
+	result.Code = int(code)
+	if result.Code == 0 {
+		deletedCount := int64(0)
+		switch data := resJSON["data"].(type) {
+		case float64:
+			deletedCount = int64(data)
+		case map[string]interface{}:
+			if count, ok := data["deleted_count"].(float64); ok {
+				deletedCount = int64(count)
+			}
+		}
+		result.Message = fmt.Sprintf("Success to remove chunks from document %s: %d chunks deleted", docID, deletedCount)
+	} else {
+		result.Message = fmt.Sprintf("Failed to remove chunks: %v", resJSON)
+	}
+	result.Duration = 0
+	return &result, nil
+}

internal/cli/user_parser.go

@@ -511,8 +511,10 @@ func (p *Parser) parseCreateCommand() (*Command, error) {
 		return p.parseCreateChat()
 	case TokenToken:
 		return p.parseCreateToken()
-	case TokenIndex:
-		return p.parseCreateIndex()
+	case TokenDatasetTable:
+		return p.parseCreateDatasetTable()
+	case TokenMetadata:
+		return p.parseCreateMetadataTable()
 	case TokenProvider:
 		return p.parseCreateProviderInstance()
 	default:
@@ -541,30 +543,10 @@ func (p *Parser) parseCreateToken() (*Command, error) {
 	return NewCommand("create_token"), nil
 }
 
-func (p *Parser) parseCreateIndex() (*Command, error) {
-	// CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
-	// CREATE INDEX DOC_META
-	p.nextToken() // consume INDEX
-
-	// Check if creating doc meta index
-	if p.curToken.Type == TokenDocMeta {
-		p.nextToken()
-		if p.curToken.Type == TokenSemicolon {
-			p.nextToken()
-		}
-		return NewCommand("create_doc_meta_index"), nil
-	}
-
-	// Otherwise, must be CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
-	if p.curToken.Type != TokenFor {
-		return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
-	}
-	p.nextToken()
-
-	if p.curToken.Type != TokenDataset {
-		return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
-	}
-	p.nextToken()
+// Internal CLI for GO
+// parseCreateDatasetTable parses: CREATE DATASET TABLE 'name' VECTOR SIZE N
+func (p *Parser) parseCreateDatasetTable() (*Command, error) {
+	p.nextToken() // consume DATASET TABLE compound token
 
 	datasetName, err := p.parseQuotedString()
 	if err != nil {
@@ -572,8 +554,12 @@ func (p *Parser) parseCreateIndex() (*Command, error) {
 	}
 
 	p.nextToken()
-	if p.curToken.Type != TokenVectorSize {
-		return nil, fmt.Errorf("expected VECTOR_SIZE after dataset name, got %s", p.curToken.Value)
+	if p.curToken.Type != TokenVector {
+		return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+	if p.curToken.Type != TokenSize {
+		return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value)
 	}
 	p.nextToken()
 
@@ -590,12 +576,30 @@ func (p *Parser) parseCreateIndex() (*Command, error) {
 		p.nextToken()
 	}
 
-	cmd := NewCommand("create_index")
+	cmd := NewCommand("create_dataset_table")
 	cmd.Params["dataset_name"] = datasetName
 	cmd.Params["vector_size"] = vectorSize
 	return cmd, nil
 }
 
+// Internal CLI for GO
+// parseCreateMetadataTable parses: CREATE METADATA TABLE
+func (p *Parser) parseCreateMetadataTable() (*Command, error) {
+	// CREATE METADATA TABLE
+	p.nextToken() // consume METADATA
+
+	if p.curToken.Type != TokenTable {
+		return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return NewCommand("create_metadata_table"), nil
+}
+
 func (p *Parser) parseCreateUser() (*Command, error) {
 	p.nextToken() // consume USER
 	userName, err := p.parseQuotedString()
@@ -801,8 +805,10 @@ func (p *Parser) parseDropCommand() (*Command, error) {
 		return p.parseDropChat()
 	case TokenToken:
 		return p.parseDropToken()
-	case TokenIndex:
-		return p.parseDropIndex()
+	case TokenDatasetTable:
+		return p.parseDropDatasetTable()
+	case TokenMetadata:
+		return p.parseDropMetadataTable()
 	case TokenInstance:
 		return p.parseDropInstance()
 	default:
@@ -827,6 +833,8 @@ func (p *Parser) parseRemoveCommand() (*Command, error) {
 	switch p.curToken.Type {
 	case TokenTag:
 		return p.parseRemoveTags()
+	case TokenChunks, TokenAll:
+		return p.parseRemoveChunk()
 	default:
 		return nil, fmt.Errorf("unknown REMOVE target: %s", p.curToken.Value)
 	}
@@ -863,30 +871,10 @@ func (p *Parser) parseDropToken() (*Command, error) {
 	return cmd, nil
 }
 
-func (p *Parser) parseDropIndex() (*Command, error) {
-	// DROP INDEX FOR DATASET 'name' OR DROP INDEX DOC_META
-	p.nextToken() // consume INDEX
-
-	// Check if dropping doc meta index
-	if p.curToken.Type == TokenDocMeta {
-		p.nextToken()
-		if p.curToken.Type == TokenSemicolon {
-			p.nextToken()
-		}
-		cmd := NewCommand("drop_doc_meta_index")
-		return cmd, nil
-	}
-
-	// Otherwise, must be DROP INDEX FOR DATASET 'name'
-	if p.curToken.Type != TokenFor {
-		return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
-	}
-	p.nextToken()
-
-	if p.curToken.Type != TokenDataset {
-		return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
-	}
-	p.nextToken()
+// Internal CLI for GO
+// parseDropDatasetTable parses: DROP DATASET TABLE 'name'
+func (p *Parser) parseDropDatasetTable() (*Command, error) {
+	p.nextToken() // consume DATASET TABLE
 
 	datasetName, err := p.parseQuotedString()
 	if err != nil {
@@ -898,11 +886,29 @@ func (p *Parser) parseDropIndex() (*Command, error) {
 		p.nextToken()
 	}
 
-	cmd := NewCommand("drop_index")
+	cmd := NewCommand("drop_dataset_table")
 	cmd.Params["dataset_name"] = datasetName
 	return cmd, nil
 }
 
+// Internal CLI for GO
+// parseDropMetadataTable parses: DROP METADATA TABLE
+func (p *Parser) parseDropMetadataTable() (*Command, error) {
+	// DROP METADATA TABLE
+	p.nextToken() // consume METADATA
+
+	if p.curToken.Type != TokenTable {
+		return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("drop_metadata_table")
+	return cmd, nil
+}
+
 func (p *Parser) parseDropUser() (*Command, error) {
 	p.nextToken() // consume USER
 	userName, err := p.parseQuotedString()
@@ -2428,15 +2434,21 @@ func (p *Parser) parseUnsetCommand() (*Command, error) {
 	return NewCommand("unset_token"), nil
 }
 
-// parseUpdateCommand parses UPDATE CHUNK command
-// UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
+// Internal
+// parseUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
 func (p *Parser) parseUpdateCommand() (*Command, error) {
 	p.nextToken() // consume UPDATE
 
-	if p.curToken.Type != TokenChunk {
-		return nil, fmt.Errorf("expected CHUNK after UPDATE")
+	if p.curToken.Type == TokenChunk {
+		return p.parseUpdateChunk()
 	}
-	p.nextToken()
+
+	return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value)
+}
+
+// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
+func (p *Parser) parseUpdateChunk() (*Command, error) {
+	p.nextToken() // consume CHUNK
 
 	// Parse chunk_id
 	chunkID, err := p.parseQuotedString()
@@ -2588,3 +2600,74 @@ func (p *Parser) parseRemoveTags() (*Command, error) {
 
 	return cmd, nil
 }
+
+
+// parseRemoveChunk parses:
+//   - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id';
+//   - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id';
+func (p *Parser) parseRemoveChunk() (*Command, error) {
+	cmd := NewCommand("remove_chunks")
+
+	// Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL
+	if p.curToken.Type == TokenAll {
+		p.nextToken() // consume ALL
+		if p.curToken.Type != TokenChunks {
+			return nil, fmt.Errorf("expected CHUNKS after ALL")
+		}
+		p.nextToken() // consume CHUNKS
+		cmd.Params["delete_all"] = true
+	} else {
+		// curToken is TokenChunks, consume it first
+		p.nextToken()
+		// Multiple chunks: REMOVE CHUNKS 'id1', 'id2' FROM DOCUMENT 'doc_id'
+		// Parse first chunk ID
+		chunkID, err := p.parseQuotedString()
+		if err != nil {
+			return nil, fmt.Errorf("expected chunk_id: %w", err)
+		}
+		chunkIDs := []string{chunkID}
+
+		// Parse additional chunk IDs separated by commas
+		for {
+			p.nextToken()
+			if p.curToken.Type == TokenComma {
+				p.nextToken()
+				chunkID, err := p.parseQuotedString()
+				if err != nil {
+					return nil, fmt.Errorf("expected chunk_id after comma: %w", err)
+				}
+				chunkIDs = append(chunkIDs, chunkID)
+			} else {
+				break
+			}
+		}
+		cmd.Params["chunk_ids"] = chunkIDs
+	}
+
+	// Expect FROM
+	if p.curToken.Type != TokenFrom {
+		return nil, fmt.Errorf("expected FROM after chunk(s)")
+	}
+	p.nextToken()
+
+	// Expect DOCUMENT
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after FROM")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd.Params["doc_id"] = docID
+	p.nextToken()
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}