Go CLI: refactor commands (#16447)

### Summary 1. Move debug commands to dev file. 2. Refactor some commands syntax --------- Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2026-07-13 16:38:26 +08:00 · 2026-06-29 17:03:26 +08:00
parent ec5cd6b1c0
commit 3202ec6abf
12 changed files with 999 additions and 1022 deletions
--- a/internal/cli/dev_parser.go
+++ b/internal/cli/dev_parser.go
@@ -0,0 +1,659 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+package cli
+
+import (
+	"fmt"
+	"strconv"
+)
+
+// INSERT CHUNKS FROM FILE "file_path"
+// INSERT METADATA FROM FILE "file_path"
+func (p *Parser) parseDevInsertCommand() (*Command, error) {
+	p.nextToken() // consume INSERT
+
+	// Expect CHUNKS or METADATA
+	if p.curToken.Type == TokenChunks {
+		return p.parseInsertChunksFromFile()
+	}
+	if p.curToken.Type == TokenMetadata {
+		return p.parseInsertMetadataFromFile()
+	}
+	return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value)
+}
+
+// Internal CLI for GO
+// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path"
+func (p *Parser) parseInsertChunksFromFile() (*Command, error) {
+	p.nextToken() // consume CHUNKS
+
+	// Expect FROM
+	if p.curToken.Type != TokenFrom {
+		return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Expect FILE
+	if p.curToken.Type != TokenFile {
+		return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Get file path (quoted string)
+	filePath, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := NewCommand("dev_insert_chunks_from_file")
+	cmd.Params["file_path"] = filePath
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+	return cmd, nil
+}
+
+// Internal CLI for GO
+// parseInsertMetadataFromFile parses: INSERT METADATA FROM FILE "file_path"
+func (p *Parser) parseInsertMetadataFromFile() (*Command, error) {
+	p.nextToken() // consume METADATA
+
+	// Expect FROM
+	if p.curToken.Type != TokenFrom {
+		return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Expect FILE
+	if p.curToken.Type != TokenFile {
+		return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Get file path (quoted string)
+	filePath, err := p.parseQuotedString()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := NewCommand("dev_insert_metadata_from_file")
+	cmd.Params["file_path"] = filePath
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+	return cmd, nil
+}
+
+// parseGetCommand parses: GET CHUNK or GET METADATA
+func (p *Parser) parseGetCommand() (*Command, error) {
+	p.nextToken() // consume GET
+
+	if p.curToken.Type == TokenChunk {
+		return p.parseGetChunk()
+	}
+	if p.curToken.Type == TokenMetadata {
+		return p.parseGetMetadata()
+	}
+
+	return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value)
+}
+
+// parseGetChunk parses: GET CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id'
+func (p *Parser) parseGetChunk() (*Command, error) {
+	p.nextToken() // consume CHUNK
+
+	// Parse chunk_id
+	chunkID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected chunk_id: %w", err)
+	}
+
+	cmd := NewCommand("dev_get_chunk")
+	cmd.Params["chunk_id"] = chunkID
+
+	p.nextToken()
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after chunk_id")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after OF")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+	if p.curToken.Type != TokenIn {
+		return nil, fmt.Errorf("expected IN after doc_id")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after IN")
+	}
+	p.nextToken()
+
+	// Parse dataset_id
+	datasetID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset_id: %w", err)
+	}
+	cmd.Params["dataset_id"] = datasetID
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
+// Internal
+// parseDevUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
+func (p *Parser) parseDevUpdateCommand() (*Command, error) {
+	p.nextToken() // consume UPDATE
+
+	if p.curToken.Type == TokenChunk {
+		return p.parseUpdateChunk()
+	}
+
+	return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value)
+}
+
+// Internal CLI for GO
+// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' SET '{"content": "..."}'
+func (p *Parser) parseUpdateChunk() (*Command, error) {
+	p.nextToken() // consume CHUNK
+
+	// Parse chunk_id
+	chunkID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected chunk_id: %w", err)
+	}
+
+	cmd := NewCommand("dev_update_chunk")
+	cmd.Params["chunk_id"] = chunkID
+
+	p.nextToken()
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after chunk_id")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after OF")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+	if p.curToken.Type != TokenIn {
+		return nil, fmt.Errorf("expected IN after doc_id")
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after IN")
+	}
+	p.nextToken()
+
+	// Parse dataset_name
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset_name: %w", err)
+	}
+	cmd.Params["dataset_name"] = datasetName
+
+	p.nextToken()
+	if p.curToken.Type != TokenSet {
+		return nil, fmt.Errorf("expected SET after dataset_name")
+	}
+	p.nextToken()
+
+	// Parse JSON body
+	jsonBody, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected JSON body: %w", err)
+	}
+	cmd.Params["json_body"] = jsonBody
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
+// parseSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}'
+func (p *Parser) parseSetMeta() (*Command, error) {
+	p.nextToken() // consume METADATA
+
+	// Expect OF
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after SET METADATA")
+	}
+	p.nextToken()
+
+	// Expect DOCUMENT
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after SET METADATA OF")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd := NewCommand("dev_set_meta")
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+	// Expect TO
+	if p.curToken.Type != TokenTo {
+		return nil, fmt.Errorf("expected TO after doc_id")
+	}
+	p.nextToken()
+
+	// Parse meta JSON
+	meta, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected meta JSON: %w", err)
+	}
+	cmd.Params["meta"] = meta
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
+// parseDevDeleteMeta parses: DELETE METADATA OF DOCUMENT 'doc_id' [KEYS '["key1", "key2"]']
+// If KEYS is not provided, deletes entire document metadata
+func (p *Parser) parseDevDeleteMeta() (*Command, error) {
+	p.nextToken() // consume METADATA
+
+	// Expect OF
+	if p.curToken.Type != TokenOf {
+		return nil, fmt.Errorf("expected OF after DELETE METADATA")
+	}
+	p.nextToken()
+
+	// Expect DOCUMENT
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after DELETE METADATA OF")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd := NewCommand("dev_delete_meta")
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+	// KEYS is optional - if not provided, delete entire document metadata
+	if p.curToken.Type != TokenKeys {
+		if p.curToken.Type == TokenSemicolon {
+			p.nextToken()
+			return cmd, nil
+		}
+		if p.curToken.Type == TokenEOF {
+			return cmd, nil
+		}
+		return nil, fmt.Errorf("expected KEYS or end of command after doc_id")
+	}
+
+	// Parse keys JSON array
+	p.nextToken()
+	keys, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected keys JSON array: %w", err)
+	}
+	cmd.Params["keys"] = keys
+
+	p.nextToken()
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+		return cmd, nil
+	}
+	if p.curToken.Type != TokenEOF {
+		return nil, fmt.Errorf("expected end of command after KEYS")
+	}
+
+	return cmd, nil
+}
+
+// parseDevRemoveTags parses: REMOVE TAGS 'tag1', 'tag2' from DATASET 'dataset_name';
+func (p *Parser) parseDevRemoveTags() (*Command, error) {
+	p.nextToken() // consume TAGS
+
+	// Parse first tag
+	tag, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected tag: %w", err)
+	}
+	tags := []string{tag}
+
+	// Parse additional tags separated by commas
+	for {
+		p.nextToken()
+		if p.curToken.Type == TokenComma {
+			p.nextToken()
+			tag, err := p.parseQuotedString()
+			if err != nil {
+				return nil, fmt.Errorf("expected tag after comma: %w", err)
+			}
+			tags = append(tags, tag)
+		} else {
+			break
+		}
+	}
+	cmd := NewCommand("dev_rm_tags")
+	cmd.Params["tags"] = tags
+
+	// Expect from
+	if p.curToken.Type != TokenFrom {
+		return nil, fmt.Errorf("expected FROM after tags")
+	}
+	p.nextToken()
+
+	// Expect DATASET
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after FROM")
+	}
+	p.nextToken()
+
+	// Parse dataset_name
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset_name: %w", err)
+	}
+	cmd.Params["dataset_name"] = datasetName
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
+// parseDevRemoveChunk parses:
+//   - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
+//   - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
+func (p *Parser) parseDevRemoveChunk() (*Command, error) {
+	cmd := NewCommand("dev_remove_chunks")
+
+	// Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL
+	if p.curToken.Type == TokenAll {
+		p.nextToken() // consume ALL
+		if p.curToken.Type != TokenChunks {
+			return nil, fmt.Errorf("expected CHUNKS after ALL")
+		}
+		p.nextToken() // consume CHUNKS
+		cmd.Params["delete_all"] = true
+	} else {
+		// curToken is TokenChunks, consume it first
+		p.nextToken()
+		// Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name' (space-separated)
+		// Parse first chunk ID
+		chunkID, err := p.parseQuotedString()
+		if err != nil {
+			return nil, fmt.Errorf("expected chunk_id: %w", err)
+		}
+		chunkIDs := []string{chunkID}
+
+		// Parse additional chunk IDs separated by spaces (each quoted)
+		for {
+			p.nextToken()
+			// Stop if we hit FROM or non-quoted token
+			if p.curToken.Type == TokenFrom || p.curToken.Type != TokenQuotedString {
+				break
+			}
+			chunkID, err := p.parseQuotedString()
+			if err != nil {
+				return nil, fmt.Errorf("expected chunk_id: %w", err)
+			}
+			chunkIDs = append(chunkIDs, chunkID)
+		}
+		cmd.Params["chunk_ids"] = chunkIDs
+	}
+
+	// Expect FROM
+	if p.curToken.Type != TokenFrom {
+		return nil, fmt.Errorf("expected FROM after chunk(s)")
+	}
+	p.nextToken()
+
+	// Expect DOCUMENT
+	if p.curToken.Type != TokenDocument {
+		return nil, fmt.Errorf("expected DOCUMENT after FROM")
+	}
+	p.nextToken()
+
+	// Parse doc_id
+	docID, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected doc_id: %w", err)
+	}
+	cmd.Params["doc_id"] = docID
+
+	p.nextToken()
+
+	// Expect IN
+	if p.curToken.Type != TokenIn {
+		return nil, fmt.Errorf("expected IN after doc_id")
+	}
+	p.nextToken()
+
+	// Expect DATASET
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected DATASET after IN")
+	}
+	p.nextToken()
+
+	// Parse dataset_name (quoted string)
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset_name: %w", err)
+	}
+	cmd.Params["dataset_name"] = datasetName
+
+	p.nextToken()
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return cmd, nil
+}
+
+// Internal CLI for GO
+// parseDevDropChunkStore parses: DROP CHUNK STORE for Dataset 'name'
+func (p *Parser) parseDevDropChunkStore() (*Command, error) {
+	p.nextToken() // consume CHUNK STORE
+
+	// Expect FOR
+	if p.curToken.Type != TokenFor {
+		return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Expect Dataset
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
+	}
+
+	p.nextToken()
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("dev_drop_chunk_store")
+	cmd.Params["dataset_name"] = datasetName
+	return cmd, nil
+}
+
+// parseDevDropMetadataStore parses: DROP METADATA STORE
+func (p *Parser) parseDevDropMetadataStore() (*Command, error) {
+	// DROP METADATA STORE
+	p.nextToken() // consume METADATA
+
+	if p.curToken.Type != TokenStore {
+		return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("dev_drop_metadata_store")
+	return cmd, nil
+}
+
+// Internal CLI for GO
+// parseDevCreateChunkStore parses: CREATE CHUNK STORE for Dataset 'name' VECTOR SIZE N
+func (p *Parser) parseDevCreateChunkStore() (*Command, error) {
+	p.nextToken() // consume CHUNK STORE compound token
+
+	// Expect FOR
+	if p.curToken.Type != TokenFor {
+		return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	// Expect Dataset
+	if p.curToken.Type != TokenDataset {
+		return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	datasetName, err := p.parseQuotedString()
+	if err != nil {
+		return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
+	}
+
+	p.nextToken()
+	if p.curToken.Type != TokenVector {
+		return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+	if p.curToken.Type != TokenSize {
+		return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	if p.curToken.Type != TokenInteger {
+		return nil, fmt.Errorf("expected vector size number, got %s", p.curToken.Value)
+	}
+	vectorSize, err := strconv.Atoi(p.curToken.Value)
+	if err != nil {
+		return nil, fmt.Errorf("invalid vector size: %s", p.curToken.Value)
+	}
+
+	p.nextToken()
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	cmd := NewCommand("dev_create_chunk_store")
+	cmd.Params["dataset_name"] = datasetName
+	cmd.Params["vector_size"] = vectorSize
+	return cmd, nil
+}
+
+// Internal CLI for GO
+// parseCreateMetadataStore parses: CREATE METADATA STORE
+func (p *Parser) parseCreateMetadataStore() (*Command, error) {
+	// CREATE METADATA STORE
+	p.nextToken() // consume METADATA
+
+	if p.curToken.Type != TokenStore {
+		return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
+	}
+	p.nextToken()
+
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+
+	return NewCommand("dev_create_metadata_store"), nil
+}
+
+func (p *Parser) parseCreateRole() (*Command, error) {
+	p.nextToken() // consume ROLE
+	roleName, err := p.parseIdentifier()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := NewCommand("create_role")
+	cmd.Params["role_name"] = roleName
+
+	p.nextToken()
+	if p.curToken.Type == TokenDescription {
+		p.nextToken()
+		description, err := p.parseQuotedString()
+		if err != nil {
+			return nil, err
+		}
+		cmd.Params["description"] = description
+		p.nextToken()
+	}
+
+	// Semicolon is optional
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+	return cmd, nil
+}