zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-26 10:23:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go CLI: refactor commands (#16447)

Browse Source 
### Summary

1. Move debug commands to dev file.
2. Refactor some commands syntax

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-06-29 17:03:26 +08:00
committed by GitHub
parent ec5cd6b1c0
commit 3202ec6abf
12 changed files with 999 additions and 1022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/cli/admin_parser.go
View File

@@ -1433,14 +1433,14 @@ func (p *Parser) parseAdminAlterProvider() (*Command, error) {
}
p.nextToken()
newModelName := ""
newInstanceName := ""
newAPIKey := ""
optionsLoop:
for {
switch p.curToken.Type {
case TokenName:
p.nextToken()
newModelName, err = p.parseQuotedString()
newInstanceName, err = p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected model name: %w", err)
}
@@ -1457,14 +1457,14 @@ optionsLoop:
}
}
if newModelName == "" && newAPIKey == "" {
if newInstanceName == "" && newAPIKey == "" {
return nil, fmt.Errorf("expected NAME or KEY after INSTANCE")
}
cmd := NewCommand("admin_alter_provider_instance")
cmd.Params["provider_name"] = providerName
cmd.Params["instance_name"] = instanceName
cmd.Params["new_model_name"] = newModelName
cmd.Params["new_instance_name"] = newInstanceName
cmd.Params["new_api_key"] = newAPIKey
p.nextToken()
@@ -2100,9 +2100,9 @@ func (p *Parser) parseAdminDeleteCommands() (*Command, error) {
p.nextToken() // consume DELETE
switch p.curToken.Type {
case TokenAPI:
return p.parseDeleteAPIServer()
return p.parseAPIDeleteAPIServer()
case TokenAdmin:
return p.parseDeleteAdminServer()
return p.parseAPIDeleteAdminServer()
case TokenProvider:
return p.parseAdminDeleteProvider()
default:

2
internal/cli/cli.go
View File

@@ -736,7 +736,7 @@ func (c *CLI) Run() error {
// the command handlers. Don't echo that back to the operator
// verbatim — log the full error server-side for debugging, and
// show only the error type/message via a sanitized wrapper.
fmt.Printf("CLI error: %s\n", sanitizeCLIError(err))
fmt.Printf("ragflow-cli error: %s\n", sanitizeCLIError(err))
}
}

94
internal/cli/cli_http.go
View File

@@ -136,7 +136,7 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) {
case "admin_list_providers":
return c.AdminListProvidersCommand(cmd)
case "admin_list_all_models":
return c.ListAllModels(cmd)
return c.CommonListAllModels(cmd)
case "list_admin_tasks":
return c.ListAdminTasks(cmd)
case "admin_list_ingestors":
@@ -293,7 +293,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.Logout()
case "api_ping_server":
return c.PingByCommand(cmd)
// Configuration commands
case "api_list_configs":
return c.ListConfigs(cmd)
case "api_set_log_level":
@@ -314,8 +313,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.APIListSearchesCommand(cmd)
case "api_list_memories":
return c.APIListMemoriesCommand(cmd)
case "list_dataset_documents":
return c.ListDatasetDocumentUserCommand(cmd)
case "search_on_datasets":
return c.SearchOnDatasets(cmd)
case "search_help":
@@ -371,17 +368,20 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.CommonListInstanceModelsSyncCommand(cmd)
case "api_list_provider_instance_tasks":
return c.APIListModelInstanceTasksCommand(cmd)
// Provider commands
case "api_show_model":
return c.CommonShowModelCommand(cmd)
case "list_all_models":
return c.ListAllModels(cmd)
// Provider commands
case "add_provider":
return c.AddProvider(cmd)
case "api_list_all_models":
return c.CommonListAllModels(cmd)
case "api_add_provider":
return c.APIAddProviderCommand(cmd)
case "api_list_providers":
return c.APIListProviders(cmd)
case "delete_provider":
return c.DeleteProvider(cmd)
case "api_delete_provider":
return c.APIDeleteProvider(cmd)
case "api_delete_provider_instance":
return c.APIDeleteProviderInstanceCommand(cmd)
case "api_drop_dataset":
return c.APIDropDatasetCommand(cmd)
case "api_drop_chat":
@@ -392,22 +392,19 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.APIDropMemoryCommand(cmd)
case "api_drop_agent":
return c.APIDropAgentCommand(cmd)
// Provider instance commands
case "api_create_provider_instance":
return c.APICreateProviderInstanceCommand(cmd)
case "api_add_provider_instance":
return c.APIAddProviderInstanceCommand(cmd)
case "api_list_provider_instances":
return c.CommonListProviderInstances(cmd)
case "alter_provider_instance":
case "api_alter_provider_instance":
return c.CommonAlterProviderInstanceCommand(cmd)
case "drop_provider_instance":
return c.DropProviderInstance(cmd)
case "drop_instance_model":
return c.DropInstanceModel(cmd)
case "api_delete_provider_instance_model":
return c.APIDeleteProviderInstanceModelCommand(cmd)
case "enable_model":
return c.CommonEnableOrDisableModel(cmd, "enable")
case "disable_model":
return c.CommonEnableOrDisableModel(cmd, "disable")
case "add_custom_model":
case "api_add_custom_model":
return c.AddCustomModel(cmd)
case "chat_to_model":
return c.ChatToModel(cmd)
@@ -446,32 +443,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ResetDefaultModel(cmd)
case "api_list_default_models":
return c.ListDefaultModels(cmd)
case "create_chunk_store":
return c.CreateChunkStore(cmd)
case "drop_chunk_store":
return c.DropChunkStore(cmd)
case "create_metadata_store":
return c.CreateMetadataStore(cmd)
case "drop_metadata_store":
return c.DropMetadataStore(cmd)
case "insert_chunks_from_file":
return c.InsertChunksFromFile(cmd)
case "insert_metadata_from_file":
return c.InsertMetadataFromFile(cmd)
case "update_chunk":
return c.UpdateChunk(cmd)
case "get_chunk":
return c.GetChunk(cmd)
case "set_meta":
return c.SetMeta(cmd)
case "delete_meta":
return c.DeleteMeta(cmd)
case "rm_tags":
return c.RmTags(cmd)
case "remove_chunks":
return c.RemoveChunks(cmd)
case "get_metadata":
return c.GetMetadata(cmd)
case "parse_documents_user_command":
return c.ParseDocumentsUserCommand(cmd)
case "user_start_ingestion_command":
@@ -482,7 +453,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ListUserIngestionTasks(cmd)
case "user_remove_task_command":
return c.UserRemoveTaskCommand(cmd)
// TODO: Implement other commands
case "user_parse_local_file_command":
return c.UserParseLocalFile(cmd)
case "api_show_admin_server":
@@ -509,8 +479,38 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ChunkCommand(cmd)
case "api_save_config_command":
return c.CommonSaveServerConfigCommand(cmd)
// File system commands
case "file_system_command":
return c.ExecuteFilesystemCommand(cmd)
// For debug
case "dev_create_chunk_store":
return c.DevCreateChunkStoreCommand(cmd)
case "dev_drop_chunk_store":
return c.DevDropChunkStoreCommand(cmd)
case "dev_create_metadata_store":
return c.DevCreateMetadataStoreCommand(cmd)
case "dev_drop_metadata_store":
return c.DevDropMetadataStoreCommand(cmd)
case "dev_insert_chunks_from_file":
return c.DevInsertChunksFromFileCommand(cmd)
case "dev_insert_metadata_from_file":
return c.DevInsertMetadataFromFileCommand(cmd)
case "dev_update_chunk":
return c.DevUpdateChunkCommand(cmd)
case "dev_get_chunk":
return c.DevGetChunkCommand(cmd)
case "dev_set_meta":
return c.DevSetMetaCommand(cmd)
case "dev_delete_meta":
return c.DevDeleteMetaCommand(cmd)
case "dev_rm_tags":
return c.DevRmTagsCommand(cmd)
case "dev_remove_chunks":
return c.DevRemoveChunksCommand(cmd)
case "dev_get_metadata":
return c.DevGetMetadataCommand(cmd)
default:
return nil, fmt.Errorf("command '%s' would be executed with API", cmd.Type)
}

6
internal/cli/common_command.go
View File

@@ -799,9 +799,9 @@ func (c *CLI) CommonAlterProviderInstanceCommand(cmd *Command) (ResponseIf, erro
payload := map[string]interface{}{}
newName, ok := cmd.Params["new_model_name"].(string)
newName, ok := cmd.Params["new_instance_name"].(string)
if ok {
payload["model_name"] = newName
payload["instance_name"] = newName
}
newAPIKey, ok := cmd.Params["new_api_key"].(string)
@@ -1387,7 +1387,7 @@ func (c *CLI) GetAPIServerInfo(serverName string) (ResponseIf, error) {
return &result, nil
}
func (c *CLI) ListAllModels(cmd *Command) (ResponseIf, error) {
func (c *CLI) CommonListAllModels(cmd *Command) (ResponseIf, error) {
page, ok := cmd.Params["page"].(int)
if !ok {

659
internal/cli/dev_parser.go Normal file
View File

@@ -0,0 +1,659 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package cli
import (
"fmt"
"strconv"
)
// INSERT CHUNKS FROM FILE "file_path"
// INSERT METADATA FROM FILE "file_path"
func (p *Parser) parseDevInsertCommand() (*Command, error) {
p.nextToken() // consume INSERT
// Expect CHUNKS or METADATA
if p.curToken.Type == TokenChunks {
return p.parseInsertChunksFromFile()
}
if p.curToken.Type == TokenMetadata {
return p.parseInsertMetadataFromFile()
}
return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value)
}
// Internal CLI for GO
// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path"
func (p *Parser) parseInsertChunksFromFile() (*Command, error) {
p.nextToken() // consume CHUNKS
// Expect FROM
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)
}
p.nextToken()
// Expect FILE
if p.curToken.Type != TokenFile {
return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)
}
p.nextToken()
// Get file path (quoted string)
filePath, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("dev_insert_chunks_from_file")
cmd.Params["file_path"] = filePath
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// Internal CLI for GO
// parseInsertMetadataFromFile parses: INSERT METADATA FROM FILE "file_path"
func (p *Parser) parseInsertMetadataFromFile() (*Command, error) {
p.nextToken() // consume METADATA
// Expect FROM
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)
}
p.nextToken()
// Expect FILE
if p.curToken.Type != TokenFile {
return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)
}
p.nextToken()
// Get file path (quoted string)
filePath, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("dev_insert_metadata_from_file")
cmd.Params["file_path"] = filePath
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseGetCommand parses: GET CHUNK or GET METADATA
func (p *Parser) parseGetCommand() (*Command, error) {
p.nextToken() // consume GET
if p.curToken.Type == TokenChunk {
return p.parseGetChunk()
}
if p.curToken.Type == TokenMetadata {
return p.parseGetMetadata()
}
return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value)
}
// parseGetChunk parses: GET CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id'
func (p *Parser) parseGetChunk() (*Command, error) {
p.nextToken() // consume CHUNK
// Parse chunk_id
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id: %w", err)
}
cmd := NewCommand("dev_get_chunk")
cmd.Params["chunk_id"] = chunkID
p.nextToken()
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF after chunk_id")
}
p.nextToken()
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
// Parse dataset_id
datasetID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_id: %w", err)
}
cmd.Params["dataset_id"] = datasetID
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// Internal
// parseDevUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
func (p *Parser) parseDevUpdateCommand() (*Command, error) {
p.nextToken() // consume UPDATE
if p.curToken.Type == TokenChunk {
return p.parseUpdateChunk()
}
return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value)
}
// Internal CLI for GO
// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' SET '{"content": "..."}'
func (p *Parser) parseUpdateChunk() (*Command, error) {
p.nextToken() // consume CHUNK
// Parse chunk_id
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id: %w", err)
}
cmd := NewCommand("dev_update_chunk")
cmd.Params["chunk_id"] = chunkID
p.nextToken()
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF after chunk_id")
}
p.nextToken()
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
// Parse dataset_name
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_name: %w", err)
}
cmd.Params["dataset_name"] = datasetName
p.nextToken()
if p.curToken.Type != TokenSet {
return nil, fmt.Errorf("expected SET after dataset_name")
}
p.nextToken()
// Parse JSON body
jsonBody, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected JSON body: %w", err)
}
cmd.Params["json_body"] = jsonBody
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}'
func (p *Parser) parseSetMeta() (*Command, error) {
p.nextToken() // consume METADATA
// Expect OF
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF after SET METADATA")
}
p.nextToken()
// Expect DOCUMENT
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after SET METADATA OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd := NewCommand("dev_set_meta")
cmd.Params["doc_id"] = docID
p.nextToken()
// Expect TO
if p.curToken.Type != TokenTo {
return nil, fmt.Errorf("expected TO after doc_id")
}
p.nextToken()
// Parse meta JSON
meta, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected meta JSON: %w", err)
}
cmd.Params["meta"] = meta
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseDevDeleteMeta parses: DELETE METADATA OF DOCUMENT 'doc_id' [KEYS '["key1", "key2"]']
// If KEYS is not provided, deletes entire document metadata
func (p *Parser) parseDevDeleteMeta() (*Command, error) {
p.nextToken() // consume METADATA
// Expect OF
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF after DELETE METADATA")
}
p.nextToken()
// Expect DOCUMENT
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after DELETE METADATA OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd := NewCommand("dev_delete_meta")
cmd.Params["doc_id"] = docID
p.nextToken()
// KEYS is optional - if not provided, delete entire document metadata
if p.curToken.Type != TokenKeys {
if p.curToken.Type == TokenSemicolon {
p.nextToken()
return cmd, nil
}
if p.curToken.Type == TokenEOF {
return cmd, nil
}
return nil, fmt.Errorf("expected KEYS or end of command after doc_id")
}
// Parse keys JSON array
p.nextToken()
keys, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected keys JSON array: %w", err)
}
cmd.Params["keys"] = keys
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
return cmd, nil
}
if p.curToken.Type != TokenEOF {
return nil, fmt.Errorf("expected end of command after KEYS")
}
return cmd, nil
}
// parseDevRemoveTags parses: REMOVE TAGS 'tag1', 'tag2' from DATASET 'dataset_name';
func (p *Parser) parseDevRemoveTags() (*Command, error) {
p.nextToken() // consume TAGS
// Parse first tag
tag, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected tag: %w", err)
}
tags := []string{tag}
// Parse additional tags separated by commas
for {
p.nextToken()
if p.curToken.Type == TokenComma {
p.nextToken()
tag, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected tag after comma: %w", err)
}
tags = append(tags, tag)
} else {
break
}
}
cmd := NewCommand("dev_rm_tags")
cmd.Params["tags"] = tags
// Expect from
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM after tags")
}
p.nextToken()
// Expect DATASET
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after FROM")
}
p.nextToken()
// Parse dataset_name
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_name: %w", err)
}
cmd.Params["dataset_name"] = datasetName
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// parseDevRemoveChunk parses:
// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
func (p *Parser) parseDevRemoveChunk() (*Command, error) {
cmd := NewCommand("dev_remove_chunks")
// Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL
if p.curToken.Type == TokenAll {
p.nextToken() // consume ALL
if p.curToken.Type != TokenChunks {
return nil, fmt.Errorf("expected CHUNKS after ALL")
}
p.nextToken() // consume CHUNKS
cmd.Params["delete_all"] = true
} else {
// curToken is TokenChunks, consume it first
p.nextToken()
// Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name' (space-separated)
// Parse first chunk ID
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id: %w", err)
}
chunkIDs := []string{chunkID}
// Parse additional chunk IDs separated by spaces (each quoted)
for {
p.nextToken()
// Stop if we hit FROM or non-quoted token
if p.curToken.Type == TokenFrom || p.curToken.Type != TokenQuotedString {
break
}
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id: %w", err)
}
chunkIDs = append(chunkIDs, chunkID)
}
cmd.Params["chunk_ids"] = chunkIDs
}
// Expect FROM
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM after chunk(s)")
}
p.nextToken()
// Expect DOCUMENT
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after FROM")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
// Expect IN
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
// Expect DATASET
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
// Parse dataset_name (quoted string)
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_name: %w", err)
}
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
// Internal CLI for GO
// parseDevDropChunkStore parses: DROP CHUNK STORE for Dataset 'name'
func (p *Parser) parseDevDropChunkStore() (*Command, error) {
p.nextToken() // consume CHUNK STORE
// Expect FOR
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
}
p.nextToken()
// Expect Dataset
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("dev_drop_chunk_store")
cmd.Params["dataset_name"] = datasetName
return cmd, nil
}
// parseDevDropMetadataStore parses: DROP METADATA STORE
func (p *Parser) parseDevDropMetadataStore() (*Command, error) {
// DROP METADATA STORE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenStore {
return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("dev_drop_metadata_store")
return cmd, nil
}
// Internal CLI for GO
// parseDevCreateChunkStore parses: CREATE CHUNK STORE for Dataset 'name' VECTOR SIZE N
func (p *Parser) parseDevCreateChunkStore() (*Command, error) {
p.nextToken() // consume CHUNK STORE compound token
// Expect FOR
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
}
p.nextToken()
// Expect Dataset
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenVector {
return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenSize {
return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenInteger {
return nil, fmt.Errorf("expected vector size number, got %s", p.curToken.Value)
}
vectorSize, err := strconv.Atoi(p.curToken.Value)
if err != nil {
return nil, fmt.Errorf("invalid vector size: %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("dev_create_chunk_store")
cmd.Params["dataset_name"] = datasetName
cmd.Params["vector_size"] = vectorSize
return cmd, nil
}
// Internal CLI for GO
// parseCreateMetadataStore parses: CREATE METADATA STORE
func (p *Parser) parseCreateMetadataStore() (*Command, error) {
// CREATE METADATA STORE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenStore {
return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("dev_create_metadata_store"), nil
}
func (p *Parser) parseCreateRole() (*Command, error) {
p.nextToken() // consume ROLE
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("create_role")
cmd.Params["role_name"] = roleName
p.nextToken()
if p.curToken.Type == TokenDescription {
p.nextToken()
description, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["description"] = description
p.nextToken()
}
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}

13
internal/cli/parser.go
View File

@@ -157,11 +157,11 @@ func (p *Parser) parseUserCommand() (*Command, error) {
case TokenDrop:
return p.parseAPIDropCommands()
case TokenAdd:
return p.parseAddCommand()
return p.parseAPIAddCommands()
case TokenDelete:
return p.parseDeleteCommand()
return p.parseAPIDeleteCommands()
case TokenAlter:
return p.parseAlterCommand()
return p.parseAPIAlterCommands()
case TokenSet:
return p.parseAPISetCommands()
case TokenReset:
@@ -169,7 +169,8 @@ func (p *Parser) parseUserCommand() (*Command, error) {
case TokenImport:
return p.parseImportCommand()
case TokenInsert:
return p.parseInsertCommand()
// Use for development only
return p.parseDevInsertCommand()
case TokenRetrieve:
return p.parseRetrieveCommand()
case TokenParse:
@@ -211,9 +212,9 @@ func (p *Parser) parseUserCommand() (*Command, error) {
case TokenUse:
return p.parseAPIUseCommands()
case TokenUpdate:
return p.parseUpdateCommand()
return p.parseDevUpdateCommand()
case TokenRemove:
return p.parseRemoveCommand()
return p.parseAPIRemoveCommands()
case TokenGet:
return p.parseGetCommand()
case TokenExplain:

96
internal/cli/user_command.go
View File

@@ -739,8 +739,8 @@ func (c *CLI) getDatasetID(datasetName string) (string, error) {
return "", fmt.Errorf("dataset '%s' not found", datasetName)
}
// GetMetadata gets metadata for one or more datasets
func (c *CLI) GetMetadata(cmd *Command) (ResponseIf, error) {
// DevGetMetadataCommand gets metadata for one or more datasets
func (c *CLI) DevGetMetadataCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1465,8 +1465,8 @@ func (c *CLI) APIUnsetAPIKeyCommand(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// CreateChunkStore creates a chunk store in doc engine
func (c *CLI) CreateChunkStore(cmd *Command) (ResponseIf, error) {
// DevCreateChunkStoreCommand creates a chunk store in doc engine
func (c *CLI) DevCreateChunkStoreCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1522,8 +1522,8 @@ func (c *CLI) CreateChunkStore(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DropChunkStore drops a chunk store in doc engine
func (c *CLI) DropChunkStore(cmd *Command) (ResponseIf, error) {
// DevDropChunkStoreCommand drops a chunk store in doc engine
func (c *CLI) DevDropChunkStoreCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1573,8 +1573,8 @@ func (c *CLI) DropChunkStore(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// CreateMetadataStore creates the document metadata store for the tenant
func (c *CLI) CreateMetadataStore(cmd *Command) (ResponseIf, error) {
// DevCreateMetadataStoreCommand creates the document metadata store for the tenant
func (c *CLI) DevCreateMetadataStoreCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1609,8 +1609,8 @@ func (c *CLI) CreateMetadataStore(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DropMetadataStore drops the document metadata store for the tenant
func (c *CLI) DropMetadataStore(cmd *Command) (ResponseIf, error) {
// DevDropMetadataStoreCommand drops the document metadata store for the tenant
func (c *CLI) DevDropMetadataStoreCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1645,13 +1645,18 @@ func (c *CLI) DropMetadataStore(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// AddProvider creates a new model provider
// APIAddProviderCommand creates a new model provider
// ADD PROVIDER <name>
func (c *CLI) AddProvider(cmd *Command) (ResponseIf, error) {
func (c *CLI) APIAddProviderCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
httpClient := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer]
if httpClient.APIKey == nil && httpClient.LoginToken == nil {
return nil, fmt.Errorf("API key not set. Please login first")
}
providerName, ok := cmd.Params["provider_name"].(string)
if !ok {
return nil, fmt.Errorf("provider name not provided")
@@ -1662,7 +1667,7 @@ func (c *CLI) AddProvider(cmd *Command) (ResponseIf, error) {
"provider_name": providerName,
}
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("PUT", "/providers", "web", nil, payload)
resp, err := httpClient.Request("PUT", "/providers", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to add provider: %w", err)
}
@@ -1702,9 +1707,9 @@ func (c *CLI) APIListProviders(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DeleteProvider deletes a provider
// APIDeleteProvider deletes a provider
// DELETE PROVIDER <name>
func (c *CLI) DeleteProvider(cmd *Command) (ResponseIf, error) {
func (c *CLI) APIDeleteProvider(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1721,12 +1726,7 @@ func (c *CLI) DeleteProvider(cmd *Command) (ResponseIf, error) {
url := fmt.Sprintf("/providers/%s", providerName)
// Build payload
payload := map[string]interface{}{
"llm_factory": providerName,
}
resp, err := httpClient.Request("DELETE", url, "web", nil, payload)
resp, err := httpClient.Request("DELETE", url, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to delete provider: %w", err)
}
@@ -1927,9 +1927,9 @@ func (c *CLI) APIDropMemoryCommand(cmd *Command) (ResponseIf, error) {
return HandleSimpleResponse(resp, "delete memory")
}
// APICreateProviderInstanceCommand creates a new provider instance
// APIAddProviderInstanceCommand creates a new provider instance
// CREATE PROVIDER <name> INSTANCE <instance_name> KEY <api_key> URL <base_url> REGION <region>
func (c *CLI) APICreateProviderInstanceCommand(cmd *Command) (ResponseIf, error) {
func (c *CLI) APIAddProviderInstanceCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -2026,9 +2026,8 @@ func (c *CLI) ShowInstanceBalance(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DropProviderInstance deletes a provider instance
// DROP INSTANCE <name> FROM PROVIDER <name>
func (c *CLI) DropProviderInstance(cmd *Command) (ResponseIf, error) {
// DELETE PROVIDER <name> INSTANCE <name>
func (c *CLI) APIDeleteProviderInstanceCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -2061,9 +2060,8 @@ func (c *CLI) DropProviderInstance(cmd *Command) (ResponseIf, error) {
return HandleSimpleResponse(resp, "drop instance")
}
// DROP MODEL <name1 name2 name3> FROM <provider_name> <instance_name>
// Remove MODEL <name1 name2 name3> FROM <provider_name> <instance_name>
func (c *CLI) DropInstanceModel(cmd *Command) (ResponseIf, error) {
// DELETE PROVIDER <name> INSTANCE <instance_name> MODELS <name1 name2 name3>
func (c *CLI) APIDeleteProviderInstanceModelCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3229,8 +3227,8 @@ func (c *CLI) AddCustomModel(cmd *Command) (ResponseIf, error) {
return HandleSimpleResponse(resp, "add custom model")
}
// InsertChunksFromFile inserts chunks from a JSON file
func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) {
// DevInsertChunksFromFileCommand inserts chunks from a JSON file
func (c *CLI) DevInsertChunksFromFileCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3243,7 +3241,7 @@ func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) {
"file_path": filePath,
}
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/insert_chunks_from_file", "web", nil, payload)
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_chunks_from_file", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to insert dataset from file: %w", err)
}
@@ -3273,8 +3271,8 @@ func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// InsertMetadataFromFile inserts metadata from a JSON file
func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) {
// DevInsertMetadataFromFileCommand inserts metadata from a JSON file
func (c *CLI) DevInsertMetadataFromFileCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3288,7 +3286,7 @@ func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) {
"file_path": filePath,
}
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/insert_metadata_from_file", "web", nil, payload)
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_metadata_from_file", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to insert metadata from file: %w", err)
}
@@ -3318,8 +3316,8 @@ func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// UpdateChunk updates a chunk in a dataset
func (c *CLI) UpdateChunk(cmd *Command) (ResponseIf, error) {
// DevUpdateChunkCommand updates a chunk in a dataset
func (c *CLI) DevUpdateChunkCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3391,8 +3389,8 @@ func (c *CLI) UpdateChunk(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// GetChunk retrieves a chunk by ID
func (c *CLI) GetChunk(cmd *Command) (ResponseIf, error) {
// DevGetChunkCommand retrieves a chunk by ID
func (c *CLI) DevGetChunkCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3434,8 +3432,8 @@ func (c *CLI) GetChunk(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// SetMeta sets metadata for a document
func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) {
// DevSetMetaCommand sets metadata for a document
func (c *CLI) DevSetMetaCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3455,7 +3453,7 @@ func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) {
"meta": metaJSON,
}
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/set_meta", "web", nil, payload)
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/dev_set_meta", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to set metadata: %w", err)
}
@@ -3485,9 +3483,9 @@ func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DeleteMeta deletes metadata for a document
// DevDeleteMetaCommand deletes metadata for a document
// If keys is provided, deletes specific keys; otherwise deletes entire document metadata
func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) {
func (c *CLI) DevDeleteMetaCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3506,7 +3504,7 @@ func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) {
payload["keys"] = keysJSON
}
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/delete_meta", "web", nil, payload)
resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/dev_delete_meta", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to delete metadata: %w", err)
}
@@ -3536,8 +3534,8 @@ func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// RmTags removes tags from chunks in a dataset
func (c *CLI) RmTags(cmd *Command) (ResponseIf, error) {
// DevRmTagsCommand removes tags from chunks in a dataset
func (c *CLI) DevRmTagsCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -3591,8 +3589,8 @@ func (c *CLI) RmTags(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// RemoveChunks removes chunks from a document
func (c *CLI) RemoveChunks(cmd *Command) (ResponseIf, error) {
// DevRemoveChunksCommand removes chunks from a document
func (c *CLI) DevRemoveChunksCommand(cmd *Command) (ResponseIf, error) {
if c.Config.CLIMode != APIMode {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}

1059
internal/cli/user_parser.go
View File

File diff suppressed because it is too large Load Diff

4
internal/cli/user_parser_test.go
View File

@@ -40,9 +40,9 @@ func TestParseAddModelWithDimensions(t *testing.T) {
}{
{
name: "Add model with detailed embedding dimensions",
input: "add model 'x1 x2 x3 x4 x5' to provider 'vllm' instance 'test' with tokens 1024 chat think vision, token 2048 chat, token 1024 think vision, token 0 embedding 2048 64 1024 2048, token 0 embedding 2048;",
input: "add model 'x1 x2 x3 x4 x5' to provider 'vllm' instance 'test' tokens 1024 chat think vision, token 2048 chat, token 1024 think vision, token 0 embedding 2048 64 1024 2048, token 0 embedding 2048;",
expected: &Command{
Type: "add_custom_model",
Type: "api_add_custom_model",
Params: map[string]interface{}{
"provider_name": "vllm",
"instance_name": "test",