From 7c1edca15e4f66503ff062701d71ad591c88a51a Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Mon, 29 Jun 2026 19:09:32 +0800 Subject: [PATCH] Go CLI: fix api commands (#16457) ### Summary As title. Signed-off-by: Jin Hai --- internal/cli/cli_http.go | 118 +- internal/cli/common_command.go | 26 +- internal/cli/dev_parser.go | 104 +- internal/cli/parser.go | 53 +- internal/cli/user_command.go | 42 +- internal/cli/user_parser.go | 1977 +++++++++++++------------- internal/cli/user_parser_test.go | 4 +- internal/engine/infinity/metadata.go | 2 +- 8 files changed, 1178 insertions(+), 1148 deletions(-) diff --git a/internal/cli/cli_http.go b/internal/cli/cli_http.go index 43e6b94a1d..de3debb22d 100644 --- a/internal/cli/cli_http.go +++ b/internal/cli/cli_http.go @@ -130,7 +130,7 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) { case "admin_list_provider_instance_models": return c.CommonListInstanceModelsCommand(cmd) case "admin_list_provider_instances": - return c.CommonListProviderInstances(cmd) + return c.CommonListProviderInstancesCommand(cmd) case "admin_show_model": return c.CommonShowModelCommand(cmd) case "admin_list_providers": @@ -160,9 +160,9 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) { case "admin_check_license": return c.AdminCheckLicenseCommand(cmd) case "admin_check_provider_with_key": - return c.CommonCheckProviderWithKey(cmd) + return c.CommonCheckProviderWithKeyCommand(cmd) case "admin_check_provider_instance": - return c.CommonCheckProviderConnection(cmd) + return c.CommonCheckProviderConnectionCommand(cmd) case "admin_show_fingerprint": return c.AdminShowFingerprintCommand(cmd) case "admin_show_license": @@ -254,9 +254,9 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) { case "admin_delete_model": return c.AdminDeleteModelsCommand(cmd) case "admin_enable_model": - return c.CommonEnableOrDisableModel(cmd, "enable") + return c.CommonEnableOrDisableModelCommand(cmd, "enable") case "admin_disable_model": - return c.CommonEnableOrDisableModel(cmd, "disable") + return c.CommonEnableOrDisableModelCommand(cmd, "disable") case "admin_show_admin_server": return c.CommonShowAdminServerCommand(cmd) case "admin_show_api_server": @@ -264,14 +264,14 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) { case "admin_show_log_level": return c.AdminShowLogLevelCommand(cmd) case "admin_list_api_servers": - return c.CommonListAPIServers(cmd) - case "add_api_server": - return c.AddAPIServer(cmd) - case "delete_api_server": - return c.DeleteAPIServer(cmd) - case "add_admin_server": + return c.CommonListAPIServersCommand(cmd) + case "api_add_api_server": + return c.AddAPIServerCommand(cmd) + case "api_delete_api_server": + return c.DeleteAPIServerCommand(cmd) + case "api_add_admin_server": return nil, fmt.Errorf("cannot add admin server in admin mode") - case "delete_admin_server": + case "api_delete_admin_server": return nil, fmt.Errorf("cannot delete admin server in admin mode") case "admin_save_config_command": return c.CommonSaveServerConfigCommand(cmd) @@ -377,9 +377,9 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { case "api_add_provider": return c.APIAddProviderCommand(cmd) case "api_list_providers": - return c.APIListProviders(cmd) + return c.APIListProvidersCommand(cmd) case "api_delete_provider": - return c.APIDeleteProvider(cmd) + return c.APIDeleteProviderCommand(cmd) case "api_delete_provider_instance": return c.APIDeleteProviderInstanceCommand(cmd) case "api_drop_dataset": @@ -395,66 +395,64 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { case "api_add_provider_instance": return c.APIAddProviderInstanceCommand(cmd) case "api_list_provider_instances": - return c.CommonListProviderInstances(cmd) + return c.CommonListProviderInstancesCommand(cmd) case "api_alter_provider_instance": return c.CommonAlterProviderInstanceCommand(cmd) case "api_delete_provider_instance_model": return c.APIDeleteProviderInstanceModelCommand(cmd) case "enable_model": - return c.CommonEnableOrDisableModel(cmd, "enable") + return c.CommonEnableOrDisableModelCommand(cmd, "enable") case "disable_model": - return c.CommonEnableOrDisableModel(cmd, "disable") + return c.CommonEnableOrDisableModelCommand(cmd, "disable") case "api_add_custom_model": - return c.AddCustomModel(cmd) - case "chat_to_model": - return c.ChatToModel(cmd) - case "think_chat_to_model": - return c.ChatToModel(cmd) - case "openai_chat": - return c.OpenaiChat(cmd) + return c.APIAddCustomModelCommand(cmd) + case "api_chat_to_model": + return c.APIChatToModelCommand(cmd) + case "api_openai_chat": + return c.APIOpenaiChatCommand(cmd) case "openai_chat_help": printOpenaiChatHelp() return nil, nil - case "embed_user_text": - return c.EmbedUserText(cmd) - case "rarank_user_document": - return c.RerankUserDocument(cmd) + case "api_embed_user_text": + return c.EmbedUserTextCommand(cmd) + case "api_rarank_user_document": + return c.APIRerankUserDocumentCommand(cmd) case "tts_user_command": - return c.TTSUserCommand(cmd) + return c.APITTSUserCommand(cmd) case "asr_user_command": - return c.ASRUserCommand(cmd) + return c.APIASRUserCommand(cmd) case "ocr_user_command": - return c.OCRUserCommand(cmd) - case "parse_file_user_command": - return c.ParseFileUserCommand(cmd) + return c.APIOCRUserCommand(cmd) + case "api_model_parse_file": + return c.APIModelParseFileCommand(cmd) case "check_provider_connection": - return c.CommonCheckProviderConnection(cmd) + return c.CommonCheckProviderConnectionCommand(cmd) case "check_provider_with_key": - return c.CommonCheckProviderWithKey(cmd) + return c.CommonCheckProviderWithKeyCommand(cmd) case "api_use_model": return c.APIUseModelCommand(cmd) case "api_use_api_server": return c.CommonUseAPIServerCommand(cmd) case "api_use_admin_server": return c.CommonUseAdminServerCommand(cmd) - case "set_default_model": - return c.SetDefaultModel(cmd) + case "api_set_default_model": + return c.APISetDefaultModelCommand(cmd) case "api_reset_default_model": - return c.ResetDefaultModel(cmd) + return c.APIResetDefaultModelCommand(cmd) case "api_list_default_models": - return c.ListDefaultModels(cmd) - case "parse_documents_user_command": - return c.ParseDocumentsUserCommand(cmd) - case "user_start_ingestion_command": - return c.UserStartIngestionCommand(cmd) - case "user_stop_ingestion_command": - return c.UserStopIngestionCommand(cmd) + return c.APIListDefaultModelsCommand(cmd) + case "api_parse_documents": + return c.APIParseDocumentsUserCommand(cmd) + case "api_start_ingestion": + return c.APIStartIngestionCommand(cmd) + case "api_stop_ingestion": + return c.APIStopIngestionCommand(cmd) case "api_list_ingestion_tasks": - return c.ListUserIngestionTasks(cmd) - case "user_remove_task_command": - return c.UserRemoveTaskCommand(cmd) + return c.APIListIngestionTasks(cmd) + case "api_remove_task": + return c.APIRemoveTaskCommand(cmd) case "user_parse_local_file_command": - return c.UserParseLocalFile(cmd) + return c.APIParseLocalFileCommand(cmd) case "api_show_admin_server": return c.CommonShowAdminServerCommand(cmd) case "api_show_api_server": @@ -462,21 +460,19 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { case "api_show_log_level": return c.APIShowLogLevelCommand(cmd) case "api_list_api_servers": - return c.CommonListAPIServers(cmd) + return c.CommonListAPIServersCommand(cmd) case "api_list_environments": return c.APIListEnvironmentsCommand(cmd) case "api_list_variables": return c.APIListVariablesCommand(cmd) - case "add_api_server": - return c.AddAPIServer(cmd) - case "delete_api_server": - return c.DeleteAPIServer(cmd) - case "add_admin_server": - return c.AddAdminServer(cmd) - case "delete_admin_server": - return c.DeleteAdminServer(cmd) - case "user_chunk_command": - return c.ChunkCommand(cmd) + case "api_add_api_server": + return c.AddAPIServerCommand(cmd) + case "api_delete_api_server": + return c.DeleteAPIServerCommand(cmd) + case "api_add_admin_server": + return c.AddAdminServerCommand(cmd) + case "api_delete_admin_server": + return c.DeleteAdminServerCommand(cmd) case "api_save_config_command": return c.CommonSaveServerConfigCommand(cmd) @@ -485,6 +481,8 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.ExecuteFilesystemCommand(cmd) // For debug + case "dev_chunk": + return c.DevChunkCommand(cmd) case "dev_create_chunk_store": return c.DevCreateChunkStoreCommand(cmd) case "dev_drop_chunk_store": diff --git a/internal/cli/common_command.go b/internal/cli/common_command.go index 5ff5b38dcd..ac67130bd3 100644 --- a/internal/cli/common_command.go +++ b/internal/cli/common_command.go @@ -429,9 +429,9 @@ func (c *CLI) CommonShowProviderInstanceBalanceCommand(cmd *Command) (ResponseIf return &result, nil } -// CommonListProviderInstances lists all instances of a provider +// CommonListProviderInstancesCommand lists all instances of a provider // LIST INSTANCES FROM PROVIDER -func (c *CLI) CommonListProviderInstances(cmd *Command) (ResponseIf, error) { +func (c *CLI) CommonListProviderInstancesCommand(cmd *Command) (ResponseIf, error) { providerName, ok := cmd.Params["provider_name"].(string) if !ok { @@ -647,7 +647,7 @@ func (c *CLI) CommonShowProviderModelCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) CommonCheckProviderWithKey(cmd *Command) (ResponseIf, error) { +func (c *CLI) CommonCheckProviderWithKeyCommand(cmd *Command) (ResponseIf, error) { providerName, ok := cmd.Params["provider_name"].(string) if !ok || providerName == "" { @@ -725,7 +725,7 @@ func (c *CLI) CommonCheckProviderWithKey(cmd *Command) (ResponseIf, error) { } } -func (c *CLI) CommonCheckProviderConnection(cmd *Command) (ResponseIf, error) { +func (c *CLI) CommonCheckProviderConnectionCommand(cmd *Command) (ResponseIf, error) { instanceName, ok := cmd.Params["instance_name"].(string) if !ok { @@ -857,7 +857,7 @@ func (c *CLI) CommonAlterProviderInstanceCommand(cmd *Command) (ResponseIf, erro } } -func (c *CLI) CommonEnableOrDisableModel(cmd *Command, status string) (ResponseIf, error) { +func (c *CLI) CommonEnableOrDisableModelCommand(cmd *Command, status string) (ResponseIf, error) { modelName, ok := cmd.Params["model_name"].(string) if !ok { @@ -925,7 +925,7 @@ func (c *CLI) CommonEnableOrDisableModel(cmd *Command, status string) (ResponseI } } -func (c *CLI) SetDefaultModel(cmd *Command) (ResponseIf, error) { +func (c *CLI) APISetDefaultModelCommand(cmd *Command) (ResponseIf, error) { modelType, ok := cmd.Params["model_type"].(string) if !ok { @@ -981,7 +981,7 @@ func (c *CLI) SetDefaultModel(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ResetDefaultModel(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIResetDefaultModelCommand(cmd *Command) (ResponseIf, error) { modelType, ok := cmd.Params["model_type"].(string) if !ok { @@ -1023,7 +1023,7 @@ func (c *CLI) ResetDefaultModel(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ListDefaultModels(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIListDefaultModelsCommand(cmd *Command) (ResponseIf, error) { var resp *Response var err error @@ -1117,7 +1117,7 @@ func (c *CLI) CommonShowAPIServerCommand(cmd *Command) (ResponseIf, error) { return result, nil } -func (c *CLI) CommonListAPIServers(cmd *Command) (ResponseIf, error) { +func (c *CLI) CommonListAPIServersCommand(cmd *Command) (ResponseIf, error) { var result CommonResponse result.Data = make([]map[string]interface{}, 0) @@ -1147,7 +1147,7 @@ func (c *CLI) CommonListAPIServers(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) AddAPIServer(cmd *Command) (ResponseIf, error) { +func (c *CLI) AddAPIServerCommand(cmd *Command) (ResponseIf, error) { apiServerName, ok := cmd.Params["server_name"].(string) if !ok { return nil, fmt.Errorf("server name not provided") @@ -1213,7 +1213,7 @@ func (c *CLI) AddAPIServer(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) DeleteAPIServer(cmd *Command) (ResponseIf, error) { +func (c *CLI) DeleteAPIServerCommand(cmd *Command) (ResponseIf, error) { apiServerName, ok := cmd.Params["server_name"].(string) if !ok { return nil, fmt.Errorf("server name not provided") @@ -1235,7 +1235,7 @@ func (c *CLI) DeleteAPIServer(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) AddAdminServer(cmd *Command) (ResponseIf, error) { +func (c *CLI) AddAdminServerCommand(cmd *Command) (ResponseIf, error) { if c.AdminServerClient != nil && c.AdminServerClient.LoginToken != nil { return nil, fmt.Errorf("admin server already login, please logout") @@ -1289,7 +1289,7 @@ func (c *CLI) AddAdminServer(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) DeleteAdminServer(cmd *Command) (ResponseIf, error) { +func (c *CLI) DeleteAdminServerCommand(cmd *Command) (ResponseIf, error) { if c.AdminServerClient == nil && c.Config.AdminClientConfig == nil { return nil, fmt.Errorf("admin server not exists") diff --git a/internal/cli/dev_parser.go b/internal/cli/dev_parser.go index 643800d95e..bcdf3a7b95 100644 --- a/internal/cli/dev_parser.go +++ b/internal/cli/dev_parser.go @@ -104,8 +104,8 @@ func (p *Parser) parseInsertMetadataFromFile() (*Command, error) { return cmd, nil } -// parseGetCommand parses: GET CHUNK or GET METADATA -func (p *Parser) parseGetCommand() (*Command, error) { +// parseDevGetCommand parses: GET CHUNK or GET METADATA +func (p *Parser) parseDevGetCommand() (*Command, error) { p.nextToken() // consume GET if p.curToken.Type == TokenChunk { @@ -260,8 +260,8 @@ func (p *Parser) parseUpdateChunk() (*Command, error) { return cmd, nil } -// parseSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}' -func (p *Parser) parseSetMeta() (*Command, error) { +// parseDevSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}' +func (p *Parser) parseDevSetMeta() (*Command, error) { p.nextToken() // consume METADATA // Expect OF @@ -613,8 +613,8 @@ func (p *Parser) parseDevCreateChunkStore() (*Command, error) { } // Internal CLI for GO -// parseCreateMetadataStore parses: CREATE METADATA STORE -func (p *Parser) parseCreateMetadataStore() (*Command, error) { +// parseDevCreateMetadataStore parses: CREATE METADATA STORE +func (p *Parser) parseDevCreateMetadataStore() (*Command, error) { // CREATE METADATA STORE p.nextToken() // consume METADATA @@ -657,3 +657,95 @@ func (p *Parser) parseCreateRole() (*Command, error) { } return cmd, nil } + +func (p *Parser) parseGetMetadata() (*Command, error) { + p.nextToken() // consume METADATA + + if p.curToken.Type != TokenOf { + return nil, fmt.Errorf("expected OF after METADATA") + } + p.nextToken() + + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected DATASET after OF") + } + p.nextToken() + + // Parse dataset names (space-separated) + var datasetNames []string + for { + name, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset name: %w", err) + } + datasetNames = append(datasetNames, name) + + p.nextToken() + + if p.curToken.Type == TokenComma { + return nil, fmt.Errorf("syntax error: dataset names must be space-separated, not comma-separated (got %q after %q)", "'", name) + } + // Stop at semicolon or non-quoted (dataset name must be quoted) + if p.curToken.Type == TokenSemicolon { + break + } + // If next token is not a quoted string, stop parsing dataset names + if p.curToken.Type != TokenQuotedString { + break + } + } + + cmd := NewCommand("dev_get_metadata") + cmd.Params["dataset_names"] = datasetNames + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +func (p *Parser) parseDevExplain() (*Command, error) { + p.nextToken() // consume EXPLAIN + + switch p.curToken.Type { + case TokenChunk: + return p.parseDevChunk(true) + default: + return nil, fmt.Errorf("expected CHUNK after EXPLAIN") + } +} + +func (p *Parser) parseDevChunk(explain bool) (*Command, error) { + p.nextToken() // consume CHUNK + + filename, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected filename: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenWith { + return nil, fmt.Errorf("expected WITH after filename") + } + p.nextToken() + + dsl, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected DSL: %w", err) + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + p.nextToken() + + cmd := NewCommand("dev_chunk") + cmd.Params["dsl"] = dsl + cmd.Params["filename"] = filename + cmd.Params["explain"] = explain + + return cmd, nil +} diff --git a/internal/cli/parser.go b/internal/cli/parser.go index ae49977b6c..aff4ff100c 100644 --- a/internal/cli/parser.go +++ b/internal/cli/parser.go @@ -148,6 +148,8 @@ func (p *Parser) parseUserCommand() (*Command, error) { return p.parseAPILogout() case TokenPing: return p.parseAPIPingServer() + case TokenRegister: + return p.parseAPIRegister() case TokenList: return p.parseAPIListCommands() case TokenShow: @@ -167,64 +169,61 @@ func (p *Parser) parseUserCommand() (*Command, error) { case TokenReset: return p.parseAPIResetCommands() case TokenImport: - return p.parseImportCommand() + return p.parseAPIImport() case TokenInsert: // Use for development only return p.parseDevInsertCommand() case TokenRetrieve: - return p.parseRetrieveCommand() + return p.parseAPIRetrieve() case TokenParse: - return p.parseParseCommand() + return p.parseParseCommands() case TokenBenchmark: return p.parseBenchmarkCommand() - case TokenRegister: - return p.parseAPIRegisterCommand() case TokenEnable: - return p.parseEnableCommand() + return p.parseAPIEnable() case TokenDisable: - return p.parseDisableCommand() - case TokenStream: - return p.parseStreamCommand() + return p.parseAPIDisable() case TokenChat: - return p.parseChatCommand() - case TokenOpenaiChat: - return p.parseOpenaiChatCommand() + return p.parseAPIChat() + case TokenStream: + return p.parseAPIStream() case TokenThink: - return p.parseThinkCommand() + return p.parseAPIThink() case TokenEmbed: - return p.parseEmbedCommand() + return p.parseAPIEmbed() case TokenRerank: - return p.parseRerankCommand() + return p.parseAPIRerank() case TokenASR: - return p.parseASRCommand() + return p.parseAPIASR() case TokenTTS: - return p.parseTTSCommand() + return p.parseAPITTS() case TokenOCR: - return p.parseOCRCommand() + return p.parseAPIOCR() case TokenCheck: - return p.parseCheckCommand() + return p.parseAPICheck() case TokenStart: return p.parseUserStartIngestion() case TokenStop: return p.parseUserStopIngestion() case TokenSave: - return p.parseAPISaveCommand() + return p.parseAPISave() case TokenUse: return p.parseAPIUseCommands() case TokenUpdate: return p.parseDevUpdateCommand() case TokenRemove: - return p.parseAPIRemoveCommands() + return p.parseAPIRemove() case TokenGet: - return p.parseGetCommand() + return p.parseDevGetCommand() case TokenExplain: - return p.parseExplainCommand() + return p.parseDevExplain() case TokenChunk: - return p.parseChunkCommand(false) - + return p.parseDevChunk(false) + case TokenOpenaiChat: + return p.parseOpenaiChat() case TokenLS, TokenCat, TokenSearch: // For context engine - return p.parseFileSystemCommand() + return p.parseFileSystemCommands() default: return nil, fmt.Errorf("unknown command: %s", p.curToken.Value) } @@ -389,7 +388,7 @@ func tokenTypeToString(t int) string { return fmt.Sprintf("token(%d)", t) } -func (p *Parser) parseFileSystemCommand() (*Command, error) { +func (p *Parser) parseFileSystemCommands() (*Command, error) { p.nextToken() // consume COMMAND cmd := NewCommand("file_system_command") diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go index 773a7793a2..e74039f03c 100644 --- a/internal/cli/user_command.go +++ b/internal/cli/user_command.go @@ -1679,8 +1679,8 @@ func (c *CLI) APIAddProviderCommand(cmd *Command) (ResponseIf, error) { return HandleSimpleResponse(resp, "add provider") } -// APIListProviders lists added providers -func (c *CLI) APIListProviders(cmd *Command) (ResponseIf, error) { +// APIListProvidersCommand lists added providers +func (c *CLI) APIListProvidersCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1707,9 +1707,9 @@ func (c *CLI) APIListProviders(cmd *Command) (ResponseIf, error) { return &result, nil } -// APIDeleteProvider deletes a provider +// APIDeleteProviderCommand deletes a provider // DELETE PROVIDER -func (c *CLI) APIDeleteProvider(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIDeleteProviderCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -2107,7 +2107,7 @@ func isValidURL(str string) bool { return u.Scheme != "" && u.Host != "" } -func (c *CLI) ChatToModel(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIChatToModelCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -2375,7 +2375,7 @@ func (c *CLI) ChatToModel(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) EmbedUserText(cmd *Command) (ResponseIf, error) { +func (c *CLI) EmbedUserTextCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -2458,7 +2458,7 @@ func (c *CLI) EmbedUserText(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) RerankUserDocument(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIRerankUserDocumentCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -2547,7 +2547,7 @@ func (c *CLI) RerankUserDocument(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) TTSUserCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APITTSUserCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -2755,7 +2755,7 @@ func (c *CLI) TTSUserCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ASRUserCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIASRUserCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -2859,7 +2859,7 @@ func (c *CLI) ASRUserCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) OCRUserCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIOCRUserCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -2955,7 +2955,7 @@ func (c *CLI) OCRUserCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ParseFileUserCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIModelParseFileCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3184,7 +3184,7 @@ func (c *CLI) APIUseModelCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) AddCustomModel(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIAddCustomModelCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3659,7 +3659,7 @@ func (c *CLI) DevRemoveChunksCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ParseDocumentsUserCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIParseDocumentsUserCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3697,7 +3697,7 @@ func (c *CLI) ParseDocumentsUserCommand(cmd *Command) (ResponseIf, error) { return HandleSimpleResponse(resp, "list documents") } -func (c *CLI) UserParseLocalFile(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIParseLocalFileCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3779,7 +3779,7 @@ func formatRequestError(action string, err error) error { } } -func (c *CLI) ListUserIngestionTasks(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIListIngestionTasks(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3912,7 +3912,7 @@ func (c *CLI) APIListVariablesCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) UserStartIngestionCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIStartIngestionCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3960,7 +3960,7 @@ func (c *CLI) UserStartIngestionCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) UserStopIngestionCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIStopIngestionCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -3999,7 +3999,7 @@ func (c *CLI) UserStopIngestionCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) UserRemoveTaskCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIRemoveTaskCommand(cmd *Command) (ResponseIf, error) { if c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].APIKey == nil && c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].LoginToken == nil { return nil, fmt.Errorf("API key not set. Please login first") } @@ -4039,7 +4039,7 @@ func (c *CLI) UserRemoveTaskCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -func (c *CLI) ChunkCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) DevChunkCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -4103,10 +4103,10 @@ func (c *CLI) ChunkCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -// OpenaiChat dispatches the parsed OPENAI_CHAT command to either a +// APIOpenaiChatCommand dispatches the parsed OPENAI_CHAT command to either a // non-streaming oneshot call or a streaming SSE call, depending on the // `stream` option. -func (c *CLI) OpenaiChat(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIOpenaiChatCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("OPENAI_CHAT is only allowed in USER mode") } diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go index da3ad6832d..175f00522d 100644 --- a/internal/cli/user_parser.go +++ b/internal/cli/user_parser.go @@ -16,16 +16,6 @@ func tokenTypeDescription(t int, tok Token) string { } // Command parsers -func (p *Parser) parseAPILogout() (*Command, error) { - cmd := NewCommand("api_logout") - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - func (p *Parser) parseAPILoginUser() (*Command, error) { cmd := NewCommand("api_login_user") @@ -62,6 +52,16 @@ func (p *Parser) parseAPILoginUser() (*Command, error) { return cmd, nil } +func (p *Parser) parseAPILogout() (*Command, error) { + cmd := NewCommand("api_logout") + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + func (p *Parser) parseAPIPingServer() (*Command, error) { cmd := NewCommand("api_ping_server") p.nextToken() @@ -72,7 +72,7 @@ func (p *Parser) parseAPIPingServer() (*Command, error) { return cmd, nil } -func (p *Parser) parseAPIRegisterCommand() (*Command, error) { +func (p *Parser) parseAPIRegister() (*Command, error) { cmd := NewCommand("api_register_user") if err := p.expectPeek(TokenUser); err != nil { @@ -119,6 +119,8 @@ func (p *Parser) parseAPIRegisterCommand() (*Command, error) { return cmd, nil } +// region LIST commands + // LIST CONFIGS; // LIST PROVIDER 'provider_name' MODELS; // LIST PROVIDER 'provider_name' INSTANCE 'instance_name' MODELS @@ -297,54 +299,6 @@ func (p *Parser) parseAPIListMemories() (*Command, error) { return NewCommand("api_list_memories"), nil } -func (p *Parser) parseGetMetadata() (*Command, error) { - p.nextToken() // consume METADATA - - if p.curToken.Type != TokenOf { - return nil, fmt.Errorf("expected OF after METADATA") - } - p.nextToken() - - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected DATASET after OF") - } - p.nextToken() - - // Parse dataset names (space-separated) - var datasetNames []string - for { - name, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset name: %w", err) - } - datasetNames = append(datasetNames, name) - - p.nextToken() - - if p.curToken.Type == TokenComma { - return nil, fmt.Errorf("syntax error: dataset names must be space-separated, not comma-separated (got %q after %q)", "'", name) - } - // Stop at semicolon or non-quoted (dataset name must be quoted) - if p.curToken.Type == TokenSemicolon { - break - } - // If next token is not a quoted string, stop parsing dataset names - if p.curToken.Type != TokenQuotedString { - break - } - } - - cmd := NewCommand("dev_get_metadata") - cmd.Params["dataset_names"] = datasetNames - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - func (p *Parser) parseAPIListAPIKeys() (*Command, error) { p.nextToken() // consume KEYS cmd := NewCommand("api_list_api_keys") @@ -466,6 +420,44 @@ func (p *Parser) parseListProviderModels(providerName string) (*Command, error) return cmd, nil } +// LIST MODELS +func (p *Parser) parseAPIListAllModels() (*Command, error) { + p.nextToken() // consume models + + cmd := NewCommand("api_list_all_models") + + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIListIngestionTasks() (*Command, error) { + p.nextToken() // consume Ingestion + + if p.curToken.Type != TokenTasks { + return nil, fmt.Errorf("expected TASKS") + } + p.nextToken() // consume TASKS + + cmd := NewCommand("api_list_ingestion_tasks") + + if p.curToken.Type == TokenFrom { + p.nextToken() + datasetID, err := p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["dataset_id"] = datasetID + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + func (p *Parser) parseAPIListDefaultModels() (*Command, error) { p.nextToken() // consume DEFAULT if p.curToken.Type != TokenModels { @@ -489,6 +481,56 @@ func (p *Parser) parseAPIListAvailableProviders() (*Command, error) { return NewCommand("api_list_available_providers"), nil } +func (p *Parser) parseAPIListAPIServers() (*Command, error) { + p.nextToken() // consume API + + var cmd *Command + switch p.curToken.Type { + case TokenServer: + p.nextToken() + cmd = NewCommand("api_list_api_servers") + default: + return nil, fmt.Errorf("expected SERVER after API") + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +func (p *Parser) parseAPIListEnvironments() (*Command, error) { + p.nextToken() // consume Envs + + cmd := NewCommand("api_list_environments") + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +func (p *Parser) parseAPIListVariables() (*Command, error) { + p.nextToken() // consume Variables + + cmd := NewCommand("api_list_variables") + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// endregion LIST commands + +// region SHOW commands + func (p *Parser) parseAPIShowCommands() (*Command, error) { p.nextToken() // consume SHOW switch p.curToken.Type { @@ -563,24 +605,6 @@ func (p *Parser) parseAPIShowVariable() (*Command, error) { return cmd, nil } -// SHOW MODEL 'model_name'; -func (p *Parser) parseAPIShowModel() (*Command, error) { - p.nextToken() // consume MODEL - - modelName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected model name: %w", err) - } - p.nextToken() // consume model_name - - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - cmd := NewCommand("api_show_model") - cmd.Params["model_name"] = modelName - return cmd, nil -} - // SHOW PROVIDER ; // SHOW PROVIDER INSTANCE ; // SHOW PROVIDER INSTANCE BALANCE; @@ -698,18 +722,101 @@ func (p *Parser) parseAPIShowProviderModel(providerName string) (*Command, error return cmd, nil } -// LIST MODELS -func (p *Parser) parseAPIListAllModels() (*Command, error) { - p.nextToken() // consume models +// SHOW MODEL 'model_name'; +func (p *Parser) parseAPIShowModel() (*Command, error) { + p.nextToken() // consume MODEL - cmd := NewCommand("api_list_all_models") + modelName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected model name: %w", err) + } + p.nextToken() // consume model_name + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + cmd := NewCommand("api_show_model") + cmd.Params["model_name"] = modelName + return cmd, nil +} + +// SHOW ADMIN SERVER +func (p *Parser) parseAPIShowAdmin() (*Command, error) { + p.nextToken() // consume ADMIN + + var cmd *Command + switch p.curToken.Type { + case TokenServer: + p.nextToken() + cmd = NewCommand("api_show_admin_server") + default: + return nil, fmt.Errorf("expected SERVER after ADMIN") + } + + // Semicolon is optional if p.curToken.Type == TokenSemicolon { p.nextToken() } return cmd, nil } +// SHOW API SERVER +func (p *Parser) parseAPIShowAPI() (*Command, error) { + p.nextToken() // consume API + + var cmd *Command + switch p.curToken.Type { + case TokenServer: + p.nextToken() + cmd = NewCommand("api_show_api_server") + + serverName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected API server name: %w", err) + } + cmd.Params["api_server_name"] = serverName + p.nextToken() + + default: + return nil, fmt.Errorf("expected SERVER after API") + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +func (p *Parser) parseAPIShowLogCommands() (*Command, error) { + p.nextToken() // consume LOG + + switch p.curToken.Type { + case TokenLevel: + return p.parseShowLogLevel() + default: + return nil, fmt.Errorf("expected LEVEL after LOG") + } +} + +func (p *Parser) parseShowLogLevel() (*Command, error) { + p.nextToken() // consume LEVEL + + cmd := NewCommand("api_show_log_level") + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// endregion SHOW commands + +// region CREATE commands + func (p *Parser) parseAPICreateCommands() (*Command, error) { p.nextToken() // consume CREATE @@ -729,7 +836,7 @@ func (p *Parser) parseAPICreateCommands() (*Command, error) { case TokenChunkStore: return p.parseDevCreateChunkStore() case TokenMetadata: - return p.parseCreateMetadataStore() + return p.parseDevCreateMetadataStore() case TokenProvider: return p.parseAPICreateProviderInstance() default: @@ -737,6 +844,104 @@ func (p *Parser) parseAPICreateCommands() (*Command, error) { } } +// CREATE DATASET 'abc'; +func (p *Parser) parseAPICreateDataset() (*Command, error) { + p.nextToken() // consume DATASET + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("api_create_dataset") + cmd.Params["dataset_name"] = datasetName + return cmd, nil +} + +// CREATE CHAT 'chat_name' +func (p *Parser) parseAPICreateChat() (*Command, error) { + p.nextToken() // consume CHAT + chatName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("api_create_chat") + cmd.Params["chat_name"] = chatName + + return cmd, nil +} + +// CREATE SEARCH 'search_name' +func (p *Parser) parseAPICreateSearch() (*Command, error) { + p.nextToken() // consume SEARCH + searchName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("api_create_search") + cmd.Params["search_name"] = searchName + + return cmd, nil +} + +// CREATE AGENT 'agent_name' +func (p *Parser) parseAPICreateAgent() (*Command, error) { + p.nextToken() // consume AGENT + + agentName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("api_create_agent") + cmd.Params["agent_name"] = agentName + return cmd, nil +} + +// CREATE MEMORY 'memory_name' +func (p *Parser) parseAPICreateMemory() (*Command, error) { + p.nextToken() // consume MEMORY + memoryName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("api_create_memory") + cmd.Params["memory_name"] = memoryName + + return cmd, nil +} + +// CREATE KEY 'key_name' func (p *Parser) parseAPICreateKey() (*Command, error) { p.nextToken() // consume KEY @@ -748,6 +953,224 @@ func (p *Parser) parseAPICreateKey() (*Command, error) { return NewCommand("api_create_api_key"), nil } +// CREATE PROVIDER INSTANCE KEY URL REGION +func (p *Parser) parseAPICreateProviderInstance() (*Command, error) { + p.nextToken() // consume PROVIDER + + providerName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected provider name: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenInstance { + return nil, fmt.Errorf("expected INSTANCE after provider name") + } + p.nextToken() + + instanceName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected instance name: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenKey { + return nil, fmt.Errorf("expected KEY after instance name") + } + p.nextToken() + + apiKey, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected API key: %w", err) + } + p.nextToken() + + baseURL := "" + region := "" +optionsLoop: + for { + switch p.curToken.Type { + case TokenRegion: + p.nextToken() + region, err = p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected region: %w", err) + } + p.nextToken() + case TokenURL: + p.nextToken() + baseURL, err = p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected base URL: %w", err) + } + p.nextToken() + default: + break optionsLoop + } + } + + cmd := NewCommand("api_create_provider_instance") + cmd.Params["provider_name"] = providerName + cmd.Params["instance_name"] = instanceName + cmd.Params["api_key"] = apiKey + if baseURL != "" { + // Only local model provider need to set URL + cmd.Params["base_url"] = baseURL + } + + if region != "" { + cmd.Params["region"] = region + } + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +// endregion CREATE commands + +// region DROP commands +func (p *Parser) parseAPIDropCommands() (*Command, error) { + p.nextToken() // consume DROP + + switch p.curToken.Type { + case TokenDataset: + return p.parseAPIDropDataset() + case TokenChat: + return p.parseAPIDropChat() + case TokenSearch: + return p.parseAPIDropSearch() + case TokenMemory: + return p.parseAPIDropMemory() + case TokenAgent: + return p.parseAPIDropAgent() + case TokenKey: + return p.parseAPIDropAPIKey() + case TokenChunkStore: + return p.parseDevDropChunkStore() + case TokenMetadata: + return p.parseDevDropMetadataStore() + default: + return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value) + } +} + +func (p *Parser) parseAPIDropDataset() (*Command, error) { + p.nextToken() // consume DATASET + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("api_drop_dataset") + cmd.Params["dataset_name"] = datasetName + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDropChat() (*Command, error) { + p.nextToken() // consume CHAT + chatName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("api_drop_chat") + cmd.Params["chat_name"] = chatName + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDropSearch() (*Command, error) { + p.nextToken() // consume SEARCH + searchName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("api_drop_search") + cmd.Params["search_name"] = searchName + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDropMemory() (*Command, error) { + p.nextToken() // consume MEMORY + memoryName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("api_drop_memory") + cmd.Params["memory_name"] = memoryName + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDropAgent() (*Command, error) { + p.nextToken() // consume AGENT + agentName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("api_drop_agent") + cmd.Params["agent_name"] = agentName + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDropAPIKey() (*Command, error) { + p.nextToken() // consume KEY + + apiKey, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("api_delete_api_key") + cmd.Params["api_key"] = apiKey + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +// endregion DROP commands + +// region ADD commands + func (p *Parser) parseAPIAddCommands() (*Command, error) { p.nextToken() // consume ADD switch p.curToken.Type { @@ -1094,33 +1517,6 @@ optionsLoop: return cmd, nil } -// syntax: add admin host '127.0.0.1:9333' user 'ccc' password 'ppp' -func (p *Parser) parseAddAdminServer() (*Command, error) { - p.nextToken() // consume ADMIN - - if p.curToken.Type != TokenHost { - return nil, fmt.Errorf("expected HOST") - } - p.nextToken() - - host, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - ip, port, err := parseHostPort(host) - if err != nil { - return nil, err - } - - cmd := NewCommand("add_admin_server") - cmd.Params["server_ip"] = ip - cmd.Params["server_port"] = port - - return cmd, nil -} - // syntax: ADD API 'abc' HOST '127.0.0.1:9333'; func (p *Parser) parseAddAPIServer() (*Command, error) { p.nextToken() // consume API @@ -1147,7 +1543,7 @@ func (p *Parser) parseAddAPIServer() (*Command, error) { return nil, err } - cmd := NewCommand("add_api_server") + cmd := NewCommand("api_add_api_server") cmd.Params["server_name"] = serverName cmd.Params["server_ip"] = ip cmd.Params["server_port"] = port @@ -1155,197 +1551,36 @@ func (p *Parser) parseAddAPIServer() (*Command, error) { return cmd, nil } -// syntax: delete api 'abc' -func (p *Parser) parseAPIDeleteAPIServer() (*Command, error) { - p.nextToken() // consume API - - serverName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() // consume model name - - cmd := NewCommand("delete_api_server") - cmd.Params["server_name"] = serverName - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// syntax: delete admin server 'abc' -func (p *Parser) parseAPIDeleteAdminServer() (*Command, error) { +// syntax: add admin host '127.0.0.1:9333' user 'ccc' password 'ppp' +func (p *Parser) parseAddAdminServer() (*Command, error) { p.nextToken() // consume ADMIN - cmd := NewCommand("delete_admin_server") - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() + if p.curToken.Type != TokenHost { + return nil, fmt.Errorf("expected HOST") } + p.nextToken() - return cmd, nil -} - -func (p *Parser) parseAPISaveCommand() (*Command, error) { - p.nextToken() // consume SAVE - switch p.curToken.Type { - case TokenConfig: - return p.parseAPISaveConfig() - default: - return nil, fmt.Errorf("unknown ADD target: %s", p.curToken.Value) - } -} - -// SAVE CONFIG AS 'path' -func (p *Parser) parseAPISaveConfig() (*Command, error) { - p.nextToken() // consume CONFIG - - if p.curToken.Type != TokenAs { - return nil, fmt.Errorf("expected AS after CONFIG") - } - p.nextToken() // consume AS - - path, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("api_save_config_command") - cmd.Params["path"] = path - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// CREATE DATASET 'abc'; -func (p *Parser) parseAPICreateDataset() (*Command, error) { - p.nextToken() // consume DATASET - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("api_create_dataset") - cmd.Params["dataset_name"] = datasetName - return cmd, nil -} - -// CREAT CHAT 'chat_name' -func (p *Parser) parseAPICreateChat() (*Command, error) { - p.nextToken() // consume CHAT - chatName, err := p.parseQuotedString() + host, err := p.parseQuotedString() if err != nil { return nil, err } p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("api_create_chat") - cmd.Params["chat_name"] = chatName - - return cmd, nil -} - -// CREAT AGENT 'agent_name' -func (p *Parser) parseAPICreateAgent() (*Command, error) { - p.nextToken() // consume AGENT - - agentName, err := p.parseQuotedString() + ip, port, err := parseHostPort(host) if err != nil { return nil, err } - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("api_create_agent") - cmd.Params["agent_name"] = agentName - return cmd, nil -} - -// CREAT SEARCH 'search_name' -func (p *Parser) parseAPICreateSearch() (*Command, error) { - p.nextToken() // consume SEARCH - searchName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("api_create_search") - cmd.Params["search_name"] = searchName + cmd := NewCommand("api_add_admin_server") + cmd.Params["server_ip"] = ip + cmd.Params["server_port"] = port return cmd, nil } -// CREAT MEMORY 'memory_name' -func (p *Parser) parseAPICreateMemory() (*Command, error) { - p.nextToken() // consume MEMORY - memoryName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } +// endregion ADD commands - cmd := NewCommand("api_create_memory") - cmd.Params["memory_name"] = memoryName - - return cmd, nil -} - -func (p *Parser) parseAPIDropCommands() (*Command, error) { - p.nextToken() // consume DROP - - switch p.curToken.Type { - case TokenDataset: - return p.parseAPIDropDataset() - case TokenChat: - return p.parseAPIDropChat() - case TokenSearch: - return p.parseAPIDropSearch() - case TokenMemory: - return p.parseAPIDropMemory() - case TokenAgent: - return p.parseAPIDropAgent() - case TokenKey: - return p.parseAPIDeleteAPIKey() - case TokenChunkStore: - return p.parseDevDropChunkStore() - case TokenMetadata: - return p.parseDevDropMetadataStore() - default: - return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value) - } -} +// region DELETE commands func (p *Parser) parseAPIDeleteCommands() (*Command, error) { p.nextToken() // consume DELETE @@ -1364,43 +1599,6 @@ func (p *Parser) parseAPIDeleteCommands() (*Command, error) { } } -func (p *Parser) parseAPIRemoveCommands() (*Command, error) { - p.nextToken() // consume REMOVE - - switch p.curToken.Type { - // API commands - case TokenIngestion: - return p.parseAPIRemoveTask() - - // Dev commands - case TokenTag: - return p.parseDevRemoveTags() - case TokenChunks, TokenAll: - return p.parseDevRemoveChunk() - default: - return nil, fmt.Errorf("unknown REMOVE target: %s", p.curToken.Value) - } -} - -func (p *Parser) parseAPIDeleteAPIKey() (*Command, error) { - p.nextToken() // consume KEY - - apiKey, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - cmd := NewCommand("api_delete_api_key") - cmd.Params["api_key"] = apiKey - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - // DELETE PROVIDER // DELETE PROVIDER INSTANCE func (p *Parser) parseAPIDeleteProvider() (*Command, error) { @@ -1486,96 +1684,44 @@ func (p *Parser) parseAPIDeleteProviderInstanceModels(providerName, instanceName return cmd, nil } -func (p *Parser) parseAPIDropDataset() (*Command, error) { - p.nextToken() // consume DATASET - datasetName, err := p.parseQuotedString() +// syntax: delete api 'abc' +func (p *Parser) parseAPIDeleteAPIServer() (*Command, error) { + p.nextToken() // consume API + + serverName, err := p.parseQuotedString() if err != nil { return nil, err } - p.nextToken() + p.nextToken() // consume model name - cmd := NewCommand("api_drop_dataset") - cmd.Params["dataset_name"] = datasetName + cmd := NewCommand("api_delete_api_server") + cmd.Params["server_name"] = serverName // Semicolon is optional if p.curToken.Type == TokenSemicolon { p.nextToken() } + return cmd, nil } -func (p *Parser) parseAPIDropSearch() (*Command, error) { - p.nextToken() // consume SEARCH - searchName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() +// syntax: delete admin server 'abc' +func (p *Parser) parseAPIDeleteAdminServer() (*Command, error) { + p.nextToken() // consume ADMIN - cmd := NewCommand("api_drop_search") - cmd.Params["search_name"] = searchName + cmd := NewCommand("api_delete_admin_server") // Semicolon is optional if p.curToken.Type == TokenSemicolon { p.nextToken() } + return cmd, nil } -func (p *Parser) parseAPIDropChat() (*Command, error) { - p.nextToken() // consume CHAT - chatName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("api_drop_chat") - cmd.Params["chat_name"] = chatName - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseAPIDropMemory() (*Command, error) { - p.nextToken() // consume MEMORY - memoryName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("api_drop_memory") - cmd.Params["memory_name"] = memoryName - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseAPIDropAgent() (*Command, error) { - p.nextToken() // consume AGENT - agentName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("api_drop_agent") - cmd.Params["agent_name"] = agentName - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} +// endregion ADD commands +// region ALTER commands func (p *Parser) parseAPIAlterCommands() (*Command, error) { p.nextToken() // consume ALTER @@ -1587,83 +1733,6 @@ func (p *Parser) parseAPIAlterCommands() (*Command, error) { } } -// CREATE PROVIDER INSTANCE KEY URL REGION -func (p *Parser) parseAPICreateProviderInstance() (*Command, error) { - p.nextToken() // consume PROVIDER - - providerName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected provider name: %w", err) - } - p.nextToken() - - if p.curToken.Type != TokenInstance { - return nil, fmt.Errorf("expected INSTANCE after provider name") - } - p.nextToken() - - instanceName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected instance name: %w", err) - } - p.nextToken() - - if p.curToken.Type != TokenKey { - return nil, fmt.Errorf("expected KEY after instance name") - } - p.nextToken() - - apiKey, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected API key: %w", err) - } - p.nextToken() - - baseURL := "" - region := "" -optionsLoop: - for { - switch p.curToken.Type { - case TokenRegion: - p.nextToken() - region, err = p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected region: %w", err) - } - p.nextToken() - case TokenURL: - p.nextToken() - baseURL, err = p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected base URL: %w", err) - } - p.nextToken() - default: - break optionsLoop - } - } - - cmd := NewCommand("api_create_provider_instance") - cmd.Params["provider_name"] = providerName - cmd.Params["instance_name"] = instanceName - cmd.Params["api_key"] = apiKey - if baseURL != "" { - // Only local model provider need to set URL - cmd.Params["base_url"] = baseURL - } - - if region != "" { - cmd.Params["region"] = region - } - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - // ALTER PROVIDER INSTANCE [NAME | KEY ] func (p *Parser) parseAPIAlterInstance() (*Command, error) { p.nextToken() // consume PROVIDER @@ -1750,26 +1819,27 @@ func (p *Parser) parseIdentifierList() ([]string, error) { return list, nil } +// endregion ALTER commands + +// region SET commands + func (p *Parser) parseAPISetCommands() (*Command, error) { p.nextToken() // consume SET - if p.curToken.Type == TokenVar { + switch p.curToken.Type { + case TokenVar: return p.parseAPISetVariable() - } - if p.curToken.Type == TokenDefault { - return p.parseSetDefault() - } - if p.curToken.Type == TokenKey { + case TokenDefault: + return p.parseAPISetDefault() + case TokenKey: return p.parseAPISetAPIKey() - } - if p.curToken.Type == TokenMetadata { - return p.parseSetMeta() - } - if p.curToken.Type == TokenLog { + case TokenLog: return p.parseAPISetLog() + case TokenMetadata: + return p.parseDevSetMeta() + default: + return nil, fmt.Errorf("unknown SET target: %s", p.curToken.Value) } - - return nil, fmt.Errorf("unknown SET target: %s", p.curToken.Value) } func (p *Parser) parseAPISetVariable() (*Command, error) { @@ -1798,7 +1868,7 @@ func (p *Parser) parseAPISetVariable() (*Command, error) { return cmd, nil } -func (p *Parser) parseSetDefault() (*Command, error) { +func (p *Parser) parseAPISetDefault() (*Command, error) { p.nextToken() // consume DEFAULT var modelType, modelNameOrID string @@ -1840,7 +1910,7 @@ func (p *Parser) parseSetDefault() (*Command, error) { } p.nextToken() - cmd := NewCommand("set_default_model") + cmd := NewCommand("api_set_default_model") cmd.Params["model_type"] = modelType if common.IsCompositeModelName(modelNameOrID) { @@ -1917,6 +1987,10 @@ func (p *Parser) parseAPISetLogLevel() (*Command, error) { return cmd, nil } +// endregion SET commands + +// region RESET commands + func (p *Parser) parseAPIResetCommands() (*Command, error) { p.nextToken() // consume RESET @@ -1979,7 +2053,9 @@ func (p *Parser) parseAPIResetKey() (*Command, error) { return NewCommand("api_unset_api_key"), nil } -func (p *Parser) parseImportCommand() (*Command, error) { +// endregion RESET commands + +func (p *Parser) parseAPIImport() (*Command, error) { p.nextToken() // consume IMPORT documentPaths, err := p.parseQuotedString() if err != nil { @@ -2013,7 +2089,7 @@ func (p *Parser) parseImportCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseRetrieveCommand() (*Command, error) { +func (p *Parser) parseAPIRetrieve() (*Command, error) { p.nextToken() // consume SEARCH // Handle help flag: -h / --help. The lexer tokenizes each leading @@ -2272,7 +2348,269 @@ func (p *Parser) parseRetrieveCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseEnableCommand() (*Command, error) { +// region ParseCommands +func (p *Parser) parseParseCommands() (*Command, error) { + p.nextToken() // consume PARSE + + switch p.curToken.Type { + case TokenDataset: + return p.parseAPIParseDataset() + case TokenWith: + return p.parseAPIModelParse() + case TokenDocument: + return p.parseAPIParseDocs() + case TokenFile: + return p.parseAPIParseLocalFile() + default: + return nil, fmt.Errorf("expected DATASET, WITH, or DOCUMENT") + } +} + +func (p *Parser) parseAPIParseDataset() (*Command, error) { + p.nextToken() // consume DATASET + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + p.nextToken() + var method string + if p.curToken.Type == TokenSync { + method = "sync" + } else if p.curToken.Type == TokenAsync { + method = "async" + } else { + return nil, fmt.Errorf("expected SYNC or ASYNC") + } + p.nextToken() + + cmd := NewCommand("parse_dataset") + cmd.Params["dataset_name"] = datasetName + cmd.Params["method"] = method + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIModelParse() (*Command, error) { + p.nextToken() // consume WITH + + modelNameOrID, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("api_model_parse_file") + + switch p.curToken.Type { + case TokenFile: + p.nextToken() + var file string + file, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["file"] = file + p.nextToken() + case TokenURL: + p.nextToken() + var url string + url, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["url"] = url + p.nextToken() + default: + return nil, fmt.Errorf("expected FILE or URL") + } + + if common.IsCompositeModelName(modelNameOrID) { + cmd.Params["composite_model_name"] = modelNameOrID + } else if common.IsUUID(modelNameOrID) { + cmd.Params["model_id"] = modelNameOrID + } else { + return nil, fmt.Errorf("invalid format of model name or ID: %s", modelNameOrID) + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +func (p *Parser) parseAPIParseDocs() (*Command, error) { + p.nextToken() // consume document + + documentsStr, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + p.nextToken() + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expected FROM") + } + p.nextToken() + + datasetID, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("api_parse_documents") + + documents := strings.Split(documentsStr, " ") + + cmd.Params["documents"] = documents + cmd.Params["dataset_id"] = datasetID + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIParseLocalFile() (*Command, error) { + p.nextToken() // consume FILE + + filename, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + cmd := NewCommand("user_parse_local_file_command") + cmd.Params["filename"] = filename + +optionsLoop: + for { + switch p.curToken.Type { + case TokenVision: + p.nextToken() + var visionModel string + visionModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["vision_model"] = visionModel + p.nextToken() + case TokenASR: + p.nextToken() + var asrModel string + asrModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["asr_model"] = asrModel + p.nextToken() + + case TokenOCR: + p.nextToken() + var ocrModel string + ocrModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["ocr_model"] = ocrModel + p.nextToken() + case TokenChat: + p.nextToken() + var chatModel string + chatModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["chat_model"] = chatModel + p.nextToken() + case TokenEmbed: + p.nextToken() + var embedModel string + embedModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["embedding_model"] = embedModel + p.nextToken() + case TokenDocParse: + p.nextToken() + var docParseModel string + docParseModel, err = p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["doc_parse_model"] = docParseModel + p.nextToken() + case TokenSemicolon: + p.nextToken() + break optionsLoop // done + default: + // No more options to process + break optionsLoop + } + } + + return cmd, nil +} + +// endregion PARSE commands + +func (p *Parser) parseBenchmarkCommand() (*Command, error) { + cmd := NewCommand("benchmark") + + p.nextToken() // consume BENCHMARK + concurrency, err := p.parseNumber() + if err != nil { + return nil, err + } + cmd.Params["concurrency"] = concurrency + + p.nextToken() + iterations, err := p.parseNumber() + if err != nil { + return nil, err + } + cmd.Params["iterations"] = iterations + + p.nextToken() + // Parse user_statement + nestedCmd, err := p.parseUserStatement() // Not only user statement + if err != nil { + return nil, err + } + cmd.Params["command"] = nestedCmd + + return cmd, nil +} + +func (p *Parser) parseUserStatement() (*Command, error) { + switch p.curToken.Type { + case TokenPing: + return p.parseAPIPingServer() + case TokenShow: + return p.parseAPIShowCommands() + case TokenList: + return p.parseAPIListCommands() + case TokenImport: + return p.parseAPIImport() + case TokenInsert: + return p.parseDevInsertCommand() + case TokenRetrieve: + return p.parseAPIRetrieve() + default: + return nil, fmt.Errorf("invalid user statement: %s", p.curToken.Value) + } +} + +func (p *Parser) parseAPIEnable() (*Command, error) { p.nextToken() // consume ENABLE if p.curToken.Type != TokenModel { @@ -2315,7 +2653,7 @@ func (p *Parser) parseEnableCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseDisableCommand() (*Command, error) { +func (p *Parser) parseAPIDisable() (*Command, error) { p.nextToken() // consume DISABLE if p.curToken.Type != TokenModel { @@ -2358,9 +2696,10 @@ func (p *Parser) parseDisableCommand() (*Command, error) { return cmd, nil } +// region MODEL commands // CHAT 'model@instance@provider' 'hello world' // CHAT WITH 'model@instance@provider' MESSAGE 'hello world' 'who are you' IMAGE 'url1' 'file0' VIDEO "url2.mov" "file1" FILE "url" "path file2" AUDIO "file.wav" -func (p *Parser) parseChatCommand() (*Command, error) { +func (p *Parser) parseAPIChat() (*Command, error) { p.nextToken() // consume CHAT var err error @@ -2520,7 +2859,7 @@ optionsLoop: break optionsLoop } } - cmd := NewCommand("chat_to_model") + cmd := NewCommand("api_chat_to_model") if modelNameOrID != "" { if common.IsCompositeModelName(modelNameOrID) { @@ -2543,7 +2882,7 @@ optionsLoop: return cmd, nil } -func (p *Parser) parseThinkCommand() (*Command, error) { +func (p *Parser) parseAPIThink() (*Command, error) { p.nextToken() // consume THINK @@ -2551,7 +2890,7 @@ func (p *Parser) parseThinkCommand() (*Command, error) { return nil, fmt.Errorf("expected CHAT after THINK") } - command, err := p.parseChatCommand() + command, err := p.parseAPIChat() if err != nil { return nil, err } @@ -2559,7 +2898,7 @@ func (p *Parser) parseThinkCommand() (*Command, error) { return command, nil } -func (p *Parser) parseStreamCommand() (*Command, error) { +func (p *Parser) parseAPIStream() (*Command, error) { p.nextToken() // consume STREAM @@ -2568,22 +2907,22 @@ func (p *Parser) parseStreamCommand() (*Command, error) { switch p.curToken.Type { case TokenChat: - command, err = p.parseChatCommand() + command, err = p.parseAPIChat() if err != nil { return nil, err } case TokenThink: - command, err = p.parseThinkCommand() + command, err = p.parseAPIThink() if err != nil { return nil, err } case TokenASR: - command, err = p.parseASRCommand() + command, err = p.parseAPIASR() if err != nil { return nil, err } case TokenTTS: - command, err = p.parseTTSCommand() + command, err = p.parseAPITTS() if err != nil { return nil, err } @@ -2595,7 +2934,7 @@ func (p *Parser) parseStreamCommand() (*Command, error) { return command, nil } -func (p *Parser) parseEmbedCommand() (*Command, error) { +func (p *Parser) parseAPIEmbed() (*Command, error) { p.nextToken() // consume EMBED if p.curToken.Type != TokenText { @@ -2653,7 +2992,7 @@ textLoop: return nil, fmt.Errorf("unexpected token after embed command: %s", p.curToken.Value) } - cmd := NewCommand("embed_user_text") + cmd := NewCommand("api_embed_user_text") if common.IsCompositeModelName(modelNameOrID) { cmd.Params["composite_model_name"] = modelNameOrID } else if common.IsUUID(modelNameOrID) { @@ -2668,7 +3007,7 @@ textLoop: return cmd, nil } -func (p *Parser) parseRerankCommand() (*Command, error) { +func (p *Parser) parseAPIRerank() (*Command, error) { p.nextToken() // consume RERANK if p.curToken.Type != TokenQuery { @@ -2727,7 +3066,7 @@ documentLoop: } p.nextToken() - cmd := NewCommand("rarank_user_document") + cmd := NewCommand("api_rarank_user_document") if common.IsCompositeModelName(modelNameOrID) { cmd.Params["composite_model_name"] = modelNameOrID } else if common.IsUUID(modelNameOrID) { @@ -2741,7 +3080,7 @@ documentLoop: return cmd, nil } -func (p *Parser) parseASRCommand() (*Command, error) { +func (p *Parser) parseAPIASR() (*Command, error) { p.nextToken() // consume ASR if p.curToken.Type != TokenWith { @@ -2798,7 +3137,7 @@ func (p *Parser) parseASRCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseTTSCommand() (*Command, error) { +func (p *Parser) parseAPITTS() (*Command, error) { p.nextToken() cmd := NewCommand("tts_user_command") @@ -2876,7 +3215,7 @@ func (p *Parser) parseTTSCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseOCRCommand() (*Command, error) { +func (p *Parser) parseAPIOCR() (*Command, error) { p.nextToken() // consume OCR if p.curToken.Type != TokenWith { @@ -2931,57 +3270,9 @@ func (p *Parser) parseOCRCommand() (*Command, error) { return cmd, nil } -func (p *Parser) parseModelParseCommand() (*Command, error) { - p.nextToken() // consume WITH +// endregion MODEL commands - modelNameOrID, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - cmd := NewCommand("parse_file_user_command") - - switch p.curToken.Type { - case TokenFile: - p.nextToken() - var file string - file, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["file"] = file - p.nextToken() - case TokenURL: - p.nextToken() - var url string - url, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["url"] = url - p.nextToken() - default: - return nil, fmt.Errorf("expected FILE or URL") - } - - if common.IsCompositeModelName(modelNameOrID) { - cmd.Params["composite_model_name"] = modelNameOrID - } else if common.IsUUID(modelNameOrID) { - cmd.Params["model_id"] = modelNameOrID - } else { - return nil, fmt.Errorf("invalid format of model name or ID: %s", modelNameOrID) - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseCheckCommand() (*Command, error) { +func (p *Parser) parseAPICheck() (*Command, error) { p.nextToken() // consume CHECK switch p.curToken.Type { @@ -2994,6 +3285,73 @@ func (p *Parser) parseCheckCommand() (*Command, error) { } } +func (p *Parser) parseUserStartIngestion() (*Command, error) { + p.nextToken() // consume Start + + if p.curToken.Type != TokenIngestion { + return nil, fmt.Errorf("expect INGESTION") + } + p.nextToken() // consume Ingestion + + documentID, err := p.parseQuotedString() + if err != nil { + return nil, err + } + p.nextToken() + + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expect FROM") + } + p.nextToken() // consume FROM + + datasetID, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("api_start_ingestion") + cmd.Params["document_id"] = documentID + cmd.Params["dataset_id"] = datasetID + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseUserStopIngestion() (*Command, error) { + p.nextToken() // consume Stop + + if p.curToken.Type != TokenIngestion { + return nil, fmt.Errorf("expect INGESTION") + } + p.nextToken() // consume Ingestion + + if p.curToken.Type != TokenTasks { + return nil, fmt.Errorf("expect TASKS") + } + p.nextToken() + + taskStr, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + tasks := strings.Split(taskStr, " ") + + cmd := NewCommand("api_stop_ingestion") + cmd.Params["tasks"] = tasks + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + func (p *Parser) parseCheckInstanceCommand() (*Command, error) { if p.curToken.Type != TokenInstance { return nil, fmt.Errorf("expected INSTANCE after CHECK") @@ -3090,6 +3448,41 @@ func (p *Parser) parseCheckProviderByKeyCommand() (*Command, error) { return cmd, nil } +func (p *Parser) parseAPISave() (*Command, error) { + p.nextToken() // consume SAVE + switch p.curToken.Type { + case TokenConfig: + return p.parseAPISaveConfig() + default: + return nil, fmt.Errorf("unknown ADD target: %s", p.curToken.Value) + } +} + +// SAVE CONFIG AS 'path' +func (p *Parser) parseAPISaveConfig() (*Command, error) { + p.nextToken() // consume CONFIG + + if p.curToken.Type != TokenAs { + return nil, fmt.Errorf("expected AS after CONFIG") + } + p.nextToken() // consume AS + + path, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("api_save_config_command") + cmd.Params["path"] = path + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + func (p *Parser) parseAPIUseCommands() (*Command, error) { p.nextToken() // consume USE @@ -3162,332 +3555,23 @@ func (p *Parser) parseAPIUseAdminServer() (*Command, error) { return cmd, nil } -func (p *Parser) parseParseCommand() (*Command, error) { - p.nextToken() // consume PARSE +func (p *Parser) parseAPIRemove() (*Command, error) { + p.nextToken() // consume REMOVE switch p.curToken.Type { - case TokenDataset: - return p.parseParseDataset() - case TokenWith: - return p.parseModelParseCommand() - case TokenDocument: - return p.parseParseDocs() - case TokenFile: - return p.parseParseLocalFileCommand() + // API commands + case TokenIngestion: + return p.parseAPIRemoveTask() + + // Dev commands + case TokenTag: + return p.parseDevRemoveTags() + case TokenChunks, TokenAll: + return p.parseDevRemoveChunk() default: - return nil, fmt.Errorf("expected DATASET, WITH, or DOCUMENT") + return nil, fmt.Errorf("unknown REMOVE target: %s", p.curToken.Value) } } - -func (p *Parser) parseParseDataset() (*Command, error) { - p.nextToken() // consume DATASET - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - p.nextToken() - var method string - if p.curToken.Type == TokenSync { - method = "sync" - } else if p.curToken.Type == TokenAsync { - method = "async" - } else { - return nil, fmt.Errorf("expected SYNC or ASYNC") - } - p.nextToken() - - cmd := NewCommand("parse_dataset") - cmd.Params["dataset_name"] = datasetName - cmd.Params["method"] = method - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseParseDocs() (*Command, error) { - p.nextToken() // consume document - - documentsStr, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - p.nextToken() - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM") - } - p.nextToken() - - datasetID, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - cmd := NewCommand("parse_documents_user_command") - - documents := strings.Split(documentsStr, " ") - - cmd.Params["documents"] = documents - cmd.Params["dataset_id"] = datasetID - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseParseLocalFileCommand() (*Command, error) { - p.nextToken() // consume FILE - - filename, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - cmd := NewCommand("user_parse_local_file_command") - cmd.Params["filename"] = filename - -optionsLoop: - for { - switch p.curToken.Type { - case TokenVision: - p.nextToken() - var visionModel string - visionModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["vision_model"] = visionModel - p.nextToken() - case TokenASR: - p.nextToken() - var asrModel string - asrModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["asr_model"] = asrModel - p.nextToken() - - case TokenOCR: - p.nextToken() - var ocrModel string - ocrModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["ocr_model"] = ocrModel - p.nextToken() - case TokenChat: - p.nextToken() - var chatModel string - chatModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["chat_model"] = chatModel - p.nextToken() - case TokenEmbed: - p.nextToken() - var embedModel string - embedModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["embedding_model"] = embedModel - p.nextToken() - case TokenDocParse: - p.nextToken() - var docParseModel string - docParseModel, err = p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["doc_parse_model"] = docParseModel - p.nextToken() - case TokenSemicolon: - p.nextToken() - break optionsLoop // done - default: - // No more options to process - break optionsLoop - } - } - - return cmd, nil -} - -func (p *Parser) parseBenchmarkCommand() (*Command, error) { - cmd := NewCommand("benchmark") - - p.nextToken() // consume BENCHMARK - concurrency, err := p.parseNumber() - if err != nil { - return nil, err - } - cmd.Params["concurrency"] = concurrency - - p.nextToken() - iterations, err := p.parseNumber() - if err != nil { - return nil, err - } - cmd.Params["iterations"] = iterations - - p.nextToken() - // Parse user_statement - nestedCmd, err := p.parseUserStatement() // Not only user statement - if err != nil { - return nil, err - } - cmd.Params["command"] = nestedCmd - - return cmd, nil -} - -func (p *Parser) parseUserStatement() (*Command, error) { - switch p.curToken.Type { - case TokenPing: - return p.parseAPIPingServer() - case TokenShow: - return p.parseAPIShowCommands() - case TokenList: - return p.parseAPIListCommands() - case TokenParse: - return p.parseParseCommand() - case TokenImport: - return p.parseImportCommand() - case TokenInsert: - return p.parseDevInsertCommand() - case TokenRetrieve: - return p.parseRetrieveCommand() - case TokenGet: - return p.parseGetCommand() - default: - return nil, fmt.Errorf("invalid user statement: %s", p.curToken.Value) - } -} - -func (p *Parser) parseUserStartIngestion() (*Command, error) { - p.nextToken() // consume Start - - if p.curToken.Type != TokenIngestion { - return nil, fmt.Errorf("expect INGESTION") - } - p.nextToken() // consume Ingestion - - documentID, err := p.parseQuotedString() - if err != nil { - return nil, err - } - p.nextToken() - - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expect FROM") - } - p.nextToken() // consume FROM - - datasetID, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("user_start_ingestion_command") - cmd.Params["document_id"] = documentID - cmd.Params["dataset_id"] = datasetID - p.nextToken() - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -// SHOW ADMIN SERVER -func (p *Parser) parseAPIShowAdmin() (*Command, error) { - p.nextToken() // consume ADMIN - - var cmd *Command - switch p.curToken.Type { - case TokenServer: - p.nextToken() - cmd = NewCommand("api_show_admin_server") - default: - return nil, fmt.Errorf("expected SERVER after ADMIN") - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseUserStopIngestion() (*Command, error) { - p.nextToken() // consume Stop - - if p.curToken.Type != TokenIngestion { - return nil, fmt.Errorf("expect INGESTION") - } - p.nextToken() // consume Ingestion - - if p.curToken.Type != TokenTasks { - return nil, fmt.Errorf("expect TASKS") - } - p.nextToken() - - taskStr, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - tasks := strings.Split(taskStr, " ") - - cmd := NewCommand("user_stop_ingestion_command") - cmd.Params["tasks"] = tasks - p.nextToken() - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseAPIListIngestionTasks() (*Command, error) { - p.nextToken() // consume Ingestion - - if p.curToken.Type != TokenTasks { - return nil, fmt.Errorf("expected TASKS") - } - p.nextToken() // consume TASKS - - cmd := NewCommand("api_list_ingestion_tasks") - - if p.curToken.Type == TokenFrom { - p.nextToken() - datasetID, err := p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["dataset_id"] = datasetID - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - func (p *Parser) parseAPIRemoveTask() (*Command, error) { p.nextToken() // consume Ingestion @@ -3501,7 +3585,7 @@ func (p *Parser) parseAPIRemoveTask() (*Command, error) { return nil, err } - cmd := NewCommand("user_remove_task_command") + cmd := NewCommand("api_remove_task") tasks := strings.Split(taskStr, " ") @@ -3515,150 +3599,7 @@ func (p *Parser) parseAPIRemoveTask() (*Command, error) { return cmd, nil } -// SHOW API SERVER -func (p *Parser) parseAPIShowAPI() (*Command, error) { - p.nextToken() // consume API - - var cmd *Command - switch p.curToken.Type { - case TokenServer: - p.nextToken() - cmd = NewCommand("api_show_api_server") - - serverName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected API server name: %w", err) - } - cmd.Params["api_server_name"] = serverName - p.nextToken() - - default: - return nil, fmt.Errorf("expected SERVER after API") - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseAPIShowLogCommands() (*Command, error) { - p.nextToken() // consume LOG - - switch p.curToken.Type { - case TokenLevel: - return p.parseShowLogLevel() - default: - return nil, fmt.Errorf("expected LEVEL after LOG") - } -} - -func (p *Parser) parseShowLogLevel() (*Command, error) { - p.nextToken() // consume LEVEL - - cmd := NewCommand("api_show_log_level") - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseAPIListAPIServers() (*Command, error) { - p.nextToken() // consume API - - var cmd *Command - switch p.curToken.Type { - case TokenServer: - p.nextToken() - cmd = NewCommand("api_list_api_servers") - default: - return nil, fmt.Errorf("expected SERVER after API") - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseAPIListEnvironments() (*Command, error) { - p.nextToken() // consume Envs - - cmd := NewCommand("api_list_environments") - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseAPIListVariables() (*Command, error) { - p.nextToken() // consume Variables - - cmd := NewCommand("api_list_variables") - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -func (p *Parser) parseExplainCommand() (*Command, error) { - p.nextToken() // consume EXPLAIN - - switch p.curToken.Type { - case TokenChunk: - return p.parseChunkCommand(true) - default: - return nil, fmt.Errorf("expected CHUNK after EXPLAIN") - } -} - -func (p *Parser) parseChunkCommand(explain bool) (*Command, error) { - p.nextToken() // consume CHUNK - - filename, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected filename: %w", err) - } - p.nextToken() - - if p.curToken.Type != TokenWith { - return nil, fmt.Errorf("expected WITH after filename") - } - p.nextToken() - - dsl, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected DSL: %w", err) - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - p.nextToken() - - cmd := NewCommand("user_chunk_command") - cmd.Params["dsl"] = dsl - cmd.Params["filename"] = filename - cmd.Params["explain"] = explain - - return cmd, nil -} - -// parseOpenaiChatCommand parses: +// parseOpenaiChat parses: // // OPENAI_CHAT // [model ] [system ] @@ -3684,7 +3625,7 @@ func (p *Parser) parseChunkCommand(explain bool) (*Command, error) { // reference_metadata { include?: bool, fields?: string[] } // metadata_condition { logic?: "and"|"or", conditions?: [{key, operator, value}] } // (See user_command.go:allowedExtraBodyKeys for the authoritative set) -func (p *Parser) parseOpenaiChatCommand() (*Command, error) { +func (p *Parser) parseOpenaiChat() (*Command, error) { p.nextToken() // consume OPENAI_CHAT if p.curToken.Type == TokenDash { @@ -3702,7 +3643,7 @@ func (p *Parser) parseOpenaiChatCommand() (*Command, error) { return nil, fmt.Errorf("OPENAI_CHAT: only -h/--help takes no args; otherwise expected chat_id and message") } - cmd := NewCommand("openai_chat") + cmd := NewCommand("api_openai_chat") // Defaults — match the OpenAI spec / RAGFlow server behavior. cmd.Params["model"] = "model" // placeholder; server resolves to dialog.llm_id diff --git a/internal/cli/user_parser_test.go b/internal/cli/user_parser_test.go index 40404dfe77..0e1c46e4d6 100644 --- a/internal/cli/user_parser_test.go +++ b/internal/cli/user_parser_test.go @@ -12,8 +12,8 @@ func TestParseChatMessageUsesCurrentModel(t *testing.T) { t.Fatalf("Parse() error = %v", err) } - if cmd.Type != "chat_to_model" { - t.Fatalf("Command Type = %v, expected chat_to_model", cmd.Type) + if cmd.Type != "api_chat_to_model" { + t.Fatalf("Command Type = %v, expected api_chat_to_model", cmd.Type) } if _, ok := cmd.Params["composite_model_name"]; ok { t.Fatal("composite_model_name should not be set") diff --git a/internal/engine/infinity/metadata.go b/internal/engine/infinity/metadata.go index 438023dfe2..dcc7ef34f8 100644 --- a/internal/engine/infinity/metadata.go +++ b/internal/engine/infinity/metadata.go @@ -225,7 +225,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri // reader naturally parses as "replace". The merge semantics exist so // that user-driven metadata edits compose with auto-extracted fields // produced by the LLM extraction pipeline. See the CLI parser in -// internal/cli/user_parser.go (parseSetMeta) for the user-facing +// internal/cli/user_parser.go (parseDevSetMeta) for the user-facing // surface that drives this engine method. func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error { tableName := buildMetadataTableName(tenantID)