diff --git a/internal/admin/enterprise_handler.go b/internal/admin/enterprise_handler.go index 3209c2f02b..aed4321df8 100644 --- a/internal/admin/enterprise_handler.go +++ b/internal/admin/enterprise_handler.go @@ -584,8 +584,8 @@ func (h *Handler) CheckProviderConnection(c *gin.Context) { } type AlterProviderInstanceRequest struct { - ModelName string `json:"model_name"` - APIKey string `json:"api_key"` + InstanceName string `json:"instance_name"` + APIKey string `json:"api_key"` } func (h *Handler) AlterProviderInstance(c *gin.Context) { @@ -625,7 +625,7 @@ func (h *Handler) AlterProviderInstance(c *gin.Context) { return } - result, err := h.service.AlterProviderInstance(userID, providerName, instanceName, req.ModelName, req.APIKey) + result, err := h.service.AlterProviderInstance(userID, providerName, instanceName, req.InstanceName, req.APIKey) if err != nil { errorResponse(c, err.Error(), 500) return diff --git a/internal/cli/admin_parser.go b/internal/cli/admin_parser.go index 739e6eb2cc..e11e1d4bcc 100644 --- a/internal/cli/admin_parser.go +++ b/internal/cli/admin_parser.go @@ -1433,14 +1433,14 @@ func (p *Parser) parseAdminAlterProvider() (*Command, error) { } p.nextToken() - newModelName := "" + newInstanceName := "" newAPIKey := "" optionsLoop: for { switch p.curToken.Type { case TokenName: p.nextToken() - newModelName, err = p.parseQuotedString() + newInstanceName, err = p.parseQuotedString() if err != nil { return nil, fmt.Errorf("expected model name: %w", err) } @@ -1457,14 +1457,14 @@ optionsLoop: } } - if newModelName == "" && newAPIKey == "" { + if newInstanceName == "" && newAPIKey == "" { return nil, fmt.Errorf("expected NAME or KEY after INSTANCE") } cmd := NewCommand("admin_alter_provider_instance") cmd.Params["provider_name"] = providerName cmd.Params["instance_name"] = instanceName - cmd.Params["new_model_name"] = newModelName + cmd.Params["new_instance_name"] = newInstanceName cmd.Params["new_api_key"] = newAPIKey p.nextToken() @@ -2100,9 +2100,9 @@ func (p *Parser) parseAdminDeleteCommands() (*Command, error) { p.nextToken() // consume DELETE switch p.curToken.Type { case TokenAPI: - return p.parseDeleteAPIServer() + return p.parseAPIDeleteAPIServer() case TokenAdmin: - return p.parseDeleteAdminServer() + return p.parseAPIDeleteAdminServer() case TokenProvider: return p.parseAdminDeleteProvider() default: diff --git a/internal/cli/cli.go b/internal/cli/cli.go index fdd5c6fba2..185c3ea3ce 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -736,7 +736,7 @@ func (c *CLI) Run() error { // the command handlers. Don't echo that back to the operator // verbatim — log the full error server-side for debugging, and // show only the error type/message via a sanitized wrapper. - fmt.Printf("CLI error: %s\n", sanitizeCLIError(err)) + fmt.Printf("ragflow-cli error: %s\n", sanitizeCLIError(err)) } } diff --git a/internal/cli/cli_http.go b/internal/cli/cli_http.go index b1b42a589e..43e6b94a1d 100644 --- a/internal/cli/cli_http.go +++ b/internal/cli/cli_http.go @@ -136,7 +136,7 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) { case "admin_list_providers": return c.AdminListProvidersCommand(cmd) case "admin_list_all_models": - return c.ListAllModels(cmd) + return c.CommonListAllModels(cmd) case "list_admin_tasks": return c.ListAdminTasks(cmd) case "admin_list_ingestors": @@ -293,7 +293,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.Logout() case "api_ping_server": return c.PingByCommand(cmd) - // Configuration commands case "api_list_configs": return c.ListConfigs(cmd) case "api_set_log_level": @@ -314,8 +313,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.APIListSearchesCommand(cmd) case "api_list_memories": return c.APIListMemoriesCommand(cmd) - case "list_dataset_documents": - return c.ListDatasetDocumentUserCommand(cmd) case "search_on_datasets": return c.SearchOnDatasets(cmd) case "search_help": @@ -371,17 +368,20 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.CommonListInstanceModelsSyncCommand(cmd) case "api_list_provider_instance_tasks": return c.APIListModelInstanceTasksCommand(cmd) + + // Provider commands case "api_show_model": return c.CommonShowModelCommand(cmd) - case "list_all_models": - return c.ListAllModels(cmd) - // Provider commands - case "add_provider": - return c.AddProvider(cmd) + case "api_list_all_models": + return c.CommonListAllModels(cmd) + case "api_add_provider": + return c.APIAddProviderCommand(cmd) case "api_list_providers": return c.APIListProviders(cmd) - case "delete_provider": - return c.DeleteProvider(cmd) + case "api_delete_provider": + return c.APIDeleteProvider(cmd) + case "api_delete_provider_instance": + return c.APIDeleteProviderInstanceCommand(cmd) case "api_drop_dataset": return c.APIDropDatasetCommand(cmd) case "api_drop_chat": @@ -392,22 +392,19 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.APIDropMemoryCommand(cmd) case "api_drop_agent": return c.APIDropAgentCommand(cmd) - // Provider instance commands - case "api_create_provider_instance": - return c.APICreateProviderInstanceCommand(cmd) + case "api_add_provider_instance": + return c.APIAddProviderInstanceCommand(cmd) case "api_list_provider_instances": return c.CommonListProviderInstances(cmd) - case "alter_provider_instance": + case "api_alter_provider_instance": return c.CommonAlterProviderInstanceCommand(cmd) - case "drop_provider_instance": - return c.DropProviderInstance(cmd) - case "drop_instance_model": - return c.DropInstanceModel(cmd) + case "api_delete_provider_instance_model": + return c.APIDeleteProviderInstanceModelCommand(cmd) case "enable_model": return c.CommonEnableOrDisableModel(cmd, "enable") case "disable_model": return c.CommonEnableOrDisableModel(cmd, "disable") - case "add_custom_model": + case "api_add_custom_model": return c.AddCustomModel(cmd) case "chat_to_model": return c.ChatToModel(cmd) @@ -446,32 +443,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.ResetDefaultModel(cmd) case "api_list_default_models": return c.ListDefaultModels(cmd) - case "create_chunk_store": - return c.CreateChunkStore(cmd) - case "drop_chunk_store": - return c.DropChunkStore(cmd) - case "create_metadata_store": - return c.CreateMetadataStore(cmd) - case "drop_metadata_store": - return c.DropMetadataStore(cmd) - case "insert_chunks_from_file": - return c.InsertChunksFromFile(cmd) - case "insert_metadata_from_file": - return c.InsertMetadataFromFile(cmd) - case "update_chunk": - return c.UpdateChunk(cmd) - case "get_chunk": - return c.GetChunk(cmd) - case "set_meta": - return c.SetMeta(cmd) - case "delete_meta": - return c.DeleteMeta(cmd) - case "rm_tags": - return c.RmTags(cmd) - case "remove_chunks": - return c.RemoveChunks(cmd) - case "get_metadata": - return c.GetMetadata(cmd) case "parse_documents_user_command": return c.ParseDocumentsUserCommand(cmd) case "user_start_ingestion_command": @@ -482,7 +453,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.ListUserIngestionTasks(cmd) case "user_remove_task_command": return c.UserRemoveTaskCommand(cmd) - // TODO: Implement other commands case "user_parse_local_file_command": return c.UserParseLocalFile(cmd) case "api_show_admin_server": @@ -509,8 +479,38 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) { return c.ChunkCommand(cmd) case "api_save_config_command": return c.CommonSaveServerConfigCommand(cmd) + + // File system commands case "file_system_command": return c.ExecuteFilesystemCommand(cmd) + + // For debug + case "dev_create_chunk_store": + return c.DevCreateChunkStoreCommand(cmd) + case "dev_drop_chunk_store": + return c.DevDropChunkStoreCommand(cmd) + case "dev_create_metadata_store": + return c.DevCreateMetadataStoreCommand(cmd) + case "dev_drop_metadata_store": + return c.DevDropMetadataStoreCommand(cmd) + case "dev_insert_chunks_from_file": + return c.DevInsertChunksFromFileCommand(cmd) + case "dev_insert_metadata_from_file": + return c.DevInsertMetadataFromFileCommand(cmd) + case "dev_update_chunk": + return c.DevUpdateChunkCommand(cmd) + case "dev_get_chunk": + return c.DevGetChunkCommand(cmd) + case "dev_set_meta": + return c.DevSetMetaCommand(cmd) + case "dev_delete_meta": + return c.DevDeleteMetaCommand(cmd) + case "dev_rm_tags": + return c.DevRmTagsCommand(cmd) + case "dev_remove_chunks": + return c.DevRemoveChunksCommand(cmd) + case "dev_get_metadata": + return c.DevGetMetadataCommand(cmd) default: return nil, fmt.Errorf("command '%s' would be executed with API", cmd.Type) } diff --git a/internal/cli/common_command.go b/internal/cli/common_command.go index 3a427d71d1..5ff5b38dcd 100644 --- a/internal/cli/common_command.go +++ b/internal/cli/common_command.go @@ -799,9 +799,9 @@ func (c *CLI) CommonAlterProviderInstanceCommand(cmd *Command) (ResponseIf, erro payload := map[string]interface{}{} - newName, ok := cmd.Params["new_model_name"].(string) + newName, ok := cmd.Params["new_instance_name"].(string) if ok { - payload["model_name"] = newName + payload["instance_name"] = newName } newAPIKey, ok := cmd.Params["new_api_key"].(string) @@ -1387,7 +1387,7 @@ func (c *CLI) GetAPIServerInfo(serverName string) (ResponseIf, error) { return &result, nil } -func (c *CLI) ListAllModels(cmd *Command) (ResponseIf, error) { +func (c *CLI) CommonListAllModels(cmd *Command) (ResponseIf, error) { page, ok := cmd.Params["page"].(int) if !ok { diff --git a/internal/cli/dev_parser.go b/internal/cli/dev_parser.go new file mode 100644 index 0000000000..643800d95e --- /dev/null +++ b/internal/cli/dev_parser.go @@ -0,0 +1,659 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package cli + +import ( + "fmt" + "strconv" +) + +// INSERT CHUNKS FROM FILE "file_path" +// INSERT METADATA FROM FILE "file_path" +func (p *Parser) parseDevInsertCommand() (*Command, error) { + p.nextToken() // consume INSERT + + // Expect CHUNKS or METADATA + if p.curToken.Type == TokenChunks { + return p.parseInsertChunksFromFile() + } + if p.curToken.Type == TokenMetadata { + return p.parseInsertMetadataFromFile() + } + return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value) +} + +// Internal CLI for GO +// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path" +func (p *Parser) parseInsertChunksFromFile() (*Command, error) { + p.nextToken() // consume CHUNKS + + // Expect FROM + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value) + } + p.nextToken() + + // Expect FILE + if p.curToken.Type != TokenFile { + return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value) + } + p.nextToken() + + // Get file path (quoted string) + filePath, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("dev_insert_chunks_from_file") + cmd.Params["file_path"] = filePath + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +// Internal CLI for GO +// parseInsertMetadataFromFile parses: INSERT METADATA FROM FILE "file_path" +func (p *Parser) parseInsertMetadataFromFile() (*Command, error) { + p.nextToken() // consume METADATA + + // Expect FROM + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value) + } + p.nextToken() + + // Expect FILE + if p.curToken.Type != TokenFile { + return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value) + } + p.nextToken() + + // Get file path (quoted string) + filePath, err := p.parseQuotedString() + if err != nil { + return nil, err + } + + cmd := NewCommand("dev_insert_metadata_from_file") + cmd.Params["file_path"] = filePath + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +// parseGetCommand parses: GET CHUNK or GET METADATA +func (p *Parser) parseGetCommand() (*Command, error) { + p.nextToken() // consume GET + + if p.curToken.Type == TokenChunk { + return p.parseGetChunk() + } + if p.curToken.Type == TokenMetadata { + return p.parseGetMetadata() + } + + return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value) +} + +// parseGetChunk parses: GET CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' +func (p *Parser) parseGetChunk() (*Command, error) { + p.nextToken() // consume CHUNK + + // Parse chunk_id + chunkID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected chunk_id: %w", err) + } + + cmd := NewCommand("dev_get_chunk") + cmd.Params["chunk_id"] = chunkID + + p.nextToken() + if p.curToken.Type != TokenOf { + return nil, fmt.Errorf("expected OF after chunk_id") + } + p.nextToken() + + if p.curToken.Type != TokenDocument { + return nil, fmt.Errorf("expected DOCUMENT after OF") + } + p.nextToken() + + // Parse doc_id + docID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected doc_id: %w", err) + } + cmd.Params["doc_id"] = docID + + p.nextToken() + if p.curToken.Type != TokenIn { + return nil, fmt.Errorf("expected IN after doc_id") + } + p.nextToken() + + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected DATASET after IN") + } + p.nextToken() + + // Parse dataset_id + datasetID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset_id: %w", err) + } + cmd.Params["dataset_id"] = datasetID + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// Internal +// parseDevUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}' +func (p *Parser) parseDevUpdateCommand() (*Command, error) { + p.nextToken() // consume UPDATE + + if p.curToken.Type == TokenChunk { + return p.parseUpdateChunk() + } + + return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value) +} + +// Internal CLI for GO +// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' SET '{"content": "..."}' +func (p *Parser) parseUpdateChunk() (*Command, error) { + p.nextToken() // consume CHUNK + + // Parse chunk_id + chunkID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected chunk_id: %w", err) + } + + cmd := NewCommand("dev_update_chunk") + cmd.Params["chunk_id"] = chunkID + + p.nextToken() + if p.curToken.Type != TokenOf { + return nil, fmt.Errorf("expected OF after chunk_id") + } + p.nextToken() + + if p.curToken.Type != TokenDocument { + return nil, fmt.Errorf("expected DOCUMENT after OF") + } + p.nextToken() + + // Parse doc_id + docID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected doc_id: %w", err) + } + cmd.Params["doc_id"] = docID + + p.nextToken() + if p.curToken.Type != TokenIn { + return nil, fmt.Errorf("expected IN after doc_id") + } + p.nextToken() + + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected DATASET after IN") + } + p.nextToken() + + // Parse dataset_name + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset_name: %w", err) + } + cmd.Params["dataset_name"] = datasetName + + p.nextToken() + if p.curToken.Type != TokenSet { + return nil, fmt.Errorf("expected SET after dataset_name") + } + p.nextToken() + + // Parse JSON body + jsonBody, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected JSON body: %w", err) + } + cmd.Params["json_body"] = jsonBody + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// parseSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}' +func (p *Parser) parseSetMeta() (*Command, error) { + p.nextToken() // consume METADATA + + // Expect OF + if p.curToken.Type != TokenOf { + return nil, fmt.Errorf("expected OF after SET METADATA") + } + p.nextToken() + + // Expect DOCUMENT + if p.curToken.Type != TokenDocument { + return nil, fmt.Errorf("expected DOCUMENT after SET METADATA OF") + } + p.nextToken() + + // Parse doc_id + docID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected doc_id: %w", err) + } + cmd := NewCommand("dev_set_meta") + cmd.Params["doc_id"] = docID + + p.nextToken() + // Expect TO + if p.curToken.Type != TokenTo { + return nil, fmt.Errorf("expected TO after doc_id") + } + p.nextToken() + + // Parse meta JSON + meta, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected meta JSON: %w", err) + } + cmd.Params["meta"] = meta + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// parseDevDeleteMeta parses: DELETE METADATA OF DOCUMENT 'doc_id' [KEYS '["key1", "key2"]'] +// If KEYS is not provided, deletes entire document metadata +func (p *Parser) parseDevDeleteMeta() (*Command, error) { + p.nextToken() // consume METADATA + + // Expect OF + if p.curToken.Type != TokenOf { + return nil, fmt.Errorf("expected OF after DELETE METADATA") + } + p.nextToken() + + // Expect DOCUMENT + if p.curToken.Type != TokenDocument { + return nil, fmt.Errorf("expected DOCUMENT after DELETE METADATA OF") + } + p.nextToken() + + // Parse doc_id + docID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected doc_id: %w", err) + } + cmd := NewCommand("dev_delete_meta") + cmd.Params["doc_id"] = docID + + p.nextToken() + // KEYS is optional - if not provided, delete entire document metadata + if p.curToken.Type != TokenKeys { + if p.curToken.Type == TokenSemicolon { + p.nextToken() + return cmd, nil + } + if p.curToken.Type == TokenEOF { + return cmd, nil + } + return nil, fmt.Errorf("expected KEYS or end of command after doc_id") + } + + // Parse keys JSON array + p.nextToken() + keys, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected keys JSON array: %w", err) + } + cmd.Params["keys"] = keys + + p.nextToken() + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + return cmd, nil + } + if p.curToken.Type != TokenEOF { + return nil, fmt.Errorf("expected end of command after KEYS") + } + + return cmd, nil +} + +// parseDevRemoveTags parses: REMOVE TAGS 'tag1', 'tag2' from DATASET 'dataset_name'; +func (p *Parser) parseDevRemoveTags() (*Command, error) { + p.nextToken() // consume TAGS + + // Parse first tag + tag, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected tag: %w", err) + } + tags := []string{tag} + + // Parse additional tags separated by commas + for { + p.nextToken() + if p.curToken.Type == TokenComma { + p.nextToken() + tag, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected tag after comma: %w", err) + } + tags = append(tags, tag) + } else { + break + } + } + cmd := NewCommand("dev_rm_tags") + cmd.Params["tags"] = tags + + // Expect from + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expected FROM after tags") + } + p.nextToken() + + // Expect DATASET + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected DATASET after FROM") + } + p.nextToken() + + // Parse dataset_name + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset_name: %w", err) + } + cmd.Params["dataset_name"] = datasetName + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// parseDevRemoveChunk parses: +// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name'; +// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name'; +func (p *Parser) parseDevRemoveChunk() (*Command, error) { + cmd := NewCommand("dev_remove_chunks") + + // Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL + if p.curToken.Type == TokenAll { + p.nextToken() // consume ALL + if p.curToken.Type != TokenChunks { + return nil, fmt.Errorf("expected CHUNKS after ALL") + } + p.nextToken() // consume CHUNKS + cmd.Params["delete_all"] = true + } else { + // curToken is TokenChunks, consume it first + p.nextToken() + // Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name' (space-separated) + // Parse first chunk ID + chunkID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected chunk_id: %w", err) + } + chunkIDs := []string{chunkID} + + // Parse additional chunk IDs separated by spaces (each quoted) + for { + p.nextToken() + // Stop if we hit FROM or non-quoted token + if p.curToken.Type == TokenFrom || p.curToken.Type != TokenQuotedString { + break + } + chunkID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected chunk_id: %w", err) + } + chunkIDs = append(chunkIDs, chunkID) + } + cmd.Params["chunk_ids"] = chunkIDs + } + + // Expect FROM + if p.curToken.Type != TokenFrom { + return nil, fmt.Errorf("expected FROM after chunk(s)") + } + p.nextToken() + + // Expect DOCUMENT + if p.curToken.Type != TokenDocument { + return nil, fmt.Errorf("expected DOCUMENT after FROM") + } + p.nextToken() + + // Parse doc_id + docID, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected doc_id: %w", err) + } + cmd.Params["doc_id"] = docID + + p.nextToken() + + // Expect IN + if p.curToken.Type != TokenIn { + return nil, fmt.Errorf("expected IN after doc_id") + } + p.nextToken() + + // Expect DATASET + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected DATASET after IN") + } + p.nextToken() + + // Parse dataset_name (quoted string) + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset_name: %w", err) + } + cmd.Params["dataset_name"] = datasetName + + p.nextToken() + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return cmd, nil +} + +// Internal CLI for GO +// parseDevDropChunkStore parses: DROP CHUNK STORE for Dataset 'name' +func (p *Parser) parseDevDropChunkStore() (*Command, error) { + p.nextToken() // consume CHUNK STORE + + // Expect FOR + if p.curToken.Type != TokenFor { + return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value) + } + p.nextToken() + + // Expect Dataset + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value) + } + p.nextToken() + + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value) + } + + p.nextToken() + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("dev_drop_chunk_store") + cmd.Params["dataset_name"] = datasetName + return cmd, nil +} + +// parseDevDropMetadataStore parses: DROP METADATA STORE +func (p *Parser) parseDevDropMetadataStore() (*Command, error) { + // DROP METADATA STORE + p.nextToken() // consume METADATA + + if p.curToken.Type != TokenStore { + return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value) + } + p.nextToken() + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("dev_drop_metadata_store") + return cmd, nil +} + +// Internal CLI for GO +// parseDevCreateChunkStore parses: CREATE CHUNK STORE for Dataset 'name' VECTOR SIZE N +func (p *Parser) parseDevCreateChunkStore() (*Command, error) { + p.nextToken() // consume CHUNK STORE compound token + + // Expect FOR + if p.curToken.Type != TokenFor { + return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value) + } + p.nextToken() + + // Expect Dataset + if p.curToken.Type != TokenDataset { + return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value) + } + p.nextToken() + + datasetName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value) + } + + p.nextToken() + if p.curToken.Type != TokenVector { + return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value) + } + p.nextToken() + if p.curToken.Type != TokenSize { + return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value) + } + p.nextToken() + + if p.curToken.Type != TokenInteger { + return nil, fmt.Errorf("expected vector size number, got %s", p.curToken.Value) + } + vectorSize, err := strconv.Atoi(p.curToken.Value) + if err != nil { + return nil, fmt.Errorf("invalid vector size: %s", p.curToken.Value) + } + + p.nextToken() + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + cmd := NewCommand("dev_create_chunk_store") + cmd.Params["dataset_name"] = datasetName + cmd.Params["vector_size"] = vectorSize + return cmd, nil +} + +// Internal CLI for GO +// parseCreateMetadataStore parses: CREATE METADATA STORE +func (p *Parser) parseCreateMetadataStore() (*Command, error) { + // CREATE METADATA STORE + p.nextToken() // consume METADATA + + if p.curToken.Type != TokenStore { + return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value) + } + p.nextToken() + + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + + return NewCommand("dev_create_metadata_store"), nil +} + +func (p *Parser) parseCreateRole() (*Command, error) { + p.nextToken() // consume ROLE + roleName, err := p.parseIdentifier() + if err != nil { + return nil, err + } + + cmd := NewCommand("create_role") + cmd.Params["role_name"] = roleName + + p.nextToken() + if p.curToken.Type == TokenDescription { + p.nextToken() + description, err := p.parseQuotedString() + if err != nil { + return nil, err + } + cmd.Params["description"] = description + p.nextToken() + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} diff --git a/internal/cli/parser.go b/internal/cli/parser.go index 7a3b4120e2..ae49977b6c 100644 --- a/internal/cli/parser.go +++ b/internal/cli/parser.go @@ -157,11 +157,11 @@ func (p *Parser) parseUserCommand() (*Command, error) { case TokenDrop: return p.parseAPIDropCommands() case TokenAdd: - return p.parseAddCommand() + return p.parseAPIAddCommands() case TokenDelete: - return p.parseDeleteCommand() + return p.parseAPIDeleteCommands() case TokenAlter: - return p.parseAlterCommand() + return p.parseAPIAlterCommands() case TokenSet: return p.parseAPISetCommands() case TokenReset: @@ -169,7 +169,8 @@ func (p *Parser) parseUserCommand() (*Command, error) { case TokenImport: return p.parseImportCommand() case TokenInsert: - return p.parseInsertCommand() + // Use for development only + return p.parseDevInsertCommand() case TokenRetrieve: return p.parseRetrieveCommand() case TokenParse: @@ -211,9 +212,9 @@ func (p *Parser) parseUserCommand() (*Command, error) { case TokenUse: return p.parseAPIUseCommands() case TokenUpdate: - return p.parseUpdateCommand() + return p.parseDevUpdateCommand() case TokenRemove: - return p.parseRemoveCommand() + return p.parseAPIRemoveCommands() case TokenGet: return p.parseGetCommand() case TokenExplain: diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go index cc78abbe96..773a7793a2 100644 --- a/internal/cli/user_command.go +++ b/internal/cli/user_command.go @@ -739,8 +739,8 @@ func (c *CLI) getDatasetID(datasetName string) (string, error) { return "", fmt.Errorf("dataset '%s' not found", datasetName) } -// GetMetadata gets metadata for one or more datasets -func (c *CLI) GetMetadata(cmd *Command) (ResponseIf, error) { +// DevGetMetadataCommand gets metadata for one or more datasets +func (c *CLI) DevGetMetadataCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1465,8 +1465,8 @@ func (c *CLI) APIUnsetAPIKeyCommand(cmd *Command) (ResponseIf, error) { return &result, nil } -// CreateChunkStore creates a chunk store in doc engine -func (c *CLI) CreateChunkStore(cmd *Command) (ResponseIf, error) { +// DevCreateChunkStoreCommand creates a chunk store in doc engine +func (c *CLI) DevCreateChunkStoreCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1522,8 +1522,8 @@ func (c *CLI) CreateChunkStore(cmd *Command) (ResponseIf, error) { return &result, nil } -// DropChunkStore drops a chunk store in doc engine -func (c *CLI) DropChunkStore(cmd *Command) (ResponseIf, error) { +// DevDropChunkStoreCommand drops a chunk store in doc engine +func (c *CLI) DevDropChunkStoreCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1573,8 +1573,8 @@ func (c *CLI) DropChunkStore(cmd *Command) (ResponseIf, error) { return &result, nil } -// CreateMetadataStore creates the document metadata store for the tenant -func (c *CLI) CreateMetadataStore(cmd *Command) (ResponseIf, error) { +// DevCreateMetadataStoreCommand creates the document metadata store for the tenant +func (c *CLI) DevCreateMetadataStoreCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1609,8 +1609,8 @@ func (c *CLI) CreateMetadataStore(cmd *Command) (ResponseIf, error) { return &result, nil } -// DropMetadataStore drops the document metadata store for the tenant -func (c *CLI) DropMetadataStore(cmd *Command) (ResponseIf, error) { +// DevDropMetadataStoreCommand drops the document metadata store for the tenant +func (c *CLI) DevDropMetadataStoreCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1645,13 +1645,18 @@ func (c *CLI) DropMetadataStore(cmd *Command) (ResponseIf, error) { return &result, nil } -// AddProvider creates a new model provider +// APIAddProviderCommand creates a new model provider // ADD PROVIDER -func (c *CLI) AddProvider(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIAddProviderCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } + httpClient := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer] + if httpClient.APIKey == nil && httpClient.LoginToken == nil { + return nil, fmt.Errorf("API key not set. Please login first") + } + providerName, ok := cmd.Params["provider_name"].(string) if !ok { return nil, fmt.Errorf("provider name not provided") @@ -1662,7 +1667,7 @@ func (c *CLI) AddProvider(cmd *Command) (ResponseIf, error) { "provider_name": providerName, } - resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("PUT", "/providers", "web", nil, payload) + resp, err := httpClient.Request("PUT", "/providers", "web", nil, payload) if err != nil { return nil, fmt.Errorf("failed to add provider: %w", err) } @@ -1702,9 +1707,9 @@ func (c *CLI) APIListProviders(cmd *Command) (ResponseIf, error) { return &result, nil } -// DeleteProvider deletes a provider +// APIDeleteProvider deletes a provider // DELETE PROVIDER -func (c *CLI) DeleteProvider(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIDeleteProvider(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -1721,12 +1726,7 @@ func (c *CLI) DeleteProvider(cmd *Command) (ResponseIf, error) { url := fmt.Sprintf("/providers/%s", providerName) - // Build payload - payload := map[string]interface{}{ - "llm_factory": providerName, - } - - resp, err := httpClient.Request("DELETE", url, "web", nil, payload) + resp, err := httpClient.Request("DELETE", url, "web", nil, nil) if err != nil { return nil, fmt.Errorf("failed to delete provider: %w", err) } @@ -1927,9 +1927,9 @@ func (c *CLI) APIDropMemoryCommand(cmd *Command) (ResponseIf, error) { return HandleSimpleResponse(resp, "delete memory") } -// APICreateProviderInstanceCommand creates a new provider instance +// APIAddProviderInstanceCommand creates a new provider instance // CREATE PROVIDER INSTANCE KEY URL REGION -func (c *CLI) APICreateProviderInstanceCommand(cmd *Command) (ResponseIf, error) { +func (c *CLI) APIAddProviderInstanceCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -2026,9 +2026,8 @@ func (c *CLI) ShowInstanceBalance(cmd *Command) (ResponseIf, error) { return &result, nil } -// DropProviderInstance deletes a provider instance -// DROP INSTANCE FROM PROVIDER -func (c *CLI) DropProviderInstance(cmd *Command) (ResponseIf, error) { +// DELETE PROVIDER INSTANCE +func (c *CLI) APIDeleteProviderInstanceCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -2061,9 +2060,8 @@ func (c *CLI) DropProviderInstance(cmd *Command) (ResponseIf, error) { return HandleSimpleResponse(resp, "drop instance") } -// DROP MODEL FROM -// Remove MODEL FROM -func (c *CLI) DropInstanceModel(cmd *Command) (ResponseIf, error) { +// DELETE PROVIDER INSTANCE MODELS +func (c *CLI) APIDeleteProviderInstanceModelCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3229,8 +3227,8 @@ func (c *CLI) AddCustomModel(cmd *Command) (ResponseIf, error) { return HandleSimpleResponse(resp, "add custom model") } -// InsertChunksFromFile inserts chunks from a JSON file -func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) { +// DevInsertChunksFromFileCommand inserts chunks from a JSON file +func (c *CLI) DevInsertChunksFromFileCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3243,7 +3241,7 @@ func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) { "file_path": filePath, } - resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/insert_chunks_from_file", "web", nil, payload) + resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_chunks_from_file", "web", nil, payload) if err != nil { return nil, fmt.Errorf("failed to insert dataset from file: %w", err) } @@ -3273,8 +3271,8 @@ func (c *CLI) InsertChunksFromFile(cmd *Command) (ResponseIf, error) { return &result, nil } -// InsertMetadataFromFile inserts metadata from a JSON file -func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) { +// DevInsertMetadataFromFileCommand inserts metadata from a JSON file +func (c *CLI) DevInsertMetadataFromFileCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3288,7 +3286,7 @@ func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) { "file_path": filePath, } - resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/insert_metadata_from_file", "web", nil, payload) + resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_metadata_from_file", "web", nil, payload) if err != nil { return nil, fmt.Errorf("failed to insert metadata from file: %w", err) } @@ -3318,8 +3316,8 @@ func (c *CLI) InsertMetadataFromFile(cmd *Command) (ResponseIf, error) { return &result, nil } -// UpdateChunk updates a chunk in a dataset -func (c *CLI) UpdateChunk(cmd *Command) (ResponseIf, error) { +// DevUpdateChunkCommand updates a chunk in a dataset +func (c *CLI) DevUpdateChunkCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3391,8 +3389,8 @@ func (c *CLI) UpdateChunk(cmd *Command) (ResponseIf, error) { return &result, nil } -// GetChunk retrieves a chunk by ID -func (c *CLI) GetChunk(cmd *Command) (ResponseIf, error) { +// DevGetChunkCommand retrieves a chunk by ID +func (c *CLI) DevGetChunkCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3434,8 +3432,8 @@ func (c *CLI) GetChunk(cmd *Command) (ResponseIf, error) { return &result, nil } -// SetMeta sets metadata for a document -func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) { +// DevSetMetaCommand sets metadata for a document +func (c *CLI) DevSetMetaCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3455,7 +3453,7 @@ func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) { "meta": metaJSON, } - resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/set_meta", "web", nil, payload) + resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/dev_set_meta", "web", nil, payload) if err != nil { return nil, fmt.Errorf("failed to set metadata: %w", err) } @@ -3485,9 +3483,9 @@ func (c *CLI) SetMeta(cmd *Command) (ResponseIf, error) { return &result, nil } -// DeleteMeta deletes metadata for a document +// DevDeleteMetaCommand deletes metadata for a document // If keys is provided, deletes specific keys; otherwise deletes entire document metadata -func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) { +func (c *CLI) DevDeleteMetaCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3506,7 +3504,7 @@ func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) { payload["keys"] = keysJSON } - resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/delete_meta", "web", nil, payload) + resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/document/dev_delete_meta", "web", nil, payload) if err != nil { return nil, fmt.Errorf("failed to delete metadata: %w", err) } @@ -3536,8 +3534,8 @@ func (c *CLI) DeleteMeta(cmd *Command) (ResponseIf, error) { return &result, nil } -// RmTags removes tags from chunks in a dataset -func (c *CLI) RmTags(cmd *Command) (ResponseIf, error) { +// DevRmTagsCommand removes tags from chunks in a dataset +func (c *CLI) DevRmTagsCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } @@ -3591,8 +3589,8 @@ func (c *CLI) RmTags(cmd *Command) (ResponseIf, error) { return &result, nil } -// RemoveChunks removes chunks from a document -func (c *CLI) RemoveChunks(cmd *Command) (ResponseIf, error) { +// DevRemoveChunksCommand removes chunks from a document +func (c *CLI) DevRemoveChunksCommand(cmd *Command) (ResponseIf, error) { if c.Config.CLIMode != APIMode { return nil, fmt.Errorf("this command is only allowed in USER mode") } diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go index 60f45d07ef..da3ad6832d 100644 --- a/internal/cli/user_parser.go +++ b/internal/cli/user_parser.go @@ -5,7 +5,6 @@ import ( "fmt" "ragflow/internal/common" "regexp" - "strconv" "strings" ) @@ -148,6 +147,8 @@ func (p *Parser) parseAPIListCommands() (*Command, error) { return p.parseAPIListProviders() case TokenProvider: return p.parseAPIListProviderCommands() + case TokenModels: + return p.parseAPIListAllModels() case TokenIngestion: return p.parseAPIListIngestionTasks() case TokenDefault: @@ -333,7 +334,7 @@ func (p *Parser) parseGetMetadata() (*Command, error) { } } - cmd := NewCommand("get_metadata") + cmd := NewCommand("dev_get_metadata") cmd.Params["dataset_names"] = datasetNames // Semicolon is optional @@ -424,11 +425,6 @@ func (p *Parser) parseListProviderInstanceCommands(providerName string) (*Comman func (p *Parser) parseListProviderInstanceModels(providerName, instanceName string) (*Command, error) { p.nextToken() // consume MODELS - if p.curToken.Type != TokenModels { - return nil, fmt.Errorf("expected MODELS") - } - p.nextToken() - cmd := NewCommand("api_list_provider_instance_models") if p.curToken.Type == TokenSync { cmd = NewCommand("api_list_provider_instance_models_sync") @@ -702,11 +698,11 @@ func (p *Parser) parseAPIShowProviderModel(providerName string) (*Command, error return cmd, nil } -// parseListModels parses LIST MODELS -func (p *Parser) parseListAllModels() (*Command, error) { +// LIST MODELS +func (p *Parser) parseAPIListAllModels() (*Command, error) { p.nextToken() // consume models - cmd := NewCommand("list_all_models") + cmd := NewCommand("api_list_all_models") if p.curToken.Type == TokenSemicolon { p.nextToken() @@ -731,7 +727,7 @@ func (p *Parser) parseAPICreateCommands() (*Command, error) { case TokenKey: return p.parseAPICreateKey() case TokenChunkStore: - return p.parseCreateChunkStore() + return p.parseDevCreateChunkStore() case TokenMetadata: return p.parseCreateMetadataStore() case TokenProvider: @@ -741,22 +737,6 @@ func (p *Parser) parseAPICreateCommands() (*Command, error) { } } -func (p *Parser) parseAddCommand() (*Command, error) { - p.nextToken() // consume ADD - switch p.curToken.Type { - case TokenProvider: - return p.parseAddProvider() - case TokenModel: - return p.parseAddModel() - case TokenAPI: - return p.parseAddAPIServer() - case TokenAdmin: - return p.parseAddAdminServer() - default: - return nil, fmt.Errorf("unknown ADD target: %s", p.curToken.Value) - } -} - func (p *Parser) parseAPICreateKey() (*Command, error) { p.nextToken() // consume KEY @@ -768,118 +748,109 @@ func (p *Parser) parseAPICreateKey() (*Command, error) { return NewCommand("api_create_api_key"), nil } -// Internal CLI for GO -// parseCreateChunkStore parses: CREATE CHUNK STORE for Dataset 'name' VECTOR SIZE N -func (p *Parser) parseCreateChunkStore() (*Command, error) { - p.nextToken() // consume CHUNK STORE compound token - - // Expect FOR - if p.curToken.Type != TokenFor { - return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value) +func (p *Parser) parseAPIAddCommands() (*Command, error) { + p.nextToken() // consume ADD + switch p.curToken.Type { + case TokenProvider: + return p.parseAPIAddProvider() + case TokenModel: + return p.parseAPIAddModel() + case TokenAPI: + return p.parseAddAPIServer() + case TokenAdmin: + return p.parseAddAdminServer() + default: + return nil, fmt.Errorf("unknown ADD target: %s", p.curToken.Value) } - p.nextToken() - - // Expect Dataset - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value) - } - p.nextToken() - - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value) - } - - p.nextToken() - if p.curToken.Type != TokenVector { - return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value) - } - p.nextToken() - if p.curToken.Type != TokenSize { - return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value) - } - p.nextToken() - - if p.curToken.Type != TokenInteger { - return nil, fmt.Errorf("expected vector size number, got %s", p.curToken.Value) - } - vectorSize, err := strconv.Atoi(p.curToken.Value) - if err != nil { - return nil, fmt.Errorf("invalid vector size: %s", p.curToken.Value) - } - - p.nextToken() - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("create_chunk_store") - cmd.Params["dataset_name"] = datasetName - cmd.Params["vector_size"] = vectorSize - return cmd, nil } -// Internal CLI for GO -// parseCreateMetadataStore parses: CREATE METADATA STORE -func (p *Parser) parseCreateMetadataStore() (*Command, error) { - // CREATE METADATA STORE - p.nextToken() // consume METADATA - - if p.curToken.Type != TokenStore { - return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value) - } - p.nextToken() - - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return NewCommand("create_metadata_store"), nil -} - -func (p *Parser) parseCreateRole() (*Command, error) { - p.nextToken() // consume ROLE - roleName, err := p.parseIdentifier() - if err != nil { - return nil, err - } - - cmd := NewCommand("create_role") - cmd.Params["role_name"] = roleName - - p.nextToken() - if p.curToken.Type == TokenDescription { - p.nextToken() - description, err := p.parseQuotedString() - if err != nil { - return nil, err - } - cmd.Params["description"] = description - p.nextToken() - } - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -// parseAddProvider parses ADD PROVIDER commands +// parseAPIAddProvider parses ADD PROVIDER commands // ADD PROVIDER -func (p *Parser) parseAddProvider() (*Command, error) { +// ADD PROVIDER INSTANCE KEY [URL | REGION ] +func (p *Parser) parseAPIAddProvider() (*Command, error) { p.nextToken() // consume PROVIDER providerName, err := p.parseQuotedString() if err != nil { return nil, fmt.Errorf("expected provider name: %w", err) } + p.nextToken() - cmd := NewCommand("add_provider") + switch p.curToken.Type { + case TokenInstance: + return p.parseAddProviderInstance(providerName) + case TokenEOF, TokenSemicolon: + p.nextToken() + default: + return nil, fmt.Errorf("unknown ADD PROVIDER target: %s", p.curToken.Value) + } + + cmd := NewCommand("api_add_provider") cmd.Params["provider_name"] = providerName p.nextToken() + return cmd, nil +} +// ADD PROVIDER INSTANCE +func (p *Parser) parseAddProviderInstance(providerName string) (*Command, error) { + p.nextToken() // consume INSTANCE + + instanceName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected instance name: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenKey { + return nil, fmt.Errorf("expected KEY after instance name") + } + p.nextToken() + + apiKey, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected API key: %w", err) + } + p.nextToken() + + baseURL := "" + region := "" +optionsLoop: + for { + switch p.curToken.Type { + case TokenRegion: + p.nextToken() + region, err = p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected region: %w", err) + } + p.nextToken() + case TokenURL: + p.nextToken() + baseURL, err = p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected base URL: %w", err) + } + p.nextToken() + default: + break optionsLoop + } + } + + cmd := NewCommand("api_add_provider_instance") + cmd.Params["provider_name"] = providerName + cmd.Params["instance_name"] = instanceName + cmd.Params["api_key"] = apiKey + if baseURL != "" { + // Only local model provider need to set URL + cmd.Params["base_url"] = baseURL + } + + if region != "" { + cmd.Params["region"] = region + } + + p.nextToken() // Semicolon is optional if p.curToken.Type == TokenSemicolon { p.nextToken() @@ -905,8 +876,8 @@ func (p *Parser) parseModelNames(raw string) ([]string, error) { return modelNames, nil } -// syntax: add model 'xxx' to provider 'vllm' instance 'test' with tokens 1024 chat think vision; -func (p *Parser) parseAddModel() (*Command, error) { +// ADD MODEL 'xxx' TO PROVIDER 'vllm' INSTANCE 'test' TOKENS 1024 CHAT THINK VISION; +func (p *Parser) parseAPIAddModel() (*Command, error) { p.nextToken() // consume MODEL if p.curToken.Type != TokenQuotedString { @@ -966,10 +937,6 @@ func (p *Parser) parseAddModel() (*Command, error) { var dimensions []int = nil models := make([]map[string]any, 0, len(modelNames)) - if p.curToken.Type != TokenWith { - return nil, fmt.Errorf("expected with") - } - p.nextToken() optionsLoop: for { @@ -1044,7 +1011,6 @@ optionsLoop: if p.curToken.Type != TokenInteger { return nil, fmt.Errorf("expected integer") } - var err error maxTokens, err = p.parseNumber() if err != nil { return nil, err @@ -1120,10 +1086,9 @@ optionsLoop: return nil, fmt.Errorf("model config count %d does not match model name count %d", len(models), len(modelNames)) } - cmd := NewCommand("add_custom_model") + cmd := NewCommand("api_add_custom_model") cmd.Params["provider_name"] = providerName cmd.Params["instance_name"] = instanceName - cmd.Params["models"] = models return cmd, nil @@ -1191,7 +1156,7 @@ func (p *Parser) parseAddAPIServer() (*Command, error) { } // syntax: delete api 'abc' -func (p *Parser) parseDeleteAPIServer() (*Command, error) { +func (p *Parser) parseAPIDeleteAPIServer() (*Command, error) { p.nextToken() // consume API serverName, err := p.parseQuotedString() @@ -1212,7 +1177,7 @@ func (p *Parser) parseDeleteAPIServer() (*Command, error) { } // syntax: delete admin server 'abc' -func (p *Parser) parseDeleteAdminServer() (*Command, error) { +func (p *Parser) parseAPIDeleteAdminServer() (*Command, error) { p.nextToken() // consume ADMIN cmd := NewCommand("delete_admin_server") @@ -1374,47 +1339,44 @@ func (p *Parser) parseAPIDropCommands() (*Command, error) { case TokenKey: return p.parseAPIDeleteAPIKey() case TokenChunkStore: - return p.parseDropChunkStore() + return p.parseDevDropChunkStore() case TokenMetadata: - return p.parseDropMetadataStore() - case TokenInstance: - return p.parseDropInstance() - case TokenModel: - return p.parseDropInstanceModel() + return p.parseDevDropMetadataStore() default: return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value) } } -func (p *Parser) parseDeleteCommand() (*Command, error) { +func (p *Parser) parseAPIDeleteCommands() (*Command, error) { p.nextToken() // consume DELETE switch p.curToken.Type { case TokenProvider: - return p.parseDeleteProvider() + return p.parseAPIDeleteProvider() case TokenMetadata: - return p.parseDeleteMeta() + return p.parseDevDeleteMeta() case TokenAdmin: - return p.parseDeleteAdminServer() + return p.parseAPIDeleteAdminServer() case TokenAPI: - return p.parseDeleteAPIServer() + return p.parseAPIDeleteAPIServer() default: return nil, fmt.Errorf("unknown DELETE target: %s", p.curToken.Value) } } -func (p *Parser) parseRemoveCommand() (*Command, error) { - p.nextToken() // consume RM +func (p *Parser) parseAPIRemoveCommands() (*Command, error) { + p.nextToken() // consume REMOVE switch p.curToken.Type { - case TokenTag: - return p.parseRemoveTags() - case TokenChunks, TokenAll: - return p.parseRemoveChunk() + // API commands case TokenIngestion: - return p.parseUserRemoveTask() - case TokenModel: - return p.parseRemoveInstanceModel() + return p.parseAPIRemoveTask() + + // Dev commands + case TokenTag: + return p.parseDevRemoveTags() + case TokenChunks, TokenAll: + return p.parseDevRemoveChunk() default: return nil, fmt.Errorf("unknown REMOVE target: %s", p.curToken.Value) } @@ -1439,67 +1401,83 @@ func (p *Parser) parseAPIDeleteAPIKey() (*Command, error) { return cmd, nil } -// Internal CLI for GO -// parseDropChunkStore parses: DROP CHUNK STORE for Dataset 'name' -func (p *Parser) parseDropChunkStore() (*Command, error) { - p.nextToken() // consume CHUNK STORE - - // Expect FOR - if p.curToken.Type != TokenFor { - return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value) - } - p.nextToken() - - // Expect Dataset - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value) - } - p.nextToken() - - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value) - } - - p.nextToken() - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("drop_chunk_store") - cmd.Params["dataset_name"] = datasetName - return cmd, nil -} - -// parseDropMetadataStore parses: DROP METADATA STORE -func (p *Parser) parseDropMetadataStore() (*Command, error) { - // DROP METADATA STORE - p.nextToken() // consume METADATA - - if p.curToken.Type != TokenStore { - return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value) - } - p.nextToken() - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - cmd := NewCommand("drop_metadata_store") - return cmd, nil -} - -// parseDeleteProvider parses DELETE PROVIDER command -func (p *Parser) parseDeleteProvider() (*Command, error) { +// DELETE PROVIDER +// DELETE PROVIDER INSTANCE +func (p *Parser) parseAPIDeleteProvider() (*Command, error) { p.nextToken() // consume PROVIDER providerName, err := p.parseQuotedString() if err != nil { return nil, fmt.Errorf("expected provider name: %w", err) } + p.nextToken() - cmd := NewCommand("delete_provider") + switch p.curToken.Type { + case TokenInstance: + return p.parseAPIDeleteProviderInstance(providerName) + case TokenEOF, TokenSemicolon: + p.nextToken() + default: + return nil, fmt.Errorf("unknown DELETE target: %s", p.curToken.Value) + } + + cmd := NewCommand("api_delete_provider") cmd.Params["provider_name"] = providerName + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +func (p *Parser) parseAPIDeleteProviderInstance(providerName string) (*Command, error) { + p.nextToken() // consume INSTANCE + + instanceName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected instance name: %w", err) + } + p.nextToken() + + cmd := NewCommand("api_delete_provider_instance") + cmd.Params["provider_name"] = providerName + cmd.Params["instance_name"] = instanceName + + switch p.curToken.Type { + case TokenModels: + return p.parseAPIDeleteProviderInstanceModels(providerName, instanceName) + case TokenEOF, TokenSemicolon: + p.nextToken() + default: + return nil, fmt.Errorf("unknown DELETE target: %s", p.curToken.Value) + } + + // Semicolon is optional + if p.curToken.Type == TokenSemicolon { + p.nextToken() + } + return cmd, nil +} + +// DELETE PROVIDER INSTANCE MODELS +// Only works for local deployed model +func (p *Parser) parseAPIDeleteProviderInstanceModels(providerName, instanceName string) (*Command, error) { + p.nextToken() // consume MODEL + + rawModels, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected model name: %w", err) + } + p.nextToken() + + modelNames := strings.Split(rawModels, " ") + + cmd := NewCommand("api_delete_provider_instance_model") + cmd.Params["instance_name"] = instanceName + cmd.Params["provider_name"] = providerName + cmd.Params["model_names"] = modelNames + p.nextToken() // Semicolon is optional if p.curToken.Type == TokenSemicolon { @@ -1598,51 +1576,17 @@ func (p *Parser) parseAPIDropAgent() (*Command, error) { return cmd, nil } -func (p *Parser) parseAlterCommand() (*Command, error) { +func (p *Parser) parseAPIAlterCommands() (*Command, error) { p.nextToken() // consume ALTER switch p.curToken.Type { case TokenProvider: - return p.parseAlterProvider() - case TokenInstance: - return p.parseAlterInstance() + return p.parseAPIAlterInstance() default: return nil, fmt.Errorf("unknown ALTER target: %s", p.curToken.Value) } } -// parseAlterProvider parses ALTER PROVIDER NAME command -func (p *Parser) parseAlterProvider() (*Command, error) { - p.nextToken() // consume PROVIDER - - providerName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected provider name: %w", err) - } - - p.nextToken() - if p.curToken.Type != TokenName { - return nil, fmt.Errorf("expected NAME") - } - p.nextToken() - - newName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected new provider name: %w", err) - } - - cmd := NewCommand("alter_provider") - cmd.Params["provider_name"] = providerName - cmd.Params["new_name"] = newName - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - // CREATE PROVIDER INSTANCE KEY URL REGION func (p *Parser) parseAPICreateProviderInstance() (*Command, error) { p.nextToken() // consume PROVIDER @@ -1720,9 +1664,20 @@ optionsLoop: return cmd, nil } -// parseAlterInstance parses ALTER INSTANCE NAME FROM PROVIDER command -func (p *Parser) parseAlterInstance() (*Command, error) { - p.nextToken() // consume INSTANCE +// ALTER PROVIDER INSTANCE [NAME | KEY ] +func (p *Parser) parseAPIAlterInstance() (*Command, error) { + p.nextToken() // consume PROVIDER + + providerName, err := p.parseQuotedString() + if err != nil { + return nil, fmt.Errorf("expected provider name: %w", err) + } + p.nextToken() + + if p.curToken.Type != TokenInstance { + return nil, fmt.Errorf("expected INSTANCE after provider name") + } + p.nextToken() instanceName, err := p.parseQuotedString() if err != nil { @@ -1730,14 +1685,14 @@ func (p *Parser) parseAlterInstance() (*Command, error) { } p.nextToken() - newModelName := "" + newInstanceName := "" newAPIKey := "" optionsLoop: for { switch p.curToken.Type { case TokenName: p.nextToken() - newModelName, err = p.parseQuotedString() + newInstanceName, err = p.parseQuotedString() if err != nil { return nil, fmt.Errorf("expected model name: %w", err) } @@ -1754,29 +1709,14 @@ optionsLoop: } } - if newModelName == "" && newAPIKey == "" { + if newInstanceName == "" && newAPIKey == "" { return nil, fmt.Errorf("expected NAME or KEY after INSTANCE") } - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM") - } - p.nextToken() - - if p.curToken.Type != TokenProvider { - return nil, fmt.Errorf("expected PROVIDER after FROM") - } - p.nextToken() - - providerName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected provider name after FROM PROVIDER: %w", err) - } - - cmd := NewCommand("alter_provider_instance") + cmd := NewCommand("api_alter_provider_instance") cmd.Params["provider_name"] = providerName cmd.Params["instance_name"] = instanceName - cmd.Params["new_model_name"] = newModelName + cmd.Params["new_instance_name"] = newInstanceName cmd.Params["new_api_key"] = newAPIKey p.nextToken() @@ -1787,87 +1727,6 @@ optionsLoop: return cmd, nil } -// parseDropInstance parses DROP INSTANCE FROM PROVIDER command -func (p *Parser) parseDropInstance() (*Command, error) { - p.nextToken() // consume INSTANCE - - instanceName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected instance name: %w", err) - } - - p.nextToken() - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM") - } - p.nextToken() - - providerName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected provider name after FROM PROVIDER: %w", err) - } - - cmd := NewCommand("drop_provider_instance") - cmd.Params["instance_name"] = instanceName - cmd.Params["provider_name"] = providerName - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -func (p *Parser) parseRemoveInstanceModel() (*Command, error) { - return p.parseDropInstanceModel() -} - -// parseDropInstanceModel parses DROP MODEL FROM command -// Only works for local deployed model -func (p *Parser) parseDropInstanceModel() (*Command, error) { - p.nextToken() // consume MODEL - - rawModelNames, err := p.parseQuotedString() - if err != nil { - return nil, err - } - modelNames, err := p.parseModelNames(rawModelNames) - if err != nil { - return nil, err - } - p.nextToken() // consume model name - - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM") - } - p.nextToken() - - providerName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected provider name after FROM PROVIDER: %w", err) - } - p.nextToken() - - instanceName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected instance name after provider name: %w", err) - } - p.nextToken() - - cmd := NewCommand("drop_instance_model") - cmd.Params["instance_name"] = instanceName - cmd.Params["provider_name"] = providerName - cmd.Params["model_names"] = modelNames - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - func (p *Parser) parseIdentifierList() ([]string, error) { var list []string @@ -2154,88 +2013,6 @@ func (p *Parser) parseImportCommand() (*Command, error) { return cmd, nil } -// parseInsertCommand parses INSERT command and dispatches to specific handler -func (p *Parser) parseInsertCommand() (*Command, error) { - p.nextToken() // consume INSERT - - // Expect CHUNKS or METADATA - if p.curToken.Type == TokenChunks { - return p.parseInsertChunksFromFile() - } - if p.curToken.Type == TokenMetadata { - return p.parseInsertMetadataFromFile() - } - return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value) -} - -// Internal CLI for GO -// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path" -func (p *Parser) parseInsertChunksFromFile() (*Command, error) { - p.nextToken() // consume CHUNKS - - // Expect FROM - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value) - } - p.nextToken() - - // Expect FILE - if p.curToken.Type != TokenFile { - return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value) - } - p.nextToken() - - // Get file path (quoted string) - filePath, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("insert_chunks_from_file") - cmd.Params["file_path"] = filePath - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - -// Internal CLI for GO -// parseInsertMetadataFromFile parses: INSERT METADATA FROM FILE "file_path" -func (p *Parser) parseInsertMetadataFromFile() (*Command, error) { - p.nextToken() // consume METADATA - - // Expect FROM - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value) - } - p.nextToken() - - // Expect FILE - if p.curToken.Type != TokenFile { - return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value) - } - p.nextToken() - - // Get file path (quoted string) - filePath, err := p.parseQuotedString() - if err != nil { - return nil, err - } - - cmd := NewCommand("insert_metadata_from_file") - cmd.Params["file_path"] = filePath - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - return cmd, nil -} - func (p *Parser) parseRetrieveCommand() (*Command, error) { p.nextToken() // consume SEARCH @@ -3352,10 +3129,6 @@ func (p *Parser) parseAPIUseModel() (*Command, error) { return nil, fmt.Errorf("invalid format of model name or ID: %s", modelNameOrID) } - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } return cmd, nil } @@ -3422,12 +3195,12 @@ func (p *Parser) parseParseDataset() (*Command, error) { } else { return nil, fmt.Errorf("expected SYNC or ASYNC") } + p.nextToken() cmd := NewCommand("parse_dataset") cmd.Params["dataset_name"] = datasetName cmd.Params["method"] = method - p.nextToken() // Semicolon is optional if p.curToken.Type == TokenSemicolon { p.nextToken() @@ -3583,8 +3356,6 @@ func (p *Parser) parseUserStatement() (*Command, error) { switch p.curToken.Type { case TokenPing: return p.parseAPIPingServer() - case TokenDelete: - return p.parseDeleteCommand() case TokenShow: return p.parseAPIShowCommands() case TokenList: @@ -3594,428 +3365,16 @@ func (p *Parser) parseUserStatement() (*Command, error) { case TokenImport: return p.parseImportCommand() case TokenInsert: - return p.parseInsertCommand() + return p.parseDevInsertCommand() case TokenRetrieve: return p.parseRetrieveCommand() case TokenGet: return p.parseGetCommand() - case TokenUpdate: - return p.parseUpdateCommand() - case TokenRemove: - return p.parseRemoveCommand() default: return nil, fmt.Errorf("invalid user statement: %s", p.curToken.Value) } } -// parseGetCommand parses: GET CHUNK or GET METADATA -func (p *Parser) parseGetCommand() (*Command, error) { - p.nextToken() // consume GET - - if p.curToken.Type == TokenChunk { - return p.parseGetChunk() - } - if p.curToken.Type == TokenMetadata { - return p.parseGetMetadata() - } - - return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value) -} - -// parseGetChunk parses: GET CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' -func (p *Parser) parseGetChunk() (*Command, error) { - p.nextToken() // consume CHUNK - - // Parse chunk_id - chunkID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected chunk_id: %w", err) - } - - cmd := NewCommand("get_chunk") - cmd.Params["chunk_id"] = chunkID - - p.nextToken() - if p.curToken.Type != TokenOf { - return nil, fmt.Errorf("expected OF after chunk_id") - } - p.nextToken() - - if p.curToken.Type != TokenDocument { - return nil, fmt.Errorf("expected DOCUMENT after OF") - } - p.nextToken() - - // Parse doc_id - docID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected doc_id: %w", err) - } - cmd.Params["doc_id"] = docID - - p.nextToken() - if p.curToken.Type != TokenIn { - return nil, fmt.Errorf("expected IN after doc_id") - } - p.nextToken() - - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected DATASET after IN") - } - p.nextToken() - - // Parse dataset_id - datasetID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset_id: %w", err) - } - cmd.Params["dataset_id"] = datasetID - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// Internal -// parseUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}' -func (p *Parser) parseUpdateCommand() (*Command, error) { - p.nextToken() // consume UPDATE - - if p.curToken.Type == TokenChunk { - return p.parseUpdateChunk() - } - - return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value) -} - -// Internal CLI for GO -// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' SET '{"content": "..."}' -func (p *Parser) parseUpdateChunk() (*Command, error) { - p.nextToken() // consume CHUNK - - // Parse chunk_id - chunkID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected chunk_id: %w", err) - } - - cmd := NewCommand("update_chunk") - cmd.Params["chunk_id"] = chunkID - - p.nextToken() - if p.curToken.Type != TokenOf { - return nil, fmt.Errorf("expected OF after chunk_id") - } - p.nextToken() - - if p.curToken.Type != TokenDocument { - return nil, fmt.Errorf("expected DOCUMENT after OF") - } - p.nextToken() - - // Parse doc_id - docID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected doc_id: %w", err) - } - cmd.Params["doc_id"] = docID - - p.nextToken() - if p.curToken.Type != TokenIn { - return nil, fmt.Errorf("expected IN after doc_id") - } - p.nextToken() - - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected DATASET after IN") - } - p.nextToken() - - // Parse dataset_name - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset_name: %w", err) - } - cmd.Params["dataset_name"] = datasetName - - p.nextToken() - if p.curToken.Type != TokenSet { - return nil, fmt.Errorf("expected SET after dataset_name") - } - p.nextToken() - - // Parse JSON body - jsonBody, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected JSON body: %w", err) - } - cmd.Params["json_body"] = jsonBody - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// parseSetMeta parses: SET METADATA OF DOCUMENT 'doc_id' TO '{"key": "value"}' -func (p *Parser) parseSetMeta() (*Command, error) { - p.nextToken() // consume METADATA - - // Expect OF - if p.curToken.Type != TokenOf { - return nil, fmt.Errorf("expected OF after SET METADATA") - } - p.nextToken() - - // Expect DOCUMENT - if p.curToken.Type != TokenDocument { - return nil, fmt.Errorf("expected DOCUMENT after SET METADATA OF") - } - p.nextToken() - - // Parse doc_id - docID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected doc_id: %w", err) - } - cmd := NewCommand("set_meta") - cmd.Params["doc_id"] = docID - - p.nextToken() - // Expect TO - if p.curToken.Type != TokenTo { - return nil, fmt.Errorf("expected TO after doc_id") - } - p.nextToken() - - // Parse meta JSON - meta, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected meta JSON: %w", err) - } - cmd.Params["meta"] = meta - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// parseDeleteMeta parses: DELETE METADATA OF DOCUMENT 'doc_id' [KEYS '["key1", "key2"]'] -// If KEYS is not provided, deletes entire document metadata -func (p *Parser) parseDeleteMeta() (*Command, error) { - p.nextToken() // consume METADATA - - // Expect OF - if p.curToken.Type != TokenOf { - return nil, fmt.Errorf("expected OF after DELETE METADATA") - } - p.nextToken() - - // Expect DOCUMENT - if p.curToken.Type != TokenDocument { - return nil, fmt.Errorf("expected DOCUMENT after DELETE METADATA OF") - } - p.nextToken() - - // Parse doc_id - docID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected doc_id: %w", err) - } - cmd := NewCommand("delete_meta") - cmd.Params["doc_id"] = docID - - p.nextToken() - // KEYS is optional - if not provided, delete entire document metadata - if p.curToken.Type != TokenKeys { - if p.curToken.Type == TokenSemicolon { - p.nextToken() - return cmd, nil - } - if p.curToken.Type == TokenEOF { - return cmd, nil - } - return nil, fmt.Errorf("expected KEYS or end of command after doc_id") - } - - // Parse keys JSON array - p.nextToken() - keys, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected keys JSON array: %w", err) - } - cmd.Params["keys"] = keys - - p.nextToken() - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - return cmd, nil - } - if p.curToken.Type != TokenEOF { - return nil, fmt.Errorf("expected end of command after KEYS") - } - - return cmd, nil -} - -// parseRemoveTags parses: REMOVE TAGS 'tag1', 'tag2' from DATASET 'dataset_name'; -func (p *Parser) parseRemoveTags() (*Command, error) { - p.nextToken() // consume TAGS - - // Parse first tag - tag, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected tag: %w", err) - } - tags := []string{tag} - - // Parse additional tags separated by commas - for { - p.nextToken() - if p.curToken.Type == TokenComma { - p.nextToken() - tag, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected tag after comma: %w", err) - } - tags = append(tags, tag) - } else { - break - } - } - cmd := NewCommand("rm_tags") - cmd.Params["tags"] = tags - - // Expect from - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM after tags") - } - p.nextToken() - - // Expect DATASET - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected DATASET after FROM") - } - p.nextToken() - - // Parse dataset_name - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset_name: %w", err) - } - cmd.Params["dataset_name"] = datasetName - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - -// parseRemoveChunk parses: -// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name'; -// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name'; -func (p *Parser) parseRemoveChunk() (*Command, error) { - cmd := NewCommand("remove_chunks") - - // Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL - if p.curToken.Type == TokenAll { - p.nextToken() // consume ALL - if p.curToken.Type != TokenChunks { - return nil, fmt.Errorf("expected CHUNKS after ALL") - } - p.nextToken() // consume CHUNKS - cmd.Params["delete_all"] = true - } else { - // curToken is TokenChunks, consume it first - p.nextToken() - // Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name' (space-separated) - // Parse first chunk ID - chunkID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected chunk_id: %w", err) - } - chunkIDs := []string{chunkID} - - // Parse additional chunk IDs separated by spaces (each quoted) - for { - p.nextToken() - // Stop if we hit FROM or non-quoted token - if p.curToken.Type == TokenFrom || p.curToken.Type != TokenQuotedString { - break - } - chunkID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected chunk_id: %w", err) - } - chunkIDs = append(chunkIDs, chunkID) - } - cmd.Params["chunk_ids"] = chunkIDs - } - - // Expect FROM - if p.curToken.Type != TokenFrom { - return nil, fmt.Errorf("expected FROM after chunk(s)") - } - p.nextToken() - - // Expect DOCUMENT - if p.curToken.Type != TokenDocument { - return nil, fmt.Errorf("expected DOCUMENT after FROM") - } - p.nextToken() - - // Parse doc_id - docID, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected doc_id: %w", err) - } - cmd.Params["doc_id"] = docID - - p.nextToken() - - // Expect IN - if p.curToken.Type != TokenIn { - return nil, fmt.Errorf("expected IN after doc_id") - } - p.nextToken() - - // Expect DATASET - if p.curToken.Type != TokenDataset { - return nil, fmt.Errorf("expected DATASET after IN") - } - p.nextToken() - - // Parse dataset_name (quoted string) - datasetName, err := p.parseQuotedString() - if err != nil { - return nil, fmt.Errorf("expected dataset_name: %w", err) - } - cmd.Params["dataset_name"] = datasetName - - p.nextToken() - - // Semicolon is optional - if p.curToken.Type == TokenSemicolon { - p.nextToken() - } - - return cmd, nil -} - func (p *Parser) parseUserStartIngestion() (*Command, error) { p.nextToken() // consume Start @@ -4129,7 +3488,7 @@ func (p *Parser) parseAPIListIngestionTasks() (*Command, error) { return cmd, nil } -func (p *Parser) parseUserRemoveTask() (*Command, error) { +func (p *Parser) parseAPIRemoveTask() (*Command, error) { p.nextToken() // consume Ingestion if p.curToken.Type != TokenTasks { diff --git a/internal/cli/user_parser_test.go b/internal/cli/user_parser_test.go index 99c347ad9f..40404dfe77 100644 --- a/internal/cli/user_parser_test.go +++ b/internal/cli/user_parser_test.go @@ -40,9 +40,9 @@ func TestParseAddModelWithDimensions(t *testing.T) { }{ { name: "Add model with detailed embedding dimensions", - input: "add model 'x1 x2 x3 x4 x5' to provider 'vllm' instance 'test' with tokens 1024 chat think vision, token 2048 chat, token 1024 think vision, token 0 embedding 2048 64 1024 2048, token 0 embedding 2048;", + input: "add model 'x1 x2 x3 x4 x5' to provider 'vllm' instance 'test' tokens 1024 chat think vision, token 2048 chat, token 1024 think vision, token 0 embedding 2048 64 1024 2048, token 0 embedding 2048;", expected: &Command{ - Type: "add_custom_model", + Type: "api_add_custom_model", Params: map[string]interface{}{ "provider_name": "vllm", "instance_name": "test", diff --git a/internal/handler/providers.go b/internal/handler/providers.go index d50fdc0fe6..290555ca1b 100644 --- a/internal/handler/providers.go +++ b/internal/handler/providers.go @@ -558,8 +558,8 @@ func (h *ProviderHandler) ShowTask(c *gin.Context) { } type AlterProviderInstanceRequest struct { - ModelName string `json:"model_name"` - APIKey string `json:"api_key"` + InstanceName string `json:"instance_name"` + APIKey string `json:"api_key"` } func (h *ProviderHandler) AlterProviderInstance(c *gin.Context) { @@ -599,7 +599,7 @@ func (h *ProviderHandler) AlterProviderInstance(c *gin.Context) { return } - code, err := h.modelProviderService.AlterProviderInstance(userID, providerName, instanceName, req.ModelName, req.APIKey) + code, err := h.modelProviderService.AlterProviderInstance(userID, providerName, instanceName, req.InstanceName, req.APIKey) if err != nil { c.JSON(http.StatusOK, gin.H{ "code": code, diff --git a/internal/service/model_service.go b/internal/service/model_service.go index fef2eb6557..e59b5cc8e1 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -32,39 +32,6 @@ import ( "gorm.io/gorm" ) -var providerCatalogNameByCanonical = map[string]string{ - "SILICONFLOW": "SiliconFlow", -} - -func canonicalProviderName(name string) (string, error) { - trimmed := strings.TrimSpace(name) - if trimmed == "" { - return "", fmt.Errorf("provider name is required") - } - factoryDAO := dao.NewLLMFactoryDAO() - if _, err := factoryDAO.GetByName(trimmed); err == nil { - return trimmed, nil - } else if !errors.Is(err, gorm.ErrRecordNotFound) { - return "", err - } - for canonical, catalog := range providerCatalogNameByCanonical { - if trimmed == catalog { - if _, err := factoryDAO.GetByName(canonical); err != nil { - return "", err - } - return canonical, nil - } - } - return "", fmt.Errorf("provider '%s' not found", name) -} - -func catalogProviderName(name string) string { - if mapped, ok := providerCatalogNameByCanonical[name]; ok { - return mapped - } - return name -} - // parseModelName parses a composite model name in format "model@instance@provider" or "model@provider" // Returns modelName, instanceName, providerName separately func parseModelName(compositeName string) (modelName, instanceName, providerName string, err error) { @@ -135,10 +102,8 @@ type CheckConnectionRequest struct { } func (m *ModelProviderService) AddModelProvider(providerName, userID string) (common.ErrorCode, error) { - canonicalName, err := canonicalProviderName(providerName) - if err != nil { - return common.CodeNotFound, err - } + providerName = strings.TrimSpace(providerName) + providerName = strings.ToLower(providerName) tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner") if err != nil { @@ -155,7 +120,7 @@ func (m *ModelProviderService) AddModelProvider(providerName, userID string) (co tenantModelProvider := &entity.TenantModelProvider{ ID: providerID, - ProviderName: canonicalName, + ProviderName: providerName, TenantID: tenantID, } err = m.modelProviderDAO.Create(tenantModelProvider) @@ -220,7 +185,7 @@ func isExcludedTenantProvider(name string) bool { return false } -func (m *ModelProviderService) DeleteModelProvider(providerName, userID string) (common.ErrorCode, error) { +func (m *ModelProviderService) DeleteModelProvider(userID, providerName string) (common.ErrorCode, error) { tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner") if err != nil { return common.CodeServerError, err @@ -239,10 +204,8 @@ func (m *ModelProviderService) DeleteModelProvider(providerName, userID string) } func (m *ModelProviderService) ListSupportedModels(providerName, instanceName, userID string) ([]map[string]interface{}, error) { - providerName, err := canonicalProviderName(providerName) - if err != nil { - return nil, err - } + providerName = strings.TrimSpace(providerName) + providerName = strings.ToLower(providerName) // Get tenant ID from user tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner") @@ -267,7 +230,7 @@ func (m *ModelProviderService) ListSupportedModels(providerName, instanceName, u return nil, err } - providerInfo := dao.GetModelProviderManager().FindProvider(catalogProviderName(providerName)) + providerInfo := dao.GetModelProviderManager().FindProvider(providerName) if providerInfo == nil { return nil, fmt.Errorf("provider %s not found", providerName) } @@ -317,10 +280,9 @@ func (m *ModelProviderService) ListSupportedModels(providerName, instanceName, u } func (m *ModelProviderService) CreateProviderInstance(providerName, instanceName, apiKey, baseURL, region, userID string) (common.ErrorCode, error) { - providerName, err := canonicalProviderName(providerName) - if err != nil { - return common.CodeNotFound, err - } + providerName = strings.TrimSpace(providerName) + providerName = strings.ToLower(providerName) + // Get tenant ID from user tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner") if err != nil { @@ -368,10 +330,8 @@ func (m *ModelProviderService) CreateProviderInstance(providerName, instanceName } func (m *ModelProviderService) ListProviderInstances(providerName, userID string) ([]map[string]interface{}, common.ErrorCode, error) { - providerName, err := canonicalProviderName(providerName) - if err != nil { - return nil, common.CodeNotFound, err - } + providerName = strings.TrimSpace(providerName) + providerName = strings.ToLower(providerName) // Get tenant ID from user tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner")