zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-13 16:38:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement Elasticsearch functions in GO (#15160)

Browse Source 
### What problem does this PR solve?

Implement Elasticsearch functions in GO (except for Search)

### Type of change

- [x] Refactoring
This commit is contained in:
qinling0210
2026-05-25 19:15:07 +08:00
committed by GitHub
parent 7d200d5bd7
commit af85aa9c7b
21 changed files with 2418 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/cli/client.go
View File

@@ -309,17 +309,16 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.ListTasksUserCommand(cmd)
case "show_task_user_command":
return c.ShowTaskUserCommand(cmd)
// Dataset, metadata commands
case "create_dataset_table":
return c.CreateDatasetInDocEngine(cmd)
case "drop_dataset_table":
return c.DropDatasetInDocEngine(cmd)
case "create_metadata_table":
return c.CreateMetadataInDocEngine(cmd)
case "drop_metadata_table":
return c.DropMetadataInDocEngine(cmd)
case "insert_dataset_from_file":
return c.InsertDatasetFromFile(cmd)
case "create_chunk_store":
return c.CreateChunkStore(cmd)
case "drop_chunk_store":
return c.DropChunkStore(cmd)
case "create_metadata_store":
return c.CreateMetadataStore(cmd)
case "drop_metadata_store":
return c.DropMetadataStore(cmd)
case "insert_chunks_from_file":
return c.InsertChunksFromFile(cmd)
case "insert_metadata_from_file":
return c.InsertMetadataFromFile(cmd)
case "update_chunk":
@@ -328,6 +327,8 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.GetChunk(cmd)
case "set_meta":
return c.SetMeta(cmd)
case "delete_meta":
return c.DeleteMeta(cmd)
case "rm_tags":
return c.RmTags(cmd)
case "remove_chunks":

28
internal/cli/lexer.go
View File

@@ -222,18 +222,6 @@ func (l *Lexer) lookupIdent(ident string) Token {
case "PASSWORD":
return Token{Type: TokenPassword, Value: ident}
case "DATASET":
// Check if followed by TABLE for compound token
if strings.ToUpper(l.peekToken()) == "TABLE" {
// Skip whitespace to TABLE
for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
l.readChar()
}
// Skip past TABLE
for isLetter(l.ch) || isDigit(l.ch) || l.ch == '_' || l.ch == '-' || l.ch == '.' {
l.readChar()
}
return Token{Type: TokenDatasetTable, Value: "DATASET TABLE"}
}
return Token{Type: TokenDataset, Value: ident}
case "DATASETS":
return Token{Type: TokenDatasets, Value: ident}
@@ -327,6 +315,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenHigh, Value: ident}
case "MAX":
return Token{Type: TokenMax, Value: ident}
case "STORE":
return Token{Type: TokenStore, Value: ident}
case "STREAM":
return Token{Type: TokenStream, Value: ident}
case "LS":
@@ -430,6 +420,18 @@ func (l *Lexer) lookupIdent(ident string) Token {
case "REMOVE":
return Token{Type: TokenRemove, Value: ident}
case "CHUNK":
// Check if followed by STORE for compound token
if strings.ToUpper(l.peekToken()) == "STORE" {
// Skip whitespace to STORE
for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
l.readChar()
}
// Skip past STORE
for isLetter(l.ch) || isDigit(l.ch) || l.ch == '_' || l.ch == '-' || l.ch == '.' {
l.readChar()
}
return Token{Type: TokenChunkStore, Value: "CHUNK STORE"}
}
return Token{Type: TokenChunk, Value: ident}
case "CHUNKS":
return Token{Type: TokenChunks, Value: ident}
@@ -465,6 +467,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenDebug, Value: ident}
case "INFO":
return Token{Type: TokenInfo, Value: ident}
case "IN":
return Token{Type: TokenIn, Value: ident}
case "WARN":
return Token{Type: TokenWarn, Value: ident}
case "ERROR":

4
internal/cli/types.go
View File

@@ -47,7 +47,7 @@ const (
TokenPassword
TokenDataset
TokenDatasets
TokenDatasetTable
TokenChunkStore
TokenOf
TokenAgents
TokenRole
@@ -91,6 +91,7 @@ const (
TokenParse
TokenImport
TokenInto
TokenIn
TokenWith
TokenParser
TokenPipeline
@@ -122,6 +123,7 @@ const (
TokenIndex
TokenVector
TokenSize
TokenStore
TokenName // For ALTER PROVIDER <name> NAME <new_name>
TokenBalance
TokenInstance

229
internal/cli/user_command.go
View File

@@ -875,8 +875,8 @@ func (c *RAGFlowClient) UnsetToken(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// CreateDataset creates a table for a dataset
func (c *RAGFlowClient) CreateDataset(cmd *Command) (ResponseIf, error) {
// CreateChunkStore creates a chunk store in doc engine
func (c *RAGFlowClient) CreateChunkStore(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -902,13 +902,13 @@ func (c *RAGFlowClient) CreateDataset(cmd *Command) (ResponseIf, error) {
"vector_size": vectorSize,
}
resp, err := c.HTTPClient.Request("POST", "/kb/doc_engine_table", "web", nil, payload)
resp, err := c.HTTPClient.Request("POST", "/tenant/chunk_store", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to create table: %w", err)
return nil, fmt.Errorf("failed to create chunk store: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to create table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
return nil, fmt.Errorf("failed to create chunk store: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
@@ -924,73 +924,16 @@ func (c *RAGFlowClient) CreateDataset(cmd *Command) (ResponseIf, error) {
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to create table for dataset: %s", datasetName)
result.Message = fmt.Sprintf("Success to create chunk store for dataset: %s", datasetName)
} else {
result.Message = fmt.Sprintf("Failed to create table: %v", resJSON)
result.Message = fmt.Sprintf("Failed to create chunk store: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// CreateDatasetInDocEngine creates a table for a dataset in doc engine
func (c *RAGFlowClient) CreateDatasetInDocEngine(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
vectorSize, ok := cmd.Params["vector_size"].(int)
if !ok {
return nil, fmt.Errorf("vector_size not provided")
}
// Get dataset ID by name
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, err
}
payload := map[string]interface{}{
"kb_id": datasetID,
"vector_size": vectorSize,
}
resp, err := c.HTTPClient.Request("POST", "/kb/doc_engine_table", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to create table: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to create table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to create table for dataset: %s", datasetName)
} else {
result.Message = fmt.Sprintf("Failed to create table: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// DropDatasetInDocEngine drops a table for a dataset in doc engine
func (c *RAGFlowClient) DropDatasetInDocEngine(cmd *Command) (ResponseIf, error) {
// DropChunkStore drops a chunk store in doc engine
func (c *RAGFlowClient) DropChunkStore(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -1010,7 +953,7 @@ func (c *RAGFlowClient) DropDatasetInDocEngine(cmd *Command) (ResponseIf, error)
"kb_id": datasetID,
}
resp, err := c.HTTPClient.Request("DELETE", "/kb/doc_engine_table", "web", nil, payload)
resp, err := c.HTTPClient.Request("DELETE", "/tenant/chunk_store", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to drop dataset: %w", err)
}
@@ -1032,27 +975,27 @@ func (c *RAGFlowClient) DropDatasetInDocEngine(cmd *Command) (ResponseIf, error)
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to drop table for dataset: %s", datasetName)
result.Message = fmt.Sprintf("Success to drop chunk store for dataset: %s", datasetName)
} else {
result.Message = fmt.Sprintf("Failed to drop table for dataset: %s: %v", datasetName, resJSON)
result.Message = fmt.Sprintf("Failed to drop chunk store for dataset: %s: %v", datasetName, resJSON)
}
result.Duration = 0
return &result, nil
}
// CreateMetadataInDocEngine creates the document metadata table for the tenant
func (c *RAGFlowClient) CreateMetadataInDocEngine(cmd *Command) (ResponseIf, error) {
// CreateMetadataStore creates the document metadata store for the tenant
func (c *RAGFlowClient) CreateMetadataStore(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
resp, err := c.HTTPClient.Request("POST", "/tenant/doc_engine_metadata_table", "web", nil, nil)
resp, err := c.HTTPClient.Request("POST", "/tenant/metadata_store", "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to create metadata table: %w", err)
return nil, fmt.Errorf("failed to create metadata store: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to create metadata table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
return nil, fmt.Errorf("failed to create metadata store: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
@@ -1068,27 +1011,27 @@ func (c *RAGFlowClient) CreateMetadataInDocEngine(cmd *Command) (ResponseIf, err
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = "Success to create metadata table"
result.Message = "Success to create metadata store"
} else {
result.Message = fmt.Sprintf("Failed to create metadata table: %v", resJSON)
result.Message = fmt.Sprintf("Failed to create metadata store: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// DropMetadataInDocEngine drops the document metadata table for the tenant
func (c *RAGFlowClient) DropMetadataInDocEngine(cmd *Command) (ResponseIf, error) {
// DropMetadataStore drops the document metadata store for the tenant
func (c *RAGFlowClient) DropMetadataStore(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
resp, err := c.HTTPClient.Request("DELETE", "/tenant/doc_engine_metadata_table", "web", nil, nil)
resp, err := c.HTTPClient.Request("DELETE", "/tenant/metadata_store", "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to drop metadata table: %w", err)
return nil, fmt.Errorf("failed to drop metadata store: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to drop metadata table: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
return nil, fmt.Errorf("failed to drop metadata store: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
@@ -1104,9 +1047,9 @@ func (c *RAGFlowClient) DropMetadataInDocEngine(cmd *Command) (ResponseIf, error
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = "Success to drop metadata table"
result.Message = "Success to drop metadata store"
} else {
result.Message = fmt.Sprintf("Failed to drop metadata table: %v", resJSON)
result.Message = fmt.Sprintf("Failed to drop metadata store: %v", resJSON)
}
result.Duration = 0
return &result, nil
@@ -2838,8 +2781,8 @@ func (c *RAGFlowClient) CESearch(cmd *Command) (ResponseIf, error) {
return &response, nil
}
// InsertDatasetFromFile inserts dataset chunks from a JSON file
func (c *RAGFlowClient) InsertDatasetFromFile(cmd *Command) (ResponseIf, error) {
// InsertChunksFromFile inserts chunks from a JSON file
func (c *RAGFlowClient) InsertChunksFromFile(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
@@ -2852,7 +2795,7 @@ func (c *RAGFlowClient) InsertDatasetFromFile(cmd *Command) (ResponseIf, error)
"file_path": filePath,
}
resp, err := c.HTTPClient.Request("POST", "/kb/insert_from_file", "web", nil, payload)
resp, err := c.HTTPClient.Request("POST", "/tenant/insert_chunks_from_file", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to insert dataset from file: %w", err)
}
@@ -2938,42 +2881,25 @@ func (c *RAGFlowClient) UpdateChunk(cmd *Command) (ResponseIf, error) {
return nil, fmt.Errorf("chunk_id not provided")
}
docID, ok := cmd.Params["doc_id"].(string)
if !ok {
return nil, fmt.Errorf("doc_id not provided")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
jsonBody, ok := cmd.Params["json_body"].(string)
if !ok {
return nil, fmt.Errorf("json_body not provided")
}
// Look up dataset_id from dataset_name
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, fmt.Errorf("failed to get dataset ID: %w", err)
}
// Try to get doc_id from the chunk retrieval endpoint
getResp, err := c.HTTPClient.Request("GET", "/chunk/get?chunk_id="+chunkID, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to get chunk info: %w", err)
}
var docID string
if getResp.StatusCode == 200 {
getJSON, err := getResp.JSON()
if err == nil {
if data, ok := getJSON["data"].(map[string]interface{}); ok {
if d, ok := data["doc_id"].(string); ok {
docID = d
}
}
}
}
if docID == "" {
return nil, fmt.Errorf("could not find document_id for chunk %s. Please provide document_id explicitly", chunkID)
jsonBody, ok := cmd.Params["json_body"].(string)
if !ok {
return nil, fmt.Errorf("json_body not provided")
}
// Parse the JSON body
@@ -3028,21 +2954,16 @@ func (c *RAGFlowClient) GetChunk(cmd *Command) (ResponseIf, error) {
return nil, fmt.Errorf("chunk_id not provided")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, err
}
docID, ok := cmd.Params["doc_id"].(string)
if !ok {
return nil, fmt.Errorf("doc_id not provided")
}
datasetID, ok := cmd.Params["dataset_id"].(string)
if !ok {
return nil, fmt.Errorf("dataset_id not provided")
}
resp, err := c.HTTPClient.Request("GET", fmt.Sprintf("/datasets/%s/documents/%s/chunks/%s", datasetID, docID, chunkID), "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to get chunk: %w", err)
@@ -3116,6 +3037,57 @@ func (c *RAGFlowClient) SetMeta(cmd *Command) (ResponseIf, error) {
return &result, nil
}
// DeleteMeta deletes metadata for a document
// If keys is provided, deletes specific keys; otherwise deletes entire document metadata
func (c *RAGFlowClient) DeleteMeta(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
docID, ok := cmd.Params["doc_id"].(string)
if !ok {
return nil, fmt.Errorf("doc_id not provided")
}
payload := map[string]interface{}{
"doc_id": docID,
}
// If keys provided, include in payload for deleting specific keys
if keysJSON, ok := cmd.Params["keys"].(string); ok {
payload["keys"] = keysJSON
}
resp, err := c.HTTPClient.Request("POST", "/document/delete_meta", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to delete metadata: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to delete metadata: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to delete metadata for document: %s", docID)
} else {
result.Message = fmt.Sprintf("Failed to delete metadata: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// RmTags removes tags from chunks in a dataset
func (c *RAGFlowClient) RmTags(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
@@ -3177,15 +3149,24 @@ func (c *RAGFlowClient) RemoveChunks(cmd *Command) (ResponseIf, error) {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
docID, ok := cmd.Params["doc_id"].(string)
if !ok {
return nil, fmt.Errorf("doc_id not provided")
}
payload := map[string]interface{}{
"doc_id": docID,
// Look up dataset ID by name
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, fmt.Errorf("dataset not found: %w", err)
}
payload := map[string]interface{}{}
// Check if delete_all is set
if deleteAll, ok := cmd.Params["delete_all"].(bool); ok && deleteAll {
payload["delete_all"] = true
@@ -3193,7 +3174,7 @@ func (c *RAGFlowClient) RemoveChunks(cmd *Command) (ResponseIf, error) {
payload["chunk_ids"] = chunkIDs
}
resp, err := c.HTTPClient.Request("POST", "/chunk/rm", "web", nil, payload)
resp, err := c.HTTPClient.Request("DELETE", "/datasets/"+datasetID+"/documents/"+docID+"/chunks", "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to remove chunks: %w", err)
}

245
internal/cli/user_parser.go
View File

@@ -621,10 +621,10 @@ func (p *Parser) parseCreateCommand() (*Command, error) {
return p.parseCreateChat()
case TokenToken:
return p.parseCreateToken()
case TokenDatasetTable:
return p.parseCreateDatasetTable()
case TokenChunkStore:
return p.parseCreateChunkStore()
case TokenMetadata:
return p.parseCreateMetadataTable()
return p.parseCreateMetadataStore()
case TokenProvider:
return p.parseCreateProviderInstance()
default:
@@ -656,9 +656,21 @@ func (p *Parser) parseCreateToken() (*Command, error) {
}
// Internal CLI for GO
// parseCreateDatasetTable parses: CREATE DATASET TABLE 'name' VECTOR SIZE N
func (p *Parser) parseCreateDatasetTable() (*Command, error) {
p.nextToken() // consume DATASET TABLE compound token
// parseCreateChunkStore parses: CREATE CHUNK STORE for Dataset 'name' VECTOR SIZE N
func (p *Parser) parseCreateChunkStore() (*Command, error) {
p.nextToken() // consume CHUNK STORE compound token
// Expect FOR
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
}
p.nextToken()
// Expect Dataset
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
@@ -688,20 +700,20 @@ func (p *Parser) parseCreateDatasetTable() (*Command, error) {
p.nextToken()
}
cmd := NewCommand("create_dataset_table")
cmd := NewCommand("create_chunk_store")
cmd.Params["dataset_name"] = datasetName
cmd.Params["vector_size"] = vectorSize
return cmd, nil
}
// Internal CLI for GO
// parseCreateMetadataTable parses: CREATE METADATA TABLE
func (p *Parser) parseCreateMetadataTable() (*Command, error) {
// CREATE METADATA TABLE
// parseCreateMetadataStore parses: CREATE METADATA STORE
func (p *Parser) parseCreateMetadataStore() (*Command, error) {
// CREATE METADATA STORE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenTable {
return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
if p.curToken.Type != TokenStore {
return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
@@ -709,7 +721,7 @@ func (p *Parser) parseCreateMetadataTable() (*Command, error) {
p.nextToken()
}
return NewCommand("create_metadata_table"), nil
return NewCommand("create_metadata_store"), nil
}
func (p *Parser) parseCreateUser() (*Command, error) {
@@ -1039,10 +1051,10 @@ func (p *Parser) parseDropCommand() (*Command, error) {
return p.parseDropChat()
case TokenToken:
return p.parseDropToken()
case TokenDatasetTable:
return p.parseDropDatasetTable()
case TokenChunkStore:
return p.parseDropChunkStore()
case TokenMetadata:
return p.parseDropMetadataTable()
return p.parseDropMetadataStore()
case TokenInstance:
return p.parseDropInstance()
case TokenModel:
@@ -1058,8 +1070,10 @@ func (p *Parser) parseDeleteCommand() (*Command, error) {
switch p.curToken.Type {
case TokenProvider:
return p.parseDeleteProvider()
case TokenMetadata:
return p.parseDeleteMeta()
default:
return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value)
return nil, fmt.Errorf("unknown DELETE target: %s", p.curToken.Value)
}
}
@@ -1108,9 +1122,21 @@ func (p *Parser) parseDropToken() (*Command, error) {
}
// Internal CLI for GO
// parseDropDatasetTable parses: DROP DATASET TABLE 'name'
func (p *Parser) parseDropDatasetTable() (*Command, error) {
p.nextToken() // consume DATASET TABLE
// parseDropChunkStore parses: DROP CHUNK STORE for Dataset 'name'
func (p *Parser) parseDropChunkStore() (*Command, error) {
p.nextToken() // consume CHUNK STORE
// Expect FOR
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR after CHUNK STORE, got %s", p.curToken.Value)
}
p.nextToken()
// Expect Dataset
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected Dataset after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
@@ -1122,26 +1148,25 @@ func (p *Parser) parseDropDatasetTable() (*Command, error) {
p.nextToken()
}
cmd := NewCommand("drop_dataset_table")
cmd := NewCommand("drop_chunk_store")
cmd.Params["dataset_name"] = datasetName
return cmd, nil
}
// Internal CLI for GO
// parseDropMetadataTable parses: DROP METADATA TABLE
func (p *Parser) parseDropMetadataTable() (*Command, error) {
// DROP METADATA TABLE
// parseDropMetadataStore parses: DROP METADATA STORE
func (p *Parser) parseDropMetadataStore() (*Command, error) {
// DROP METADATA STORE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenTable {
return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
if p.curToken.Type != TokenStore {
return nil, fmt.Errorf("expected STORE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("drop_metadata_table")
cmd := NewCommand("drop_metadata_store")
return cmd, nil
}
@@ -1545,7 +1570,7 @@ func (p *Parser) parseShowInstance() (*Command, error) {
return cmd, nil
}
// parseShowInstance parses SHOW BALANCE FROM <provider_name> <instance_name>
// parseShowBalance parses SHOW BALANCE FROM <provider_name> <instance_name>
func (p *Parser) parseShowBalance() (*Command, error) {
p.nextToken() // consume INSTANCE
@@ -2166,20 +2191,20 @@ func (p *Parser) parseImportCommand() (*Command, error) {
func (p *Parser) parseInsertCommand() (*Command, error) {
p.nextToken() // consume INSERT
// Expect DATASET or METADATA
if p.curToken.Type == TokenDataset {
return p.parseInsertDatasetFromFile()
// Expect CHUNKS or METADATA
if p.curToken.Type == TokenChunks {
return p.parseInsertChunksFromFile()
}
if p.curToken.Type == TokenMetadata {
return p.parseInsertMetadataFromFile()
}
return nil, fmt.Errorf("expected DATASET or METADATA after INSERT, got %s", p.curToken.Value)
return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value)
}
// Internal CLI for GO
// parseInsertDatasetFromFile parses: INSERT DATASET FROM FILE "file_path"
func (p *Parser) parseInsertDatasetFromFile() (*Command, error) {
p.nextToken() // consume DATASET
// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path"
func (p *Parser) parseInsertChunksFromFile() (*Command, error) {
p.nextToken() // consume CHUNKS
// Expect FROM
if p.curToken.Type != TokenFrom {
@@ -2199,7 +2224,7 @@ func (p *Parser) parseInsertDatasetFromFile() (*Command, error) {
return nil, err
}
cmd := NewCommand("insert_dataset_from_file")
cmd := NewCommand("insert_chunks_from_file")
cmd.Params["file_path"] = filePath
p.nextToken()
@@ -3261,6 +3286,8 @@ func (p *Parser) parseUserStatement() (*Command, error) {
switch p.curToken.Type {
case TokenPing:
return p.parsePingServer()
case TokenDelete:
return p.parseDeleteCommand()
case TokenShow:
return p.parseShowCommand()
case TokenCreate:
@@ -3389,7 +3416,7 @@ func (p *Parser) parseGetCommand() (*Command, error) {
return nil, fmt.Errorf("unknown GET target: %s", p.curToken.Value)
}
// parseGetChunk parses: GET CHUNK 'chunk_id' OF DATASET 'dataset_name' DOCUMENT 'doc_id'
// parseGetChunk parses: GET CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id'
func (p *Parser) parseGetChunk() (*Command, error) {
p.nextToken() // consume CHUNK
@@ -3408,21 +3435,8 @@ func (p *Parser) parseGetChunk() (*Command, error) {
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after OF")
}
p.nextToken()
// Parse dataset_name
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_name: %w", err)
}
cmd.Params["dataset_name"] = datasetName
p.nextToken()
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after dataset_name")
return nil, fmt.Errorf("expected DOCUMENT after OF")
}
p.nextToken()
@@ -3433,6 +3447,24 @@ func (p *Parser) parseGetChunk() (*Command, error) {
}
cmd.Params["doc_id"] = docID
p.nextToken()
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
// Parse dataset_id
datasetID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_id: %w", err)
}
cmd.Params["dataset_id"] = datasetID
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
@@ -3455,7 +3487,7 @@ func (p *Parser) parseUpdateCommand() (*Command, error) {
}
// Internal CLI for GO
// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DOCUMENT 'doc_id' IN DATASET 'dataset_id' SET '{"content": "..."}'
func (p *Parser) parseUpdateChunk() (*Command, error) {
p.nextToken() // consume CHUNK
@@ -3474,8 +3506,26 @@ func (p *Parser) parseUpdateChunk() (*Command, error) {
}
p.nextToken()
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after OF")
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
@@ -3555,6 +3605,65 @@ func (p *Parser) parseSetMeta() (*Command, error) {
return cmd, nil
}
// parseDeleteMeta parses: DELETE METADATA OF DOCUMENT 'doc_id' [KEYS '["key1", "key2"]']
// If KEYS is not provided, deletes entire document metadata
func (p *Parser) parseDeleteMeta() (*Command, error) {
p.nextToken() // consume METADATA
// Expect OF
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF after DELETE METADATA")
}
p.nextToken()
// Expect DOCUMENT
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after DELETE METADATA OF")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd := NewCommand("delete_meta")
cmd.Params["doc_id"] = docID
p.nextToken()
// KEYS is optional - if not provided, delete entire document metadata
if p.curToken.Type != TokenKeys {
if p.curToken.Type == TokenSemicolon {
p.nextToken()
return cmd, nil
}
if p.curToken.Type == TokenEOF {
return cmd, nil
}
return nil, fmt.Errorf("expected KEYS or end of command after doc_id")
}
// Parse keys JSON array
p.nextToken()
keys, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected keys JSON array: %w", err)
}
cmd.Params["keys"] = keys
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
return cmd, nil
}
if p.curToken.Type != TokenEOF {
return nil, fmt.Errorf("expected end of command after KEYS")
}
return cmd, nil
}
// parseRemoveTags parses: REMOVE TAGS 'tag1', 'tag2' from DATASET 'dataset_name';
func (p *Parser) parseRemoveTags() (*Command, error) {
p.nextToken() // consume TAGS
@@ -3611,8 +3720,8 @@ func (p *Parser) parseRemoveTags() (*Command, error) {
}
// parseRemoveChunk parses:
// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id';
// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id';
// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name';
func (p *Parser) parseRemoveChunk() (*Command, error) {
cmd := NewCommand("remove_chunks")
@@ -3627,7 +3736,7 @@ func (p *Parser) parseRemoveChunk() (*Command, error) {
} else {
// curToken is TokenChunks, consume it first
p.nextToken()
// Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' (space-separated)
// Multiple chunks: REMOVE CHUNKS 'id1' 'id2' FROM DOCUMENT 'doc_id' IN DATASET 'dataset_name' (space-separated)
// Parse first chunk ID
chunkID, err := p.parseQuotedString()
if err != nil {
@@ -3669,6 +3778,28 @@ func (p *Parser) parseRemoveChunk() (*Command, error) {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
// Expect IN
if p.curToken.Type != TokenIn {
return nil, fmt.Errorf("expected IN after doc_id")
}
p.nextToken()
// Expect DATASET
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after IN")
}
p.nextToken()
// Parse dataset_name (quoted string)
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset_name: %w", err)
}
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional