zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-31 13:03:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement Delete in GO and refactor functions (#13974)

Browse Source 
### What problem does this PR solve?

Implement Delete in GO and refactor functions

### Type of change

- [x] Refactoring

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a remove_chunks command to delete specific or all chunks from a
document.
  * Added new endpoints for chunk removal and chunk update.

* **Refactor**
* Renamed index commands to dataset/metadata table terminology and
updated REST routes accordingly.
* Updated chunk update flow to a JSON POST style and improved metadata
error messages.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
qinling0210
2026-04-09 09:52:31 +08:00
committed by GitHub
parent 3b7723855c
commit 82fa85c837
22 changed files with 1470 additions and 1133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
internal/cli/user_parser.go
View File

@@ -511,8 +511,10 @@ func (p *Parser) parseCreateCommand() (*Command, error) {
return p.parseCreateChat()
case TokenToken:
return p.parseCreateToken()
case TokenIndex:
return p.parseCreateIndex()
case TokenDatasetTable:
return p.parseCreateDatasetTable()
case TokenMetadata:
return p.parseCreateMetadataTable()
case TokenProvider:
return p.parseCreateProviderInstance()
default:
@@ -541,30 +543,10 @@ func (p *Parser) parseCreateToken() (*Command, error) {
return NewCommand("create_token"), nil
}
func (p *Parser) parseCreateIndex() (*Command, error) {
// CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
// CREATE INDEX DOC_META
p.nextToken() // consume INDEX
// Check if creating doc meta index
if p.curToken.Type == TokenDocMeta {
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("create_doc_meta_index"), nil
}
// Otherwise, must be CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
}
p.nextToken()
// Internal CLI for GO
// parseCreateDatasetTable parses: CREATE DATASET TABLE 'name' VECTOR SIZE N
func (p *Parser) parseCreateDatasetTable() (*Command, error) {
p.nextToken() // consume DATASET TABLE compound token
datasetName, err := p.parseQuotedString()
if err != nil {
@@ -572,8 +554,12 @@ func (p *Parser) parseCreateIndex() (*Command, error) {
}
p.nextToken()
if p.curToken.Type != TokenVectorSize {
return nil, fmt.Errorf("expected VECTOR_SIZE after dataset name, got %s", p.curToken.Value)
if p.curToken.Type != TokenVector {
return nil, fmt.Errorf("expected VECTOR after dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenSize {
return nil, fmt.Errorf("expected SIZE after VECTOR, got %s", p.curToken.Value)
}
p.nextToken()
@@ -590,12 +576,30 @@ func (p *Parser) parseCreateIndex() (*Command, error) {
p.nextToken()
}
cmd := NewCommand("create_index")
cmd := NewCommand("create_dataset_table")
cmd.Params["dataset_name"] = datasetName
cmd.Params["vector_size"] = vectorSize
return cmd, nil
}
// Internal CLI for GO
// parseCreateMetadataTable parses: CREATE METADATA TABLE
func (p *Parser) parseCreateMetadataTable() (*Command, error) {
// CREATE METADATA TABLE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenTable {
return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("create_metadata_table"), nil
}
func (p *Parser) parseCreateUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()
@@ -801,8 +805,10 @@ func (p *Parser) parseDropCommand() (*Command, error) {
return p.parseDropChat()
case TokenToken:
return p.parseDropToken()
case TokenIndex:
return p.parseDropIndex()
case TokenDatasetTable:
return p.parseDropDatasetTable()
case TokenMetadata:
return p.parseDropMetadataTable()
case TokenInstance:
return p.parseDropInstance()
default:
@@ -827,6 +833,8 @@ func (p *Parser) parseRemoveCommand() (*Command, error) {
switch p.curToken.Type {
case TokenTag:
return p.parseRemoveTags()
case TokenChunks, TokenAll:
return p.parseRemoveChunk()
default:
return nil, fmt.Errorf("unknown REMOVE target: %s", p.curToken.Value)
}
@@ -863,30 +871,10 @@ func (p *Parser) parseDropToken() (*Command, error) {
return cmd, nil
}
func (p *Parser) parseDropIndex() (*Command, error) {
// DROP INDEX FOR DATASET 'name' OR DROP INDEX DOC_META
p.nextToken() // consume INDEX
// Check if dropping doc meta index
if p.curToken.Type == TokenDocMeta {
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("drop_doc_meta_index")
return cmd, nil
}
// Otherwise, must be DROP INDEX FOR DATASET 'name'
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
}
p.nextToken()
// Internal CLI for GO
// parseDropDatasetTable parses: DROP DATASET TABLE 'name'
func (p *Parser) parseDropDatasetTable() (*Command, error) {
p.nextToken() // consume DATASET TABLE
datasetName, err := p.parseQuotedString()
if err != nil {
@@ -898,11 +886,29 @@ func (p *Parser) parseDropIndex() (*Command, error) {
p.nextToken()
}
cmd := NewCommand("drop_index")
cmd := NewCommand("drop_dataset_table")
cmd.Params["dataset_name"] = datasetName
return cmd, nil
}
// Internal CLI for GO
// parseDropMetadataTable parses: DROP METADATA TABLE
func (p *Parser) parseDropMetadataTable() (*Command, error) {
// DROP METADATA TABLE
p.nextToken() // consume METADATA
if p.curToken.Type != TokenTable {
return nil, fmt.Errorf("expected TABLE after METADATA, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("drop_metadata_table")
return cmd, nil
}
func (p *Parser) parseDropUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()
@@ -2428,15 +2434,21 @@ func (p *Parser) parseUnsetCommand() (*Command, error) {
return NewCommand("unset_token"), nil
}
// parseUpdateCommand parses UPDATE CHUNK command
// UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
// Internal
// parseUpdateCommand parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
func (p *Parser) parseUpdateCommand() (*Command, error) {
p.nextToken() // consume UPDATE
if p.curToken.Type != TokenChunk {
return nil, fmt.Errorf("expected CHUNK after UPDATE")
if p.curToken.Type == TokenChunk {
return p.parseUpdateChunk()
}
p.nextToken()
return nil, fmt.Errorf("unknown UPDATE target: %s", p.curToken.Value)
}
// parseUpdateChunk parses: UPDATE CHUNK 'chunk_id' OF DATASET 'dataset_name' SET '{"content": "..."}'
func (p *Parser) parseUpdateChunk() (*Command, error) {
p.nextToken() // consume CHUNK
// Parse chunk_id
chunkID, err := p.parseQuotedString()
@@ -2588,3 +2600,74 @@ func (p *Parser) parseRemoveTags() (*Command, error) {
return cmd, nil
}
// parseRemoveChunk parses:
// - REMOVE CHUNKS 'chunk_id1', 'chunk_id2' FROM DOCUMENT 'doc_id';
// - REMOVE ALL CHUNKS FROM DOCUMENT 'doc_id';
func (p *Parser) parseRemoveChunk() (*Command, error) {
cmd := NewCommand("remove_chunks")
// Check if ALL CHUNKS - if we came here from TokenAll case, curToken is already ALL
if p.curToken.Type == TokenAll {
p.nextToken() // consume ALL
if p.curToken.Type != TokenChunks {
return nil, fmt.Errorf("expected CHUNKS after ALL")
}
p.nextToken() // consume CHUNKS
cmd.Params["delete_all"] = true
} else {
// curToken is TokenChunks, consume it first
p.nextToken()
// Multiple chunks: REMOVE CHUNKS 'id1', 'id2' FROM DOCUMENT 'doc_id'
// Parse first chunk ID
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id: %w", err)
}
chunkIDs := []string{chunkID}
// Parse additional chunk IDs separated by commas
for {
p.nextToken()
if p.curToken.Type == TokenComma {
p.nextToken()
chunkID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected chunk_id after comma: %w", err)
}
chunkIDs = append(chunkIDs, chunkID)
} else {
break
}
}
cmd.Params["chunk_ids"] = chunkIDs
}
// Expect FROM
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM after chunk(s)")
}
p.nextToken()
// Expect DOCUMENT
if p.curToken.Type != TokenDocument {
return nil, fmt.Errorf("expected DOCUMENT after FROM")
}
p.nextToken()
// Parse doc_id
docID, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected doc_id: %w", err)
}
cmd.Params["doc_id"] = docID
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}