Implement Elasticsearch functions in GO (#15160)

### What problem does this PR solve?

Implement Elasticsearch functions in GO (except for Search)

### Type of change

- [x] Refactoring

internal/handler/chunk.go

@@ -133,7 +133,7 @@ func (h *ChunkHandler) Get(c *gin.Context) {
 		return
 	}
 
-	chunkID := c.Query("chunk_id")
+	chunkID := c.Param("chunk_id")
 	if chunkID == "" {
 		c.JSON(http.StatusBadRequest, gin.H{
 			"code":    400,
@@ -337,7 +337,7 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
 	})
 }
 
-// Remove handles chunk removal requests
+// RemoveChunks handles chunk removal requests
 // @Summary Remove Chunks
 // @Description Remove chunks from a document
 // @Tags chunks
@@ -345,14 +345,24 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
 // @Produce json
 // @Param request body service.RemoveChunksRequest true "remove chunks request"
 // @Success 200 {object} map[string]interface{}
-// @Router /v1/chunk/rm [post]
-func (h *ChunkHandler) Remove(c *gin.Context) {
+// @Router /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks [delete]
+func (h *ChunkHandler) RemoveChunks(c *gin.Context) {
 	user, errorCode, errorMessage := GetUser(c)
 	if errorCode != common.CodeSuccess {
 		jsonError(c, errorCode, errorMessage)
 		return
 	}
 
+	// Get document_id from URL path
+	docID := c.Param("document_id")
+	if docID == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "document_id is required",
+		})
+		return
+	}
+
 	var req service.RemoveChunksRequest
 	if err := c.ShouldBindJSON(&req); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{
@@ -362,6 +372,8 @@ func (h *ChunkHandler) Remove(c *gin.Context) {
 		return
 	}
 
+	req.DocID = docID
+
 	if req.DocID == "" {
 		c.JSON(http.StatusBadRequest, gin.H{
 			"code":    400,

internal/handler/document.go

@@ -565,6 +565,124 @@ func (h *DocumentHandler) SetMeta(c *gin.Context) {
 	})
 }
 
+// DeleteMetaRequest represents the request for deleting document metadata
+type DeleteMetaRequest struct {
+	DocID string `json:"doc_id" binding:"required"`
+	Keys  string `json:"keys"` // optional - if provided, deletes specific keys; otherwise deletes entire document metadata
+}
+
+// DeleteMeta handles the delete metadata request for a document
+// If Keys is provided, deletes specific metadata keys; otherwise deletes entire document metadata
+// @Summary Delete Document Metadata
+// @Description Delete metadata keys or entire document metadata for a specific document
+// @Tags documents
+// @Accept json
+// @Produce json
+// @Security ApiKeyAuth
+// @Param request body DeleteMetaRequest true "metadata keys to delete or empty to delete all"
+// @Success 200 {object} map[string]interface{}
+// @Router /v1/document/delete_meta [post]
+func (h *DocumentHandler) DeleteMeta(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		jsonError(c, errorCode, errorMessage)
+		return
+	}
+
+	var req DeleteMetaRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    1,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	if req.DocID == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    1,
+			"message": "doc_id is required",
+		})
+		return
+	}
+
+	// Authorization: user must be able to access the document's dataset.
+	doc, err := h.documentService.GetDocumentByID(req.DocID)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    1,
+			"message": "document not found",
+		})
+		return
+	}
+	if !h.datasetService.Accessible(doc.KbID, user.ID) {
+		jsonError(c, common.CodeAuthenticationError, "No authorization.")
+		return
+	}
+
+	// If Keys is provided, parse and delete specific keys; otherwise delete entire document metadata
+	if req.Keys != "" {
+		// Parse keys JSON string - expected to be a list of key names to delete
+		var keys []string
+		if err := json.Unmarshal([]byte(req.Keys), &keys); err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"code":    1,
+				"message": "Json syntax error: " + err.Error(),
+			})
+			return
+		}
+
+		if keys == nil || len(keys) == 0 {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"code":    1,
+				"message": "keys list is required",
+			})
+			return
+		}
+
+		err := h.documentService.DeleteDocumentMetadata(req.DocID, keys)
+		if err != nil {
+			errMsg := err.Error()
+			if strings.Contains(errMsg, "no such document") || strings.Contains(errMsg, "document not found") {
+				c.JSON(http.StatusBadRequest, gin.H{
+					"code":    1,
+					"message": errMsg,
+				})
+			} else {
+				c.JSON(http.StatusInternalServerError, gin.H{
+					"code":    1,
+					"message": "Failed to delete metadata: " + errMsg,
+				})
+			}
+			return
+		}
+	} else {
+		// Delete entire document metadata
+		err := h.documentService.DeleteDocumentAllMetadata(req.DocID)
+		if err != nil {
+			errMsg := err.Error()
+			if strings.Contains(errMsg, "no such document") || strings.Contains(errMsg, "document not found") {
+				c.JSON(http.StatusBadRequest, gin.H{
+					"code":    1,
+					"message": errMsg,
+				})
+			} else {
+				c.JSON(http.StatusInternalServerError, gin.H{
+					"code":    1,
+					"message": "Failed to delete metadata: " + errMsg,
+				})
+			}
+			return
+		}
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"message": "success",
+		"data":    true,
+	})
+}
+
 type ParseDocumentRequest struct {
 	Documents []string `json:"documents" binding:"required"`
 }

internal/handler/kb.go

@@ -17,11 +17,8 @@
 package handler
 
 import (
-	"encoding/json"
 	"net/http"
-	"os"
 	"ragflow/internal/common"
-	"ragflow/internal/engine"
 	"ragflow/internal/service"
 	"strings"
 
@@ -426,174 +423,4 @@ func (h *KnowledgebaseHandler) GetBasicInfo(c *gin.Context) {
 	}
 
 	jsonResponse(c, common.CodeSuccess, map[string]interface{}{}, "success")
-}
-
-// CreateDatasetInDocEngine handles the create dataset request for a knowledge base
-// @Summary Create Dataset in Doc Engine
-// @Description Create the Infinity table for a knowledge base
-// @Tags knowledgebase
-// @Accept json
-// @Produce json
-// @Security ApiKeyAuth
-// @Param request body service.CreateDatasetTableRequest true "create dataset request"
-// @Success 200 {object} map[string]interface{}
-// @Router /v1/kb/doc_engine_table [post]
-func (h *KnowledgebaseHandler) CreateDatasetInDocEngine(c *gin.Context) {
-	user, errorCode, errorMessage := GetUser(c)
-	if errorCode != common.CodeSuccess {
-		jsonError(c, errorCode, errorMessage)
-		return
-	}
-
-	var req service.CreateDatasetTableRequest
-	if err := c.ShouldBindJSON(&req); err != nil {
-		jsonError(c, common.CodeDataError, err.Error())
-		return
-	}
-
-	// Check authorization
-	if !h.kbService.Accessible(req.KBID, user.ID) {
-		jsonError(c, common.CodeAuthenticationError, "No authorization.")
-		return
-	}
-
-	result, code, err := h.kbService.CreateDatasetInDocEngine(&req)
-	if err != nil {
-		jsonError(c, code, err.Error())
-		return
-	}
-
-	jsonResponse(c, common.CodeSuccess, result, "success")
-}
-
-// DeleteDatasetInDocEngineRequest represents the request for deleting a dataset table
-type DeleteDatasetInDocEngineRequest struct {
-	KBID string `json:"kb_id" binding:"required"`
-}
-
-// DeleteDatasetInDocEngine handles the delete dataset request for a knowledge base
-// @Summary Delete Dataset in Doc Engine
-// @Description Delete the Infinity table for a knowledge base
-// @Tags knowledgebase
-// @Accept json
-// @Produce json
-// @Security ApiKeyAuth
-// @Param request body DeleteDatasetInDocEngineRequest true "delete dataset request"
-// @Success 200 {object} map[string]interface{}
-// @Router /v1/kb/doc_engine_table [delete]
-func (h *KnowledgebaseHandler) DeleteDatasetInDocEngine(c *gin.Context) {
-	user, errorCode, errorMessage := GetUser(c)
-	if errorCode != common.CodeSuccess {
-		jsonError(c, errorCode, errorMessage)
-		return
-	}
-
-	var req DeleteDatasetInDocEngineRequest
-	if err := c.ShouldBindJSON(&req); err != nil {
-		jsonError(c, common.CodeDataError, err.Error())
-		return
-	}
-
-	// Check authorization
-	if !h.kbService.Accessible(req.KBID, user.ID) {
-		jsonError(c, common.CodeAuthenticationError, "No authorization.")
-		return
-	}
-
-	code, err := h.kbService.DeleteDatasetInDocEngine(req.KBID)
-	if err != nil {
-		jsonError(c, code, err.Error())
-		return
-	}
-
-	jsonResponse(c, common.CodeSuccess, nil, "success")
-}
-
-// InsertDatasetFromFileRequest request for inserting chunks into dataset from file
-type InsertDatasetFromFileRequest struct {
-	FilePath string `json:"file_path" binding:"required"`
-}
-
-// @Summary Insert chunks into dataset from file
-// @Description Internal: Insert into dataset table from a JSON file (table name extracted from file)
-// @Tags knowledgebase
-// @Accept json
-// @Produce json
-// @Security ApiKeyAuth
-// @Param request body InsertDatasetFromFileRequest true "insert dataset request"
-// @Success 200 {object} map[string]interface{}
-// @Router /v1/kb/insert_from_file [post]
-func (h *KnowledgebaseHandler) InsertDatasetFromFile(c *gin.Context) {
-	_, errorCode, errorMessage := GetUser(c)
-	if errorCode != common.CodeSuccess {
-		jsonError(c, errorCode, errorMessage)
-		return
-	}
-
-	var req InsertDatasetFromFileRequest
-	if err := c.ShouldBindJSON(&req); err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{
-			"code":    400,
-			"message": err.Error(),
-		})
-		return
-	}
-
-	if req.FilePath == "" {
-		c.JSON(http.StatusBadRequest, gin.H{
-			"code":    400,
-			"message": "file_path is required",
-		})
-		return
-	}
-
-	// Read the JSON file
-	data, err := os.ReadFile(req.FilePath)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{
-			"code":    400,
-			"message": "failed to read file: " + err.Error(),
-		})
-		return
-	}
-
-	// Parse JSON - format: {"table_name": ..., "knowledgebase_id": ..., "chunks": [...]}
-	var debugFormat struct {
-		TableNamePrefix string                   `json:"table_name"`
-		KnowledgebaseID string                   `json:"knowledgebase_id"`
-		Chunks          []map[string]interface{} `json:"chunks"`
-	}
-
-	if err := json.Unmarshal(data, &debugFormat); err != nil || debugFormat.Chunks == nil {
-		c.JSON(http.StatusBadRequest, gin.H{
-			"code":    400,
-			"message": "invalid JSON format: expected {\"table_name\": ..., \"knowledgebase_id\": ..., \"chunks\": [...]}",
-		})
-		return
-	}
-
-	if len(debugFormat.Chunks) == 0 {
-		c.JSON(http.StatusBadRequest, gin.H{
-			"code":    400,
-			"message": "no chunks found in file",
-		})
-		return
-	}
-
-	// Get the document engine and insert
-	docEngine := engine.Get()
-	result, err := docEngine.InsertChunks(c.Request.Context(), debugFormat.Chunks, debugFormat.TableNamePrefix, debugFormat.KnowledgebaseID)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{
-			"code":    500,
-			"message": "failed to insert into dataset: " + err.Error(),
-		})
-		return
-	}
-
-	c.JSON(http.StatusOK, gin.H{
-		"code":    0,
-		"data":    result,
-		"message": "success",
-	})
 }

internal/handler/tenant.go

@@ -32,13 +32,15 @@ import (
 type TenantHandler struct {
 	tenantService *service.TenantService
 	userService   *service.UserService
+	kbService     *service.KnowledgebaseService
 }
 
 // NewTenantHandler create tenant handler
-func NewTenantHandler(tenantService *service.TenantService, userService *service.UserService) *TenantHandler {
+func NewTenantHandler(tenantService *service.TenantService, userService *service.UserService, kbService *service.KnowledgebaseService) *TenantHandler {
 	return &TenantHandler{
 		tenantService: tenantService,
 		userService:   userService,
+		kbService:     kbService,
 	}
 }
 
@@ -192,16 +194,16 @@ func (h *TenantHandler) TenantList(c *gin.Context) {
 	})
 }
 
-// CreateMetadataInDocEngine handles the create doc meta table request
-// @Summary Create Doc Meta Table
-// @Description Create the document metadata table for a tenant
+// CreateMetadataStore handles the create metadata store request
+// @Summary Create Metadata Store
+// @Description Create the metadata store for a tenant
 // @Tags tenants
 // @Accept json
 // @Produce json
 // @Security ApiKeyAuth
 // @Success 200 {object} map[string]interface{}
-// @Router /v1/tenant/doc_engine_metadata_table [post]
-func (h *TenantHandler) CreateMetadataInDocEngine(c *gin.Context) {
+// @Router /v1/tenant/metadata_store [post]
+func (h *TenantHandler) CreateMetadataStore(c *gin.Context) {
 	user, errorCode, errorMessage := GetUser(c)
 	if errorCode != common.CodeSuccess {
 		jsonError(c, errorCode, errorMessage)
@@ -211,7 +213,7 @@ func (h *TenantHandler) CreateMetadataInDocEngine(c *gin.Context) {
 	// Use user.ID as tenant ID (user IS the tenant in user mode)
 	tenantID := user.ID
 
-	code, err := h.tenantService.CreateMetadataInDocEngine(tenantID)
+	code, err := h.tenantService.CreateMetadataStore(tenantID)
 	if err != nil {
 		jsonError(c, code, err.Error())
 		return
@@ -224,16 +226,16 @@ func (h *TenantHandler) CreateMetadataInDocEngine(c *gin.Context) {
 	})
 }
 
-// DeleteMetadataInDocEngine handles the delete doc meta table request
-// @Summary Delete Metadata In Doc Engine
-// @Description Delete the document metadata table for a tenant
+// DeleteMetadataStore handles the delete metadata store request
+// @Summary Delete Metadata Store
+// @Description Delete the metadata store for a tenant
 // @Tags tenants
 // @Accept json
 // @Produce json
 // @Security ApiKeyAuth
 // @Success 200 {object} map[string]interface{}
-// @Router /v1/tenant/doc_engine_metadata_table [delete]
-func (h *TenantHandler) DeleteMetadataInDocEngine(c *gin.Context) {
+// @Router /v1/tenant/metadata_store [delete]
+func (h *TenantHandler) DeleteMetadataStore(c *gin.Context) {
 	user, errorCode, errorMessage := GetUser(c)
 	if errorCode != common.CodeSuccess {
 		jsonError(c, errorCode, errorMessage)
@@ -243,7 +245,7 @@ func (h *TenantHandler) DeleteMetadataInDocEngine(c *gin.Context) {
 	// Use user.ID as tenant ID (user IS the tenant in user mode)
 	tenantID := user.ID
 
-	code, err := h.tenantService.DeleteMetadataInDocEngine(tenantID)
+	code, err := h.tenantService.DeleteMetadataStore(tenantID)
 	if err != nil {
 		jsonError(c, code, err.Error())
 		return
@@ -256,6 +258,201 @@ func (h *TenantHandler) DeleteMetadataInDocEngine(c *gin.Context) {
 	})
 }
 
+// CreateChunkTableRequest represents the request for creating a chunk table
+type CreateChunkTableRequest struct {
+	KBID       string `json:"kb_id" binding:"required"`
+	VectorSize int    `json:"vector_size" binding:"required"`
+}
+
+// CreateChunkStore handles the create chunk store request
+// @Summary Create Chunk Store
+// @Description Create the chunk store for a knowledge base
+// @Tags tenants
+// @Accept json
+// @Produce json
+// @Security ApiKeyAuth
+// @Param request body CreateChunkTableRequest true "create chunk store request"
+// @Success 200 {object} map[string]interface{}
+// @Router /v1/tenant/chunk_store [post]
+func (h *TenantHandler) CreateChunkStore(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		jsonError(c, errorCode, errorMessage)
+		return
+	}
+
+	var req CreateChunkTableRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		jsonError(c, common.CodeDataError, err.Error())
+		return
+	}
+
+	// Check authorization - user must have access to this kb
+	if !h.kbService.Accessible(req.KBID, user.ID) {
+		jsonError(c, common.CodeAuthenticationError, "No authorization.")
+		return
+	}
+
+	serviceReq := &service.CreateDatasetTableRequest{
+		KBID:       req.KBID,
+		VectorSize: req.VectorSize,
+	}
+	result, code, err := h.tenantService.CreateChunkStore(serviceReq)
+	if err != nil {
+		jsonError(c, code, err.Error())
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    common.CodeSuccess,
+		"message": "success",
+		"data":    result,
+	})
+}
+
+// DeleteChunkTableRequest represents the request for deleting a chunk table
+type DeleteChunkTableRequest struct {
+	KBID string `json:"kb_id" binding:"required"`
+}
+
+// DeleteChunkStore handles the delete chunk store request
+// @Summary Delete Chunk Store
+// @Description Delete the chunk store for a knowledge base
+// @Tags tenants
+// @Accept json
+// @Produce json
+// @Security ApiKeyAuth
+// @Param request body DeleteChunkTableRequest true "delete chunk store request"
+// @Success 200 {object} map[string]interface{}
+// @Router /v1/tenant/chunk_store [delete]
+func (h *TenantHandler) DeleteChunkStore(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		jsonError(c, errorCode, errorMessage)
+		return
+	}
+
+	var req DeleteChunkTableRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		jsonError(c, common.CodeDataError, err.Error())
+		return
+	}
+
+	// Check authorization
+	if !h.kbService.Accessible(req.KBID, user.ID) {
+		jsonError(c, common.CodeAuthenticationError, "No authorization.")
+		return
+	}
+
+	code, err := h.tenantService.DeleteChunkStore(req.KBID)
+	if err != nil {
+		jsonError(c, code, err.Error())
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    common.CodeSuccess,
+		"message": "success",
+		"data":    nil,
+	})
+}
+
+// InsertChunksFromFileRequest request for inserting chunks from file
+type InsertChunksFromFileRequest struct {
+	FilePath string `json:"file_path" binding:"required"`
+}
+
+// @Summary Insert chunks into dataset from JSON file
+// @Description Internal: Insert chunks into dataset table from a JSON file
+// @Tags tenants
+// @Accept json
+// @Produce json
+// @Security ApiKeyAuth
+// @Param request body InsertChunksFromFileRequest true "insert chunks request"
+// @Success 200 {object} map[string]interface{}
+// @Router /v1/tenant/insert_chunks_from_file [post]
+func (h *TenantHandler) InsertChunksFromFile(c *gin.Context) {
+	_, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		jsonError(c, errorCode, errorMessage)
+		return
+	}
+
+	var req InsertChunksFromFileRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": err.Error(),
+		})
+		return
+	}
+
+	if req.FilePath == "" {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "file_path is required",
+		})
+		return
+	}
+
+	// Read the JSON file
+	data, err := os.ReadFile(req.FilePath)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "failed to read file: " + err.Error(),
+		})
+		return
+	}
+
+	// Parse JSON - format: {"index_name"/"table_name": ..., "knowledgebase_id": ..., "chunks": [...]}
+	var debugFormat struct {
+		IndexName       string `json:"index_name"`
+		TableName       string `json:"table_name"`
+		KnowledgebaseID string `json:"knowledgebase_id"`
+		Chunks          []map[string]interface{} `json:"chunks"`
+	}
+
+	if err := json.Unmarshal(data, &debugFormat); err != nil || debugFormat.Chunks == nil {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "invalid JSON format: expected {\"index_name\"/\"table_name\": ..., \"knowledgebase_id\": ..., \"chunks\": [...]}",
+		})
+		return
+	}
+
+	if len(debugFormat.Chunks) == 0 {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"code":    400,
+			"message": "no chunks found in file",
+		})
+		return
+	}
+
+	// Support both index_name (ES) and table_name (Infinity) in JSON
+	indexName := debugFormat.IndexName
+	if indexName == "" {
+		indexName = debugFormat.TableName
+	}
+
+	// Get the document engine and insert
+	docEngine := engine.Get()
+	result, err := docEngine.InsertChunks(c.Request.Context(), debugFormat.Chunks, indexName, debugFormat.KnowledgebaseID)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"code":    500,
+			"message": "failed to insert into dataset: " + err.Error(),
+		})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"data":    result,
+		"message": "success",
+	})
+}
+
 // InsertMetadataFromFileRequest request for inserting metadata from file
 type InsertMetadataFromFileRequest struct {
 	FilePath string `json:"file_path" binding:"required"`