mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-04 23:00:30 +08:00
feat[go]: datasets/<dataset_id>/chunks DELETE (#16185)
### What problem does this PR solve?
As title:
`documents.POST("/ingest", r.documentHandler.Ingest)`:
---
<img width="3750" height="2039" alt="image"
src="https://github.com/user-attachments/assets/533c1c3d-af3e-47e6-9f51-a278539b7066"
/>
`datasets.DELETE("/:dataset_id/chunks", r.chunkHandler.StopParsing)`
---
<img width="3621" height="2040" alt="image"
src="https://github.com/user-attachments/assets/022adcdb-1e47-4883-9611-1a695c34007d"
/>
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -38,6 +38,7 @@ type chunkService interface {
|
||||
UpdateChunk(req *service.UpdateChunkRequest, userID string) error
|
||||
RemoveChunks(req *service.RemoveChunksRequest, userID string) (int64, error)
|
||||
Parse(userID, datasetID string, req *service.ParseFileRequest) (map[string]interface{}, common.ErrorCode, error)
|
||||
StopParsing(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error)
|
||||
}
|
||||
|
||||
// ChunkHandler chunk handler
|
||||
@@ -224,8 +225,8 @@ func (h *ChunkHandler) Parse(c *gin.Context) {
|
||||
})
|
||||
return
|
||||
}
|
||||
datasetID := strings.TrimSpace(c.Param("dataset_id"))
|
||||
if datasetID == "" {
|
||||
datasetId := strings.TrimSpace(c.Param("dataset_id"))
|
||||
if datasetId == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"code": common.CodeBadRequest,
|
||||
"message": "dataset_id is required",
|
||||
@@ -243,7 +244,7 @@ func (h *ChunkHandler) Parse(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
data, code, err := h.chunkService.Parse(userID, datasetID, &req)
|
||||
data, code, err := h.chunkService.Parse(userID, datasetId, &req)
|
||||
if code != common.CodeSuccess {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"code": code,
|
||||
@@ -353,6 +354,59 @@ func parseAvailableQuery(raw string) (int, bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ChunkHandler) StopParsing(c *gin.Context) {
|
||||
user, errorCode, errorMessage := GetUser(c)
|
||||
if errorCode != common.CodeSuccess {
|
||||
jsonError(c, errorCode, errorMessage)
|
||||
return
|
||||
}
|
||||
|
||||
datasetID := c.Param("dataset_id")
|
||||
if datasetID == "" {
|
||||
jsonError(c, common.CodeDataError, "dataset_id is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req service.StopParsingRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
jsonError(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.DocumentIDs) == 0 {
|
||||
jsonError(c, common.CodeDataError, "`document_ids` is required")
|
||||
return
|
||||
}
|
||||
|
||||
resp, code, err := h.chunkService.StopParsing(user.ID, datasetID, req)
|
||||
if err != nil {
|
||||
var data interface{}
|
||||
if resp != nil {
|
||||
data = resp.Data
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"code": code,
|
||||
"data": data,
|
||||
"message": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
message := "success"
|
||||
var data interface{}
|
||||
if resp != nil {
|
||||
if resp.Message != "" {
|
||||
message = resp.Message
|
||||
}
|
||||
data = resp.Data
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"code": common.CodeSuccess,
|
||||
"data": data,
|
||||
"message": message,
|
||||
})
|
||||
}
|
||||
|
||||
// List retrieves chunks for a document.
|
||||
// @Summary List Chunks
|
||||
// @Description Retrieve paginated chunks for a document with optional filtering.
|
||||
|
||||
@@ -23,6 +23,7 @@ type mockChunkSvc struct {
|
||||
listFn func(req *service.ListChunksRequest, userID string) (*service.ListChunksResponse, error)
|
||||
switchChunksFn func(userID, datasetID, documentID string, availableInt int, chunkIDs []string) error
|
||||
updateChunkFn func(req *service.UpdateChunkRequest, userID string) error
|
||||
stopParsingFn func(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error)
|
||||
}
|
||||
|
||||
func (m *mockChunkSvc) RetrievalTest(req *service.RetrievalTestRequest, userID string) (*service.RetrievalTestResponse, error) {
|
||||
@@ -58,6 +59,12 @@ func (m *mockChunkSvc) UpdateChunk(req *service.UpdateChunkRequest, userID strin
|
||||
func (m *mockChunkSvc) RemoveChunks(*service.RemoveChunksRequest, string) (int64, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
func (m *mockChunkSvc) StopParsing(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
|
||||
if m.stopParsingFn != nil {
|
||||
return m.stopParsingFn(userID, datasetID, req)
|
||||
}
|
||||
panic("not implemented")
|
||||
}
|
||||
func (m *mockChunkSvc) Parse(string, string, *service.ParseFileRequest) (map[string]interface{}, common.ErrorCode, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
@@ -84,6 +91,18 @@ func setupChunkRetrievalTestNoAuth() *gin.Engine {
|
||||
return r
|
||||
}
|
||||
|
||||
func setupChunkStopParsingTest(userID string) (*gin.Engine, *mockChunkSvc) {
|
||||
mock := &mockChunkSvc{}
|
||||
h := &ChunkHandler{chunkService: mock}
|
||||
gin.SetMode(gin.TestMode)
|
||||
r := gin.New()
|
||||
r.Use(func(c *gin.Context) {
|
||||
c.Set("user", &entity.User{ID: userID})
|
||||
})
|
||||
r.DELETE("/api/v1/datasets/:dataset_id/chunks", h.StopParsing)
|
||||
return r, mock
|
||||
}
|
||||
|
||||
func setupChunkHandlerWithUser(userID string, mock *mockChunkSvc) (*gin.Engine, *ChunkHandler) {
|
||||
h := &ChunkHandler{chunkService: mock}
|
||||
gin.SetMode(gin.TestMode)
|
||||
@@ -254,6 +273,103 @@ func TestChunkRetrieval_EmptyQuestion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkStopParsing_Success(t *testing.T) {
|
||||
r, mock := setupChunkStopParsingTest("user1")
|
||||
mock.stopParsingFn = func(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
|
||||
if userID != "user1" {
|
||||
t.Fatalf("expected user1, got %q", userID)
|
||||
}
|
||||
if datasetID != "kb1" {
|
||||
t.Fatalf("expected kb1, got %q", datasetID)
|
||||
}
|
||||
if len(req.DocumentIDs) != 2 || req.DocumentIDs[0] != "doc1" || req.DocumentIDs[1] != "doc2" {
|
||||
t.Fatalf("unexpected document IDs: %#v", req.DocumentIDs)
|
||||
}
|
||||
return nil, common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("DELETE", "/api/v1/datasets/kb1/chunks", strings.NewReader(`{"document_ids":["doc1","doc2"]}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeSuccess) {
|
||||
t.Fatalf("expected code 0, got %v: %s", resp["code"], w.Body.String())
|
||||
}
|
||||
if resp["message"] != "success" {
|
||||
t.Fatalf("expected success message, got %v", resp["message"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkStopParsingRouteRequiresDocumentIDs(t *testing.T) {
|
||||
r, mock := setupChunkStopParsingTest("user1")
|
||||
mock.stopParsingFn = func(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
|
||||
t.Fatal("service should not be called when document_ids is missing")
|
||||
return nil, common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("DELETE", "/api/v1/datasets/kb1/chunks", strings.NewReader(`{}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeDataError) {
|
||||
t.Fatalf("expected data error, got %v: %s", resp["code"], w.Body.String())
|
||||
}
|
||||
if resp["message"] != "`document_ids` is required" {
|
||||
t.Fatalf("unexpected message: %v", resp["message"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkStopParsing_InvalidStateIncludesPythonErrorCode(t *testing.T) {
|
||||
r, mock := setupChunkStopParsingTest("user1")
|
||||
mock.stopParsingFn = func(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
|
||||
return &service.StopParsingResponse{
|
||||
Data: map[string]interface{}{"error_code": "DOC_STOP_PARSING_INVALID_STATE"},
|
||||
}, common.CodeDataError, errors.New("Can't stop parsing document that has not started or already completed")
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("DELETE", "/api/v1/datasets/kb1/chunks", strings.NewReader(`{"document_ids":["doc1"]}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeDataError) {
|
||||
t.Fatalf("expected data error, got %v: %s", resp["code"], w.Body.String())
|
||||
}
|
||||
data, ok := resp["data"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected data object, got %T", resp["data"])
|
||||
}
|
||||
if data["error_code"] != "DOC_STOP_PARSING_INVALID_STATE" {
|
||||
t.Fatalf("unexpected error_code: %v", data["error_code"])
|
||||
}
|
||||
if resp["message"] != "Can't stop parsing document that has not started or already completed" {
|
||||
t.Fatalf("unexpected message: %v", resp["message"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkRetrieval_WhitespaceQuestion(t *testing.T) {
|
||||
r, _ := setupChunkRetrievalTest("user1")
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ type documentServiceIface interface {
|
||||
ListIngestionTasks(userID string, datasetID *string, page, pageSize int) ([]*entity.IngestionTask, error)
|
||||
IngestDocuments(datasetID, userID string, docIDs []string) ([]*service.ParseDocumentResponse, error)
|
||||
StopIngestionTasks(tasks []string, userID string) ([]*entity.IngestionTask, error)
|
||||
Ingest(userID string, req *service.IngestDocumentRequest) (common.ErrorCode, error)
|
||||
RemoveIngestionTasks(tasks []string, userID string) ([]map[string]string, error)
|
||||
BatchUpdateDocumentStatus(userID, datasetID, status string, DocumentIDs []string) (map[string]interface{}, common.ErrorCode, error)
|
||||
}
|
||||
@@ -874,6 +875,37 @@ func (h *DocumentHandler) SetMeta(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
func (h *DocumentHandler) Ingest(c *gin.Context) {
|
||||
user, errorCode, errorMessage := GetUser(c)
|
||||
if errorCode != common.CodeSuccess {
|
||||
jsonError(c, errorCode, errorMessage)
|
||||
return
|
||||
}
|
||||
|
||||
userID := strings.TrimSpace(user.ID)
|
||||
if userID == "" {
|
||||
jsonError(c, common.CodeAuthenticationError, "No Authentication")
|
||||
return
|
||||
}
|
||||
|
||||
var req service.IngestDocumentRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
jsonError(c, common.CodeBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if code, err := h.documentService.Ingest(userID, &req); err != nil {
|
||||
jsonError(c, code, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"code": common.CodeSuccess,
|
||||
"message": "success",
|
||||
"data": true,
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteMetaRequest represents the request for deleting document metadata
|
||||
type DeleteMetaRequest struct {
|
||||
DocID string `json:"doc_id" binding:"required"`
|
||||
|
||||
@@ -45,6 +45,19 @@ type fakeDocumentService struct {
|
||||
metadataErr error
|
||||
metadataKBID string
|
||||
metadataDocIDs []string
|
||||
ingestCode common.ErrorCode
|
||||
ingestErr error
|
||||
ingestUserID string
|
||||
ingestReq *service.IngestDocumentRequest
|
||||
}
|
||||
|
||||
func (f *fakeDocumentService) Ingest(userID string, req *service.IngestDocumentRequest) (common.ErrorCode, error) {
|
||||
f.ingestUserID = userID
|
||||
f.ingestReq = req
|
||||
if f.ingestCode != 0 || f.ingestErr != nil {
|
||||
return f.ingestCode, f.ingestErr
|
||||
}
|
||||
return common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
func (f *fakeDocumentService) UpdateDatasetDocument(userID, datasetID, documentID string, req *service.UpdateDatasetDocumentRequest, present map[string]bool) (*service.UpdateDatasetDocumentResponse, common.ErrorCode, error) {
|
||||
@@ -176,6 +189,21 @@ func setupGinContextWithUser(method, path, body string) (*gin.Context, *httptest
|
||||
return c, w
|
||||
}
|
||||
|
||||
func setupDocumentIngestRoute(userID string, svc *fakeDocumentService) *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
h := &DocumentHandler{
|
||||
documentService: svc,
|
||||
datasetService: service.NewDatasetService(),
|
||||
}
|
||||
r := gin.New()
|
||||
r.Use(func(c *gin.Context) {
|
||||
c.Set("user", &entity.User{ID: userID})
|
||||
c.Set("user_id", userID)
|
||||
})
|
||||
r.POST("/api/v1/documents/ingest", h.Ingest)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestDeleteDocumentsHandler_Success(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@@ -323,6 +351,116 @@ func TestDeleteDocumentsHandler_MissingDatasetID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentHandlerIngestMatchesPythonResponseShape(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
fake := &fakeDocumentService{}
|
||||
h := &DocumentHandler{
|
||||
documentService: fake,
|
||||
datasetService: service.NewDatasetService(),
|
||||
}
|
||||
|
||||
c, w := setupGinContextWithUser("POST", "/api/v1/documents/ingest", `{"doc_ids":["doc-1"],"run":"1"}`)
|
||||
h.Ingest(c)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeSuccess) {
|
||||
t.Fatalf("expected top-level code 0, got %v", resp["code"])
|
||||
}
|
||||
if resp["data"] != true {
|
||||
t.Fatalf("expected top-level data=true, got %#v", resp["data"])
|
||||
}
|
||||
if _, ok := resp["data"].(map[string]interface{}); ok {
|
||||
t.Fatalf("response must not nest code/message under data: %#v", resp["data"])
|
||||
}
|
||||
if fake.ingestUserID != "user-1" {
|
||||
t.Fatalf("expected user-1, got %q", fake.ingestUserID)
|
||||
}
|
||||
if fake.ingestReq == nil || len(fake.ingestReq.DocIDs) != 1 || fake.ingestReq.DocIDs[0] != "doc-1" {
|
||||
t.Fatalf("unexpected ingest request: %#v", fake.ingestReq)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentIngestRoutePassesPythonBodyToService(t *testing.T) {
|
||||
fake := &fakeDocumentService{}
|
||||
r := setupDocumentIngestRoute("user-1", fake)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("POST", "/api/v1/documents/ingest", strings.NewReader(`{"doc_ids":["doc-1","doc-2"],"run":1,"delete":true,"apply_kb":true}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeSuccess) || resp["data"] != true {
|
||||
t.Fatalf("unexpected response: %s", w.Body.String())
|
||||
}
|
||||
if fake.ingestUserID != "user-1" {
|
||||
t.Fatalf("userID = %q, want user-1", fake.ingestUserID)
|
||||
}
|
||||
if fake.ingestReq == nil {
|
||||
t.Fatal("service did not receive ingest request")
|
||||
}
|
||||
if len(fake.ingestReq.DocIDs) != 2 || fake.ingestReq.DocIDs[0] != "doc-1" || fake.ingestReq.DocIDs[1] != "doc-2" {
|
||||
t.Fatalf("doc_ids = %#v, want [doc-1 doc-2]", fake.ingestReq.DocIDs)
|
||||
}
|
||||
if fmt.Sprint(fake.ingestReq.Run) != "1" {
|
||||
t.Fatalf("run = %#v, want 1", fake.ingestReq.Run)
|
||||
}
|
||||
if !fake.ingestReq.Delete {
|
||||
t.Fatal("delete = false, want true")
|
||||
}
|
||||
if !fake.ingestReq.ApplyKB {
|
||||
t.Fatal("apply_kb = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentHandlerIngestPropagatesServiceErrorCode(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
fake := &fakeDocumentService{
|
||||
ingestCode: common.CodeAuthenticationError,
|
||||
ingestErr: fmt.Errorf("No authorization."),
|
||||
}
|
||||
h := &DocumentHandler{
|
||||
documentService: fake,
|
||||
datasetService: service.NewDatasetService(),
|
||||
}
|
||||
|
||||
c, w := setupGinContextWithUser("POST", "/api/v1/documents/ingest", `{"doc_ids":["doc-1"],"run":"1"}`)
|
||||
h.Ingest(c)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["code"] != float64(common.CodeAuthenticationError) {
|
||||
t.Fatalf("expected auth error code, got %v", resp["code"])
|
||||
}
|
||||
if resp["message"] != "No authorization." {
|
||||
t.Fatalf("unexpected message: %v", resp["message"])
|
||||
}
|
||||
if resp["data"] != nil {
|
||||
t.Fatalf("expected nil data, got %#v", resp["data"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopParseDocumentsHandler_EmptyDocIDs(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user