feat(go-api): add Go support for POST /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks (#16256)

## Summary Add the Go implementation of `POST /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks`. This wires the full create-chunk path in Go: - router and handler registration - request/response structs - chunk creation service logic - embedding generation - chunk insert into doc engine - chunk/token counter increment - `tag_feas` validation - `image_base64` decoding and chunk image storage/merge - unit tests for handler and service ## Testing Unit tests: - `/usr/local/go/bin/go test ./internal/handler` - `/usr/local/go/bin/go test ./internal/service/chunk` - `/usr/local/go/bin/go test ./internal/service` - `/usr/local/go/bin/go test ./...` All passed locally. Manual curl checks: - basic text chunk: Go passed - chunk with `important_keywords` / `questions` / `tag_kwd` / `tag_feas`: Go passed - blank content validation: Go matched expected `code=102` - invalid `image_base64` validation: Go matched expected `code=102` - image upload and repeated image upload / merge path: Go passed twice
2026-07-17 13:17:20 +08:00 · 2026-06-25 14:15:29 +08:00
parent d44359826d
commit 54fb5b0fa7
7 changed files with 1179 additions and 5 deletions
--- a/internal/handler/chunk.go
+++ b/internal/handler/chunk.go
@@ -17,6 +17,7 @@ package handler

 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"ragflow/internal/common"
@@ -38,6 +39,7 @@ type chunkService interface {
 	UpdateChunk(req *service.UpdateChunkRequest, userID string) error
 	RemoveChunks(req *service.RemoveChunksRequest, userID string) (int64, error)
 	Parse(userID, datasetID string, req *service.ParseFileRequest) (map[string]interface{}, common.ErrorCode, error)
+	AddChunk(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error)
 	StopParsing(userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error)
 }

@@ -788,3 +790,131 @@ func (h *ChunkHandler) RemoveChunks(c *gin.Context) {
 		"message": "success",
 	})
 }
+
+func addChunkStringField(rawBody map[string]json.RawMessage, field string) (string, error) {
+	raw, ok := rawBody[field]
+	if !ok {
+		return "", nil
+	}
+	var value string
+	if err := json.Unmarshal(raw, &value); err != nil {
+		return "", fmt.Errorf("`%s` must be a string", field)
+	}
+	return value, nil
+}
+
+func addChunkStringPtrField(rawBody map[string]json.RawMessage, field string) (*string, error) {
+	raw, ok := rawBody[field]
+	if !ok {
+		return nil, nil
+	}
+	var value string
+	if err := json.Unmarshal(raw, &value); err != nil {
+		return nil, fmt.Errorf("`%s` must be a string", field)
+	}
+	return &value, nil
+}
+
+func addChunkStringListField(rawBody map[string]json.RawMessage, field, listMessage, elementMessage string) ([]string, error) {
+	raw, ok := rawBody[field]
+	if !ok {
+		return nil, nil
+	}
+	var values []interface{}
+	if err := json.Unmarshal(raw, &values); err != nil {
+		return nil, errors.New(listMessage)
+	}
+	result := make([]string, len(values))
+	for i, value := range values {
+		str, ok := value.(string)
+		if !ok {
+			return nil, errors.New(elementMessage)
+		}
+		result[i] = str
+	}
+	return result, nil
+}
+
+func addChunkResponseMessage(code common.ErrorCode, err error) string {
+	if code == common.CodeServerError {
+		common.Warn("add chunk failed", zap.String("error", err.Error()))
+		return "Failed to add chunk"
+	}
+	return err.Error()
+}
+
+func (h *ChunkHandler) AddChunk(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		jsonError(c, errorCode, errorMessage)
+		return
+	}
+
+	userID := user.ID
+	datasetID, documentID := strings.TrimSpace(c.Param("dataset_id")), strings.TrimSpace(c.Param("document_id"))
+
+	var rawBody map[string]json.RawMessage
+	if err := json.NewDecoder(c.Request.Body).Decode(&rawBody); err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	content, err := addChunkStringField(rawBody, "content")
+	if err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	importantKeywords, err := addChunkStringListField(rawBody, "important_keywords", "`important_keywords` is required to be a list", "`important_keywords` must be a list of strings")
+	if err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	questions, err := addChunkStringListField(rawBody, "questions", "`questions` is required to be a list", "`questions` must be a list of strings")
+	if err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	tagKwd, err := addChunkStringListField(rawBody, "tag_kwd", "`tag_kwd` is required to be a list", "`tag_kwd` must be a list of strings")
+	if err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	imageBase64, err := addChunkStringPtrField(rawBody, "image_base64")
+	if err != nil {
+		jsonError(c, common.CodeArgumentError, err.Error())
+		return
+	}
+	var tagFeas interface{}
+	if raw, ok := rawBody["tag_feas"]; ok {
+		if err := json.Unmarshal(raw, &tagFeas); err != nil {
+			jsonError(c, common.CodeArgumentError, err.Error())
+			return
+		}
+	}
+
+	req := service.AddChunkRequest{
+		DatasetID:         datasetID,
+		DocumentID:        documentID,
+		Content:           content,
+		ImportantKeywords: importantKeywords,
+		Questions:         questions,
+		TagKwd:            tagKwd,
+		TagFeas:           tagFeas,
+		ImageBase64:       imageBase64,
+	}
+
+	resp, err := h.chunkService.AddChunk(&req, userID)
+	if err != nil {
+		if codedErr, ok := err.(service.ErrorCoder); ok {
+			jsonError(c, codedErr.Code(), addChunkResponseMessage(codedErr.Code(), err))
+			return
+		}
+		jsonError(c, common.CodeServerError, addChunkResponseMessage(common.CodeServerError, err))
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"code":    0,
+		"data":    resp,
+		"message": "success",
+	})
+}
--- a/internal/handler/chunk_test.go
+++ b/internal/handler/chunk_test.go
@@ -20,6 +20,7 @@ import (
 // Only the methods actually called by the test are set; others panic.
 type mockChunkSvc struct {
 	retrievalTestFn func(req *service.RetrievalTestRequest, userID string) (*service.RetrievalTestResponse, error)
+	addChunkFn      func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error)
 	listFn          func(req *service.ListChunksRequest, userID string) (*service.ListChunksResponse, error)
 	switchChunksFn  func(userID, datasetID, documentID string, availableInt int, chunkIDs []string) error
 	updateChunkFn   func(req *service.UpdateChunkRequest, userID string) error
@@ -68,6 +69,12 @@ func (m *mockChunkSvc) StopParsing(userID, datasetID string, req service.StopPar
 func (m *mockChunkSvc) Parse(string, string, *service.ParseFileRequest) (map[string]interface{}, common.ErrorCode, error) {
 	panic("not implemented")
 }
+func (m *mockChunkSvc) AddChunk(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+	if m.addChunkFn != nil {
+		return m.addChunkFn(req, userID)
+	}
+	return &service.AddChunkResponse{Chunk: map[string]interface{}{"id": "chunk-1"}}, nil
+}

 func setupChunkRetrievalTest(userID string) (*gin.Engine, *mockChunkSvc) {
 	mock := &mockChunkSvc{}
@@ -563,3 +570,194 @@ func TestChunkRetrieval_ServiceError(t *testing.T) {
 		t.Errorf("internal error details leaked to response: %q", msg)
 	}
 }
+
+type addChunkTestError struct {
+	code common.ErrorCode
+	msg  string
+}
+
+func (e addChunkTestError) Error() string          { return e.msg }
+func (e addChunkTestError) Code() common.ErrorCode { return e.code }
+
+func TestChunkHandlerAddChunkSuccess(t *testing.T) {
+	mock := &mockChunkSvc{
+		addChunkFn: func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+			if userID != "user1" {
+				t.Fatalf("userID = %q, want user1", userID)
+			}
+			if req.DatasetID != "kb1" || req.DocumentID != "doc1" || req.Content != "chunk body" {
+				t.Fatalf("unexpected request: %#v", req)
+			}
+			return &service.AddChunkResponse{Chunk: map[string]interface{}{"id": "chunk-1", "content": req.Content}}, nil
+		},
+	}
+
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	r.Use(func(c *gin.Context) {
+		c.Set("user", &entity.User{ID: "user1"})
+	})
+	h := &ChunkHandler{chunkService: mock}
+	r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(`{"content":"chunk body"}`))
+	req.Header.Set("Content-Type", "application/json")
+	r.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["code"] != float64(common.CodeSuccess) {
+		t.Fatalf("expected success code, got %v", resp["code"])
+	}
+}
+
+func TestChunkHandlerAddChunkPathIDsOverrideBody(t *testing.T) {
+	mock := &mockChunkSvc{
+		addChunkFn: func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+			if req.DatasetID != "kb1" || req.DocumentID != "doc1" {
+				t.Fatalf("path IDs were not preserved: %#v", req)
+			}
+			if req.Content != "chunk body" {
+				t.Fatalf("unexpected content: %#v", req)
+			}
+			return &service.AddChunkResponse{Chunk: map[string]interface{}{"id": "chunk-1"}}, nil
+		},
+	}
+
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	r.Use(func(c *gin.Context) {
+		c.Set("user", &entity.User{ID: "user1"})
+	})
+	h := &ChunkHandler{chunkService: mock}
+	r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(`{"dataset_id":"evil-kb","document_id":"evil-doc","content":"chunk body"}`))
+	req.Header.Set("Content-Type", "application/json")
+	r.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+func TestChunkHandlerAddChunkCodedError(t *testing.T) {
+	mock := &mockChunkSvc{
+		addChunkFn: func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+			return nil, addChunkTestError{code: common.CodeDataError, msg: "`content` is required"}
+		},
+	}
+
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	r.Use(func(c *gin.Context) {
+		c.Set("user", &entity.User{ID: "user1"})
+	})
+	h := &ChunkHandler{chunkService: mock}
+	r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(`{"content":" "}`))
+	req.Header.Set("Content-Type", "application/json")
+	r.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["code"] != float64(common.CodeDataError) {
+		t.Fatalf("expected data error code, got %v", resp["code"])
+	}
+}
+
+func TestChunkHandlerAddChunkValidatesListFields(t *testing.T) {
+	tests := []struct {
+		name    string
+		body    string
+		wantMsg string
+	}{
+		{
+			name:    "important keywords type",
+			body:    `{"content":"chunk body","important_keywords":{}}`,
+			wantMsg: "`important_keywords` is required to be a list",
+		},
+		{
+			name:    "tag kwd element type",
+			body:    `{"content":"chunk body","tag_kwd":[1]}`,
+			wantMsg: "`tag_kwd` must be a list of strings",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mock := &mockChunkSvc{
+				addChunkFn: func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+					t.Fatal("service should not be called for invalid request")
+					return nil, nil
+				},
+			}
+			gin.SetMode(gin.TestMode)
+			r := gin.New()
+			r.Use(func(c *gin.Context) {
+				c.Set("user", &entity.User{ID: "user1"})
+			})
+			h := &ChunkHandler{chunkService: mock}
+			r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+			w := httptest.NewRecorder()
+			req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(tt.body))
+			req.Header.Set("Content-Type", "application/json")
+			r.ServeHTTP(w, req)
+
+			var resp map[string]interface{}
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			if resp["message"] != tt.wantMsg {
+				t.Fatalf("message = %v, want %q", resp["message"], tt.wantMsg)
+			}
+		})
+	}
+}
+
+func TestChunkHandlerAddChunkHidesServerErrorDetails(t *testing.T) {
+	mock := &mockChunkSvc{
+		addChunkFn: func(req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+			return nil, addChunkTestError{code: common.CodeServerError, msg: "encode chunk embedding: provider secret"}
+		},
+	}
+
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	r.Use(func(c *gin.Context) {
+		c.Set("user", &entity.User{ID: "user1"})
+	})
+	h := &ChunkHandler{chunkService: mock}
+	r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(`{"content":"chunk body"}`))
+	req.Header.Set("Content-Type", "application/json")
+	r.ServeHTTP(w, req)
+
+	var resp map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["message"] != "Failed to add chunk" {
+		t.Fatalf("message = %v, want generic failure", resp["message"])
+	}
+	if strings.Contains(w.Body.String(), "provider secret") {
+		t.Fatalf("server error details leaked: %s", w.Body.String())
+	}
+}