Revert "feat: Go knowledge compiler with scheduler-driven dataset compilation" (#17897)

Reverts infiniflow/ragflow#17881

internal/handler/compilation_status_test.go

@@ -1,195 +0,0 @@
-//
-//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-package handler
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"testing"
-
-	"github.com/gin-gonic/gin"
-	"github.com/glebarez/sqlite"
-	"gorm.io/gorm"
-
-	"ragflow/internal/common"
-	"ragflow/internal/dao"
-	"ragflow/internal/entity"
-	dataset "ragflow/internal/service/dataset"
-)
-
-// setupCompilationStatusHandlerDB migrates the minimal schema for the
-// GET /datasets/:id/compilation/status handler and pushes it onto dao.DB.
-func setupCompilationStatusHandlerDB(t *testing.T) *gorm.DB {
-	t.Helper()
-	db, err := gorm.Open(sqlite.Open("file:"+url.QueryEscape(t.Name())+"?mode=memory&cache=shared"), &gorm.Config{
-		TranslateError: true,
-	})
-	if err != nil {
-		t.Fatalf("failed to open sqlite: %v", err)
-	}
-	if err := db.AutoMigrate(
-		&entity.Knowledgebase{},
-		&entity.KnowledgeCompileDataset{},
-	); err != nil {
-		t.Fatalf("failed to migrate test schema: %v", err)
-	}
-	origDB := dao.DB
-	dao.DB = db
-	t.Cleanup(func() { dao.DB = origDB })
-	return db
-}
-
-func insertCompilationStatusHandlerKB(t *testing.T, kbID, ownerID string) {
-	t.Helper()
-	status := string(entity.StatusValid)
-	kb := &entity.Knowledgebase{
-		ID:         kbID,
-		TenantID:   ownerID,
-		Name:       "compile-status-handler-kb",
-		EmbdID:     "BAAI/bge-large-zh-v1.5@Builtin",
-		CreatedBy:  ownerID,
-		Permission: string(entity.TenantPermissionMe),
-		Status:     &status,
-	}
-	if err := dao.DB.Create(kb).Error; err != nil {
-		t.Fatalf("insert kb: %v", err)
-	}
-}
-
-func newCompilationStatusHandlerRouter() *gin.Engine {
-	gin.SetMode(gin.TestMode)
-	h := NewDatasetsHandler(dataset.NewDatasetService(), nil)
-	r := gin.New()
-	r.GET("/api/v1/datasets/:dataset_id/compilation/status", func(c *gin.Context) {
-		c.Set("user", &entity.User{ID: "user-1"})
-		h.GetCompilationStatus(c)
-	})
-	return r
-}
-
-type compilationStatusResponse struct {
-	Code    int                    `json:"code"`
-	Message string                 `json:"message"`
-	Data    map[string]interface{} `json:"data"`
-}
-
-func getCompilationStatus(t *testing.T, r *gin.Engine, datasetID string) (int, compilationStatusResponse) {
-	t.Helper()
-	resp := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodGet,
-		"/api/v1/datasets/"+datasetID+"/compilation/status", nil)
-	r.ServeHTTP(resp, req)
-	var body compilationStatusResponse
-	if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {
-		t.Fatalf("unmarshal response: %v body=%s", err, resp.Body.String())
-	}
-	return resp.Code, body
-}
-
-// TestCompilationStatusHandler_NoRowIdle verifies a dataset with no scheduling
-// row returns idle with zero counts.
-func TestCompilationStatusHandler_NoRowIdle(t *testing.T) {
-	db := setupCompilationStatusHandlerDB(t)
-	insertCompilationStatusHandlerKB(t, "kb-status-idle", "user-1")
-	_ = db
-
-	status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-idle")
-	if status != http.StatusOK {
-		t.Fatalf("status=%d want 200", status)
-	}
-	if body.Code != int(common.CodeSuccess) {
-		t.Fatalf("code=%d message=%q", body.Code, body.Message)
-	}
-	if body.Data["state"] != entity.DatasetStateIdle {
-		t.Fatalf("state=%v want idle", body.Data["state"])
-	}
-	if n, _ := body.Data["inflight"].(float64); n != 0 {
-		t.Fatalf("inflight=%v want 0", body.Data["inflight"])
-	}
-	if n, _ := body.Data["backlog"].(float64); n != 0 {
-		t.Fatalf("backlog=%v want 0", body.Data["backlog"])
-	}
-}
-
-// TestCompilationStatusHandler_FullOutput locks the JSON contract for a row
-// with state, inflight/backlog counts, and error diagnostic.
-func TestCompilationStatusHandler_FullOutput(t *testing.T) {
-	db := setupCompilationStatusHandlerDB(t)
-	insertCompilationStatusHandlerKB(t, "kb-status-full", "user-1")
-
-	row := entity.KnowledgeCompileDataset{
-		DatasetID:      "kb-status-full",
-		TenantID:       "user-1",
-		BacklogDocIDs:  `[{"doc_id":"d2","event_type":"completed","seq":2}]`,
-		InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1}]`,
-		State:          entity.DatasetStatePending,
-		ErrorMsg:       "merge failed: boom",
-	}
-	if err := db.Create(&row).Error; err != nil {
-		t.Fatalf("insert scheduling row: %v", err)
-	}
-
-	status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-full")
-	if status != http.StatusOK {
-		t.Fatalf("status=%d want 200", status)
-	}
-	if body.Code != int(common.CodeSuccess) {
-		t.Fatalf("code=%d message=%q", body.Code, body.Message)
-	}
-	if body.Data["state"] != entity.DatasetStatePending {
-		t.Fatalf("state=%v want pending", body.Data["state"])
-	}
-	if n, _ := body.Data["inflight"].(float64); n != 1 {
-		t.Fatalf("inflight=%v want 1", body.Data["inflight"])
-	}
-	if n, _ := body.Data["backlog"].(float64); n != 1 {
-		t.Fatalf("backlog=%v want 1", body.Data["backlog"])
-	}
-	if body.Data["error"] != "merge failed: boom" {
-		t.Fatalf("error=%v want %q", body.Data["error"], "merge failed: boom")
-	}
-}
-
-// TestCompilationStatusHandler_Unauthorized verifies a user who does not own the
-// dataset is rejected with a data error (HTTP 200 + non-zero code, matching the
-// handler's ErrorWithCode contract).
-func TestCompilationStatusHandler_Unauthorized(t *testing.T) {
-	db := setupCompilationStatusHandlerDB(t)
-	// KB is owned by user-1; the router sets user to user-1, so this test must
-	// exercise the case where the KB belongs to a different owner. We insert the
-	// KB under a different owner tenant than the request user by re-pointing the
-	// KB owner to "other-owner".
-	insertCompilationStatusHandlerKB(t, "kb-status-forbidden", "other-owner")
-	if err := db.Create(&entity.KnowledgeCompileDataset{
-		DatasetID:      "kb-status-forbidden",
-		TenantID:       "other-owner",
-		BacklogDocIDs:  "[]",
-		InflightDocIDs: "[]",
-		State:          entity.DatasetStateRunning,
-	}).Error; err != nil {
-		t.Fatalf("insert scheduling row: %v", err)
-	}
-
-	_, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-forbidden")
-	if body.Code != int(common.CodeDataError) {
-		t.Fatalf("code=%d want %d", body.Code, common.CodeDataError)
-	}
-	if body.Message != "no authorization" {
-		t.Fatalf("message=%q want %q", body.Message, "no authorization")
-	}
-}

internal/handler/components_testpkg/components_handler_test.go

@@ -134,8 +134,8 @@ func TestComponentsHandler_NoFilter(t *testing.T) {
 
 // TestComponentsHandler_FilterIngestion verifies the
 // ?category=ingestion filter returns the ingestion components
-// (Compiler, Extractor, File, Parser, Tokenizer + 9 chunker variants).
-// Names must be sorted ascending (plan §4 task 1 stable output).
+// (Extractor, File, Parser, Tokenizer + 9 chunker variants). Names
+// must be sorted ascending (plan §4 task 1 stable output).
 func TestComponentsHandler_FilterIngestion(t *testing.T) {
 	eng := newComponentsTestRig(t)
 	w := doRequest(t, eng, "/api/v1/components?category=ingestion")
@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
 	_, _, data := decodeEnvelope(t, w.Body.Bytes())
 
 	wantNames := []string{
-		"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+		"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
 		"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
 		"titlechunker", "tokenchunker", "tokenizer",
 	}
@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
 	_, _, data := decodeEnvelope(t, w.Body.Bytes())
 
 	wantNames := []string{
-		"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+		"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
 		"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
 		"titlechunker", "tokenchunker", "tokenizer",
 	}
@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
 	}
 	_, _, data := decodeEnvelope(t, w.Body.Bytes())
 	wantNames := []string{
-		"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+		"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
 		"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
 		"titlechunker", "tokenchunker", "tokenizer",
 	}

internal/handler/dataset.go

@@ -1008,28 +1008,114 @@ func (h *DatasetsHandler) AggregateTags(c *gin.Context) {
 	common.SuccessWithData(c, result, "success")
 }
 
-// GetCompilationStatus returns the dataset-level knowledge-compile lifecycle
-// state (scheduler contract for API_PROXY_SCHEME=go/hybrid). It replaces the
-// Python-era TraceIndex task-progress endpoint for the Go backend.
-func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {
+// RunIndex Run an indexing task (graph/raptor/mindmap) for a dataset.
+func (h *DatasetsHandler) RunIndex(c *gin.Context) {
 	user, errorCode, errorMessage := GetUser(c)
 	if errorCode != common.CodeSuccess {
 		common.ErrorWithCode(c, errorCode, errorMessage)
 		return
 	}
+
 	datasetID := strings.TrimSpace(c.Param("dataset_id"))
 	if datasetID == "" {
 		common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")
 		return
 	}
+
 	userID := strings.TrimSpace(user.ID)
+	if userID == "" {
+		common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")
+		return
+	}
+
 	ctx := c.Request.Context()
-	st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID)
+	indexType := strings.ToLower(strings.TrimSpace(c.Query("type")))
+	data, code, err := h.datasetsService.RunIndex(ctx, userID, datasetID, indexType)
 	if err != nil {
 		common.ErrorWithCode(c, code, err.Error())
 		return
 	}
-	common.SuccessWithData(c, st, "success")
+
+	common.SuccessWithData(c, data, "success")
+}
+
+// TraceIndex Trace an indexing task (graph/raptor/mindmap) for a dataset.
+func (h *DatasetsHandler) TraceIndex(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		common.ErrorWithCode(c, errorCode, errorMessage)
+		return
+	}
+
+	datasetID := strings.TrimSpace(c.Param("dataset_id"))
+	if datasetID == "" {
+		common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")
+		return
+	}
+
+	userID := strings.TrimSpace(user.ID)
+	if userID == "" {
+		common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")
+		return
+	}
+
+	ctx := c.Request.Context()
+
+	indexType := strings.ToLower(strings.TrimSpace(c.Query("type")))
+	result, code, err := h.datasetsService.TraceIndex(ctx, datasetID, userID, indexType)
+	if err != nil {
+		common.ErrorWithCode(c, code, err.Error())
+		return
+	}
+	if result == nil {
+		common.SuccessWithData(c, map[string]interface{}{}, "success")
+		return
+	}
+
+	common.SuccessWithData(c, result, "success")
+}
+
+// DeleteIndex Delete an indexing task (graph/raptor/mindmap) for a dataset.
+func (h *DatasetsHandler) DeleteIndex(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		common.ErrorWithCode(c, errorCode, errorMessage)
+		return
+	}
+
+	datasetID := strings.TrimSpace(c.Param("dataset_id"))
+	if datasetID == "" {
+		common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")
+		return
+	}
+
+	userID := strings.TrimSpace(user.ID)
+	if userID == "" {
+		common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")
+		return
+	}
+
+	indexType := strings.ToLower(strings.TrimSpace(c.Param("index_type")))
+	if indexType == "" {
+		indexType = strings.ToLower(strings.TrimSpace(c.Query("type")))
+	}
+
+	wipeArg := strings.ToLower(strings.TrimSpace(c.DefaultQuery("wipe", "true")))
+	wipe := true
+	switch wipeArg {
+	case "false", "0", "no", "off":
+		wipe = false
+	}
+
+	ctx := c.Request.Context()
+
+	code, err := h.datasetsService.DeleteIndex(ctx, userID, datasetID, indexType, wipe)
+	if err != nil {
+		common.ErrorWithCode(c, code, err.Error())
+		return
+	}
+
+	common.SuccessWithData(c, map[string]interface{}{}, "success")
 }
 
 // ListMetadataFlattened handles GET /api/v1/datasets/metadata/flattened.

internal/handler/dataset_artifact.go

@@ -119,9 +119,7 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {
 		common.ErrorWithCode(c, common.CodeDataError, err.Error())
 		return
 	}
-	// Python's list_wiki_pages returns {total, items}; align the Go port so the
-	// shared frontend (which reads data.items) stays compatible.
-	common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")
+	common.SuccessWithData(c, gin.H{"total": total, "pages": items}, "success")
 }
 
 // UpdateArtifact handles PUT /artifacts/<page_type>/<slug> — edit a wiki page.
@@ -241,9 +239,7 @@ func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {
 		common.ErrorWithCode(c, common.CodeDataError, err.Error())
 		return
 	}
-	// Python's list_wiki_topics returns {total, items}; align the Go port so the
-	// shared frontend (which reads data.items) stays compatible.
-	common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")
+	common.SuccessWithData(c, gin.H{"total": total, "topics": items}, "success")
 }
 
 // GetArtifactAlteration handles GET /artifacts/alteration — wiki alteration summary.