diff --git a/internal/handler/agent.go b/internal/handler/agent.go
index 9d1a51d0a4..6b451f4fe9 100644
--- a/internal/handler/agent.go
+++ b/internal/handler/agent.go
@@ -29,12 +29,15 @@ import (
"time"
"github.com/gin-gonic/gin"
+ "github.com/google/uuid"
"go.uber.org/zap"
"ragflow/internal/agent/canvas"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ pipelinepkg "ragflow/internal/ingestion/pipeline"
+ "ragflow/internal/ingestion/task"
"ragflow/internal/service"
"ragflow/internal/service/file"
"ragflow/internal/utility"
@@ -91,6 +94,28 @@ type AgentHandler struct {
// tenant-only authorization (i.e. cannot verify the doc, so the
// check is skipped — same shape as the pre-port behaviour).
documentService documentAccessChecker
+ // redisGet fetches a raw string from Redis. Defaults to the global
+ // client (redis.Get) so production behaviour is unchanged; tests inject
+ // a miniredis-backed getter to exercise GetAgentLogs without a live
+ // Redis (mirrors the newExecutor injection pattern).
+ redisGet func(key string) (string, error)
+ // redisStore writes the debug-run log array. Defaults to the global
+ // client (redis.Get, which satisfies task.DebugLogStore); tests inject a
+ // miniredis-backed writer so runCanvasPipelineDebug can be exercised without a
+ // live Redis.
+ redisStore task.DebugLogStore
+ // newExecutor builds the pipeline executor for a debug run. Defaults to
+ // task.NewPipelineExecutor; tests inject a fake so the debug path can be
+ // driven without a real canvas/DSL.
+ newExecutor func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error)
+}
+
+// debugExecutor is the minimal executor surface runCanvasPipelineDebug needs. It is an
+// interface (not *task.PipelineExecutor) so tests can substitute a fake that
+// emits progress through the attached sink. *task.PipelineExecutor satisfies it.
+type debugExecutor interface {
+ WithProgressSink(sink pipelinepkg.ProgressSink) *task.PipelineExecutor
+ Execute(ctx context.Context) (*task.PipelineResult, error)
}
// WithDocumentService injects the document service used by
@@ -110,9 +135,37 @@ func NewAgentHandler(agentService *service.AgentService, fileService *file.FileS
chatRunner: agentService,
fileService: fileService,
loader: agentService,
+ redisGet: func(key string) (string, error) { return redis.Get().Get(key) },
+ redisStore: redis.Get(),
+ newExecutor: func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {
+ return task.NewPipelineExecutor(taskCtx, canvasID, docBulkSize)
+ },
}
}
+// WithRedisGetter overrides the Redis string getter used by GetAgentLogs.
+// Tests pass a miniredis-backed getter so the debug-log polling endpoint can
+// be exercised end-to-end without a live Redis.
+func (h *AgentHandler) WithRedisGetter(f func(key string) (string, error)) *AgentHandler {
+ h.redisGet = f
+ return h
+}
+
+// WithRedisStore overrides the Redis writer used by runCanvasPipelineDebug to persist
+// the debug-run log. Tests pass a miniredis-backed writer.
+func (h *AgentHandler) WithRedisStore(s task.DebugLogStore) *AgentHandler {
+ h.redisStore = s
+ return h
+}
+
+// WithNewExecutor overrides the pipeline executor factory used by
+// runCanvasPipelineDebug. Tests inject a fake that emits progress through the
+// attached sink without a real canvas/DSL.
+func (h *AgentHandler) WithNewExecutor(f func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error)) *AgentHandler {
+ h.newExecutor = f
+ return h
+}
+
// ListAgents lists agent canvases for the current user.
// @Summary List Agents
// @Description List agent canvases accessible to the current user (Home dashboard tile)
@@ -486,6 +539,127 @@ func readUserInput(c *gin.Context) string {
return c.Query("user_input")
}
+// runCanvasPipelineDebug runs a canvas in pipeline dry-run (debug) mode: it builds
+// an in-memory debug TaskContext and executes the pipeline synchronously,
+// returning the produced chunks. The run is fully synchronous: the complete
+// debug log (including the terminal END marker) is flushed to Redis BEFORE
+// this method returns, so the message_id it yields is for *replay* —
+// re-fetching the log after a page refresh or from a shared link — not for
+// incremental streaming; the front-end's poll therefore receives a finished
+// log on the first request.
+//
+// It performs no persistence (no MinIO upload, index insert, or pipeline_log)
+// because the debug context carries no KB (KB.ID == ""), which is the single
+// debug signal used across the ingestion pipeline: the tokenizer skips
+// embedding and the executor skips the persist stage. kb_id == "" occurs ONLY
+// in debug mode; production ingestion always supplies a KB, so this path is
+// never taken in normal operation. Debug takes no kb_id at all: it never
+// resolves an embedder or writes to a KB, and parse/chunk work without it.
+// Callers extract the (fileName, fileData) pair from their own request shape
+// and pass them in.
+func (h *AgentHandler) runCanvasPipelineDebug(ctx context.Context, user *entity.User,
+ canvasID, fileName string, fileData []byte) (*task.PipelineResult, error) {
+ // messageID is the polling key the front-end reads from the run response
+ // (see web/src/pages/agent/hooks/use-run-dataflow.ts:40) to fetch the debug
+ // log. It is generated up-front so the response always carries a stable key.
+ messageID := uuid.New().String()
+
+ // Force an empty KB: a debug context has no knowledgebase, so kb_id == ""
+ // holds and the debug signal is unambiguous.
+ taskCtx := task.NewDebugTaskContext(user.ID, canvasID, fileName, fileData)
+
+ exec, err := h.newExecutor(taskCtx, canvasID, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ // Attach a sink that records component progress into the
+ // [{component_id, trace}] array the front-end polls for.
+ sink := task.NewDebugLogSink(canvasID, messageID, h.redisStore)
+ exec.WithProgressSink(sink)
+
+ result, runErr := exec.Execute(ctx)
+ // Flush even on error so the front-end can poll the failure log; the END
+ // marker carries the error and keeps the run detectable as finished.
+ sink.Flush(ctx, runErr)
+ if result == nil {
+ // The executor may return (nil, err) on failure; synthesize a result so
+ // the polling key survives to the response and the front-end can still
+ // reach the failure log written just above.
+ result = &task.PipelineResult{}
+ }
+ result.MessageID = messageID
+ return result, runErr
+}
+
+// respondWithDebugResult writes the outcome of a dataflow debug run. On a run
+// failure it keeps a non-success code (errors must not be masked as success)
+// but still carries message_id in the error envelope's data, because
+// runCanvasPipelineDebug already flushed the failure log to Redis — the front-end
+// needs that key to poll the failure timeline. On success it returns the
+// message_id (the polling key) together with the produced chunks. Empty chunk
+// slices become `[]` rather than `null` so the response shape stays stable.
+// This is the single wire-shape helper for the chat/completions DataFlow debug
+// entry point, and it matches the contract the front-end expects:
+// `data.data.message_id` drives the log-poll loop and `data.data.chunks`
+// renders the debug output.
+func respondWithDebugResult(c *gin.Context, result *task.PipelineResult, err error) {
+ if err != nil {
+ messageID := ""
+ if result != nil {
+ messageID = result.MessageID
+ }
+ common.ResponseWithCodeData(c, common.CodeServerError, gin.H{"message_id": messageID}, err.Error())
+ return
+ }
+ messageID := ""
+ var chunks []map[string]any
+ if result != nil {
+ messageID = result.MessageID
+ chunks = result.Chunks
+ }
+ if len(chunks) == 0 {
+ chunks = []map[string]any{}
+ }
+ common.SuccessWithData(c, gin.H{"message_id": messageID, "chunks": chunks}, "success")
+}
+
+// extractChatDebugFile pulls the first uploaded file from a chat/completions
+// request into (fileName, bytes) for a dataflow debug run. Unlike a debug
+// multipart upload, chat/completions `files` are storage references
+// (id/name/created_by); the raw bytes are fetched from the per-user downloads
+// bucket via the file service. When no usable file is present it returns a
+// default name and a nil slice so the pipeline can still run file-less.
+func (h *AgentHandler) extractChatDebugFile(ctx context.Context, files [][]map[string]interface{}, user *entity.User) (string, []byte) {
+ if len(files) == 0 {
+ return "debug", nil
+ }
+ fileList := files[0]
+ if len(fileList) == 0 {
+ return "debug", nil
+ }
+ fd := fileList[0]
+ name, _ := fd["name"].(string)
+ id, _ := fd["id"].(string)
+ if id == "" {
+ if name == "" {
+ return "debug", nil
+ }
+ return name, nil
+ }
+ data, err := h.fileService.DownloadAgentFile(ctx, user.ID, id)
+ if err != nil || len(data) == 0 {
+ if name == "" {
+ return "debug", nil
+ }
+ return name, nil
+ }
+ if name == "" {
+ name = "debug"
+ }
+ return name, data
+}
+
// sanitiseRunEventError passes through the error event payload
// unchanged. The runner serialises canvas.ErrorEvent ({"message": ...})
// before push, so when the payload round-trips through JSON the
@@ -889,7 +1063,16 @@ type agentChatCompletionsRequest struct {
Model string `json:"model"`
Messages []map[string]interface{} `json:"messages"`
ReturnTrace bool `json:"return_trace"`
- Files []map[string]interface{} `json:"files"`
+ // Files carries the uploaded file references for a run. The RAGFlow web
+ // front-end wraps the file list one extra level (a list of per-turn file
+ // lists), so the wire shape is `[[{id, name, ...}]]`. This mirrors the
+ // Python agent_api.py:1611 contract `queue_dataflow(..., files[0], 0)`,
+ // where `files[0]` (the first inner list) is the set of files for this
+ // run. The dataflow debug extractor and the agent run path both unwrap
+ // the outer layer: `Files[0]` reaches the file dicts. The web contract
+ // is 2D, so a plain 1D `[]map[string]interface{}` would fail to decode
+ // and 400 the whole request.
+ Files [][]map[string]interface{} `json:"files"`
}
// extractLastUserContent returns the content of the last message in
@@ -1028,6 +1211,29 @@ func (h *AgentHandler) AgentChatCompletions(c *gin.Context) {
return
}
+ // DataFlow canvas: run a synchronous, side-effect-free debug (dry-run)
+ // and return the parsed chunks inline — reusing this existing
+ // chat/completions endpoint instead of a dedicated dataflow/debug route.
+ // This mirrors the Python agent_api.py:1569 DataFlow branch, which is
+ // reached only on the non-openai, non-session path (openai-compatible
+ // requests return their choices shape above and never enter debug). The
+ // The canvas is loaded only to read its category, and the debug path is
+ // skipped when the loader is absent or the canvas is not a DataFlow. No
+ // kb_id is involved: the debug context always runs KB-less (see
+ // runCanvasPipelineDebug).
+ if h.loader != nil {
+ if cv, lerr := h.loader.LoadCanvasByID(c.Request.Context(), user.ID, req.AgentID); lerr == nil && cv.CanvasCategory == "dataflow_canvas" {
+ fileName, fileData := h.extractChatDebugFile(c.Request.Context(), req.Files, user)
+ if len(fileData) == 0 {
+ common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataflow debug requires an uploaded file")
+ return
+ }
+ result, derr := h.runCanvasPipelineDebug(c.Request.Context(), user, req.AgentID, fileName, fileData)
+ respondWithDebugResult(c, result, derr)
+ return
+ }
+ }
+
// Real canvas run — derive userInput from `query` first, then fall
// back to the last user message (covers the front-end that posts
// running_hint_text without a top-level `query`).
@@ -1052,7 +1258,16 @@ func (h *AgentHandler) AgentChatCompletions(c *gin.Context) {
req.SessionID = utility.GenerateToken()
}
- events, err := h.chatRunner.RunAgent(c.Request.Context(), user.ID, req.AgentID, req.SessionID, "", userInput, req.Files)
+ // The web contract wraps `files` one level ([[{...}]]); unwrap the
+ // outer layer so RunAgent receives the inner 1D file list, matching the
+ // Python canvas file component's `kwargs.get("file")[0]` unwrap. When no
+ // files are present (the common agent-chat case) pass nil so RunAgent's
+ // `len(files) > 0` guard sees an empty run.
+ var chatFiles []map[string]interface{}
+ if len(req.Files) > 0 {
+ chatFiles = req.Files[0]
+ }
+ events, err := h.chatRunner.RunAgent(c.Request.Context(), user.ID, req.AgentID, req.SessionID, "", userInput, chatFiles)
if err != nil {
common.Warn("agent chat completions: RunAgent failed",
append([]zap.Field{
@@ -1314,12 +1529,35 @@ func (h *AgentHandler) TestDBConnection(c *gin.Context) {
common.SuccessWithData(c, true, "success")
}
+// parseAgentLogs decodes the debug-log payload stored in Redis by the pipeline
+// run. The Python pipeline (rag/flow/pipeline.py callback) writes a JSON
+// *array* ([{component_id, trace:[...]}]) that the front-end's
+// useFetchMessageTrace (web/src/hooks/use-agent-request.ts) requires as
+// response.data. A missing, empty, or corrupt payload collapses to an empty
+// object so the contract from get_agent_logs (api/apps/restful_apis/agent_api.py:1087)
+// — `code == 0` and `data` is a (possibly empty) dict/array — stays satisfied
+// instead of 500ing on bad bytes.
+func parseAgentLogs(payload string) interface{} {
+ if strings.TrimSpace(payload) == "" {
+ return map[string]interface{}{}
+ }
+ var data interface{}
+ if err := json.Unmarshal([]byte(payload), &data); err != nil {
+ return map[string]interface{}{}
+ }
+ return data
+}
+
// GetAgentLogs GET /api/v1/agents/:canvas_id/logs/:message_id
//
// Reads "{agent_id}-{message_id}-logs" from Redis (same key format
-// used by the Python agent API in api/apps/restful_apis/agent_api.py
-// line 920). Missing key returns an empty dict so the test contract
-// `data is dict` and `code == 0` are both satisfied.
+// used by the Python agent API in api/apps/restful_apis/agent_api.py) and
+// returns it as-is. Because runCanvasPipelineDebug flushes the ENTIRE log (including
+// the END marker) before returning the run response, this endpoint returns a
+// completed snapshot on the first poll — it is a replay/fetch of an already
+// finished debug log, not a streaming subscription. Missing key returns an
+// empty dict so the test contract `data is dict` and `code == 0` are both
+// satisfied when nothing has been written yet.
func (h *AgentHandler) GetAgentLogs(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
@@ -1335,10 +1573,10 @@ func (h *AgentHandler) GetAgentLogs(c *gin.Context) {
}
key := fmt.Sprintf("%s-%s-logs", canvasID, messageID)
- payload, rerr := redis.Get().Get(key)
- data := map[string]interface{}{}
- if rerr == nil && payload != "" {
- _ = json.Unmarshal([]byte(payload), &data)
+ payload, rerr := h.redisGet(key)
+ var data interface{} = map[string]interface{}{}
+ if rerr == nil {
+ data = parseAgentLogs(payload)
}
common.SuccessWithData(c, data, "success")
}
diff --git a/internal/handler/agent_logs_test.go b/internal/handler/agent_logs_test.go
new file mode 100644
index 0000000000..793b4fe9f0
--- /dev/null
+++ b/internal/handler/agent_logs_test.go
@@ -0,0 +1,675 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/alicebob/miniredis/v2"
+ "github.com/gin-gonic/gin"
+ goredis "github.com/redis/go-redis/v9"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ pipelinepkg "ragflow/internal/ingestion/pipeline"
+ "ragflow/internal/ingestion/task"
+ "ragflow/internal/service"
+)
+
+// TestParseAgentLogs pins the contract between GetAgentLogs and the
+// front-end DataFlow Log box. The Python pipeline (rag/flow/pipeline.py
+// callback) stores a JSON *array* ([{component_id, trace:[...]}]); the
+// front-end's useFetchMessageTrace (web/src/hooks/use-agent-request.ts)
+// requires response.data to be that array. The original handler unmarshalled
+// into a map[string]interface{} and silently dropped the array, leaving the
+// Log box stuck on an empty SkeletonCard. A missing/empty/corrupt payload must
+// still collapse to an empty object to mirror the Python get_agent_logs
+// missing-key contract (api/apps/restful_apis/agent_api.py:1087).
+func TestParseAgentLogs(t *testing.T) {
+ cases := []struct {
+ name string
+ payload string
+ wantKind byte // '[' for array, '{' for object
+ }{
+ {
+ name: "missing_key_returns_empty_object",
+ payload: "",
+ wantKind: '{',
+ },
+ {
+ name: "present_array_returns_array",
+ payload: `[` +
+ `{"component_id":"File","trace":[` +
+ `{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}` +
+ `]},` +
+ `{"component_id":"END","trace":[{"progress":1,"message":"done","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` +
+ `]`,
+ wantKind: '[',
+ },
+ {
+ name: "corrupt_payload_falls_back_to_empty_object",
+ payload: `{"component_id":`,
+ wantKind: '{',
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := parseAgentLogs(tc.payload)
+ b, err := json.Marshal(got)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if len(b) == 0 || b[0] != tc.wantKind {
+ t.Fatalf("want JSON kind %q, got %s", tc.wantKind, string(b))
+ }
+ if tc.wantKind == '[' {
+ var arr []map[string]interface{}
+ if err := json.Unmarshal(b, &arr); err != nil {
+ t.Fatalf("round-trip: %v", err)
+ }
+ if arr[0]["component_id"] != "File" {
+ t.Fatalf("component_id lost after round-trip: %v", arr[0])
+ }
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Fatalf("END marker lost after round-trip: %v", arr[len(arr)-1])
+ }
+ }
+ })
+ }
+}
+
+// TestGetAgentLogs_E2EViaMiniredis exercises the full HTTP handler path
+// (auth -> canvas-access -> Redis fetch -> shape) against an in-memory
+// miniredis so we can assert the wire response shape the front-end depends on:
+// - present key -> response.data is the JSON array ([...]) with the END marker
+// - missing key -> response.data is an empty object ("{}"), matching the
+// Python get_agent_logs missing-key contract.
+//
+// The Redis client is injected via WithRedisGetter (a miniredis-backed
+// go-redis client); the canvas-access gate is satisfied by a real in-memory
+// UserCanvas row, mirroring TestGetAgentWebhookLogsReturnsEmptyPoll.
+func TestGetAgentLogs_E2EViaMiniredis(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ db := setupHandlerAgentsTestDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})
+
+ mr, err := miniredis.Run()
+ if err != nil {
+ t.Fatalf("miniredis.Run: %v", err)
+ }
+ t.Cleanup(mr.Close)
+
+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ logKey := "c1-msg1-logs"
+ arrayPayload := `[` +
+ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` +
+ `{"component_id":"END","trace":[{"progress":1,"message":"done","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` +
+ `]`
+ if err := rdb.Set(context.Background(), logKey, arrayPayload, 0).Err(); err != nil {
+ t.Fatalf("seed redis: %v", err)
+ }
+
+ h := NewAgentHandler(service.NewAgentService(), nil).
+ WithRedisGetter(func(key string) (string, error) {
+ return rdb.Get(context.Background(), key).Result()
+ })
+
+ run := func(messageID string) map[string]interface{} {
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)
+ c.Set("user", &entity.User{ID: "u1"})
+ c.Set("user_id", "u1")
+ c.Params = gin.Params{
+ {Key: "canvas_id", Value: "c1"},
+ {Key: "message_id", Value: messageID},
+ }
+ h.GetAgentLogs(c)
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode (msg=%s): %v body=%s", messageID, err, w.Body.String())
+ }
+ return resp
+ }
+
+ // Present key -> data must be the array, with File first and END last.
+ present := run("msg1")
+ if code, _ := present["code"].(float64); int(code) != int(common.CodeSuccess) {
+ t.Fatalf("present code=%v want 0 body=%s", code, mustJSON(present))
+ }
+ dataRaw, _ := json.Marshal(present["data"])
+ trimmed := strings.TrimSpace(string(dataRaw))
+ if len(trimmed) == 0 || trimmed[0] != '[' {
+ t.Fatalf("present response.data must be a JSON array, got %s", string(dataRaw))
+ }
+ var arr []map[string]interface{}
+ if err := json.Unmarshal(dataRaw, &arr); err != nil {
+ t.Fatalf("present data not array: %v body=%s", err, string(dataRaw))
+ }
+ if arr[0]["component_id"] != "File" {
+ t.Errorf("first component_id=%v want File", arr[0]["component_id"])
+ }
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Errorf("last component_id=%v want END", arr[len(arr)-1]["component_id"])
+ }
+
+ // Missing key -> data must be an empty object (Python parity), code 0.
+ missing := run("missing")
+ if code, _ := missing["code"].(float64); int(code) != int(common.CodeSuccess) {
+ t.Fatalf("missing code=%v want 0 body=%s", code, mustJSON(missing))
+ }
+ dataRawMissing, _ := json.Marshal(missing["data"])
+ if strings.TrimSpace(string(dataRawMissing)) != "{}" {
+ t.Fatalf("missing response.data must be {} (object), got %s", string(dataRawMissing))
+ }
+}
+
+// clientConsidersComplete replicates the front-end completion predicate from
+// web/src/pages/agent/hooks/use-fetch-pipeline-log.ts: the run is "really
+// done" only when the LAST array element has component_id == "END" AND its
+// first trace entry carries a non-empty message. If any part of that signal
+// is missing, the Log box keeps polling forever.
+func clientConsidersComplete(arr []map[string]interface{}) bool {
+ if len(arr) == 0 {
+ return false
+ }
+ last := arr[len(arr)-1]
+ if last["component_id"] != "END" {
+ return false
+ }
+ trace, ok := last["trace"].([]interface{})
+ if !ok || len(trace) == 0 {
+ return false
+ }
+ first, ok := trace[0].(map[string]interface{})
+ if !ok {
+ return false
+ }
+ msg, _ := first["message"].(string)
+ return strings.TrimSpace(msg) != ""
+}
+
+// TestGetAgentLogs_EndSignalCompletion pins the exact signal the front-end
+// polls for: the response array must end with an END element whose first
+// trace message is non-empty. The handler must preserve that signal
+// byte-for-byte through JSON round-tripping.
+func TestGetAgentLogs_EndSignalCompletion(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ db := setupHandlerAgentsTestDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})
+
+ mr, err := miniredis.Run()
+ if err != nil {
+ t.Fatalf("miniredis.Run: %v", err)
+ }
+ t.Cleanup(mr.Close)
+
+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ // End element with a NON-empty message -> client must consider it done.
+ goodPayload := `[` +
+ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` +
+ `{"component_id":"END","trace":[{"progress":1,"message":"run finished","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` +
+ `]`
+ if err := rdb.Set(context.Background(), "c1-msg-good-logs", goodPayload, 0).Err(); err != nil {
+ t.Fatalf("seed redis: %v", err)
+ }
+
+ // End element with an EMPTY message -> client must NOT consider it done
+ // (would poll forever). Locks the contract that the debug writer must
+ // emit a non-empty END message.
+ badPayload := `[` +
+ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` +
+ `{"component_id":"END","trace":[{"progress":1,"message":"","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` +
+ `]`
+ if err := rdb.Set(context.Background(), "c1-msg-bad-logs", badPayload, 0).Err(); err != nil {
+ t.Fatalf("seed redis: %v", err)
+ }
+
+ h := NewAgentHandler(service.NewAgentService(), nil).
+ WithRedisGetter(func(key string) (string, error) {
+ return rdb.Get(context.Background(), key).Result()
+ })
+
+ call := func(messageID string) []map[string]interface{} {
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)
+ c.Set("user", &entity.User{ID: "u1"})
+ c.Set("user_id", "u1")
+ c.Params = gin.Params{
+ {Key: "canvas_id", Value: "c1"},
+ {Key: "message_id", Value: messageID},
+ }
+ h.GetAgentLogs(c)
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode (msg=%s): %v body=%s", messageID, err, w.Body.String())
+ }
+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {
+ t.Fatalf("code=%v want 0 body=%s", code, mustJSON(resp))
+ }
+ dataRaw, _ := json.Marshal(resp["data"])
+ var arr []map[string]interface{}
+ if err := json.Unmarshal(dataRaw, &arr); err != nil {
+ t.Fatalf("data not array (msg=%s): %v body=%s", messageID, err, string(dataRaw))
+ }
+ return arr
+ }
+
+ good := call("msg-good")
+ if !clientConsidersComplete(good) {
+ t.Fatalf("expected client to consider run complete for non-empty END message; arr=%s", mustJSON(good))
+ }
+
+ bad := call("msg-bad")
+ if clientConsidersComplete(bad) {
+ t.Fatalf("client must NOT consider run complete for empty END message (would poll forever); arr=%s", mustJSON(bad))
+ }
+}
+
+// capturedStore is an in-memory task.DebugLogStore used to assert that
+// runCanvasPipelineDebug actually wrote the debug log array under the expected key.
+type capturedStore struct {
+ mu sync.Mutex
+ data map[string]string
+}
+
+func (s *capturedStore) Set(key, value string, _ time.Duration) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.data == nil {
+ s.data = map[string]string{}
+ }
+ s.data[key] = value
+ return true
+}
+
+func (s *capturedStore) get(key string) (string, bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ v, ok := s.data[key]
+ return v, ok
+}
+
+// fakeDebugExecutor is a debugExecutor stand-in for runCanvasPipelineDebug. It
+// captures the sink attached via WithProgressSink and replays a couple of
+// component lifecycle events on Execute, mirroring what the real
+// PipelineExecutor emits through its progress callback.
+type fakeDebugExecutor struct {
+ capturedSink pipelinepkg.ProgressSink
+ result *task.PipelineResult
+ runErr error
+}
+
+func (f *fakeDebugExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *task.PipelineExecutor {
+ f.capturedSink = sink
+ return nil // return ignored by runCanvasPipelineDebug
+}
+
+func (f *fakeDebugExecutor) Execute(ctx context.Context) (*task.PipelineResult, error) {
+ if f.capturedSink != nil {
+ f.capturedSink.OnComponentTotal(ctx, "t1", 1)
+ f.capturedSink.OnComponentProgress(ctx, pipelinepkg.ProgressEvent{
+ TaskID: "t1", Component: "File", Phase: 0, Message: "File Started",
+ })
+ f.capturedSink.OnComponentProgress(ctx, pipelinepkg.ProgressEvent{
+ TaskID: "t1", Component: "File", Phase: 1, Message: "File Done",
+ })
+ }
+ return f.result, f.runErr
+}
+
+// TestRespondWithDebugResult_Shape pins the wire contract the front-end reads
+// after kicking off a debug run: response.data must carry both message_id (the
+// polling key) and chunks (the rendered output). Empty chunks must stay an
+// empty array (never null), and a run error must surface as a server-error
+// code rather than success.
+func TestRespondWithDebugResult_Shape(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ // Success with chunks: data.message_id + data.chunks both present.
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ respondWithDebugResult(c, &task.PipelineResult{
+ MessageID: "m-abc",
+ Chunks: []map[string]any{{"content": "hello"}},
+ }, nil)
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode: %v body=%s", err, w.Body.String())
+ }
+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {
+ t.Fatalf("code=%v want 0 body=%s", code, mustJSON(resp))
+ }
+ data, ok := resp["data"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("data not object: %v", resp)
+ }
+ if data["message_id"] != "m-abc" {
+ t.Fatalf("data.message_id=%v want m-abc", data["message_id"])
+ }
+ chunks, ok := data["chunks"].([]interface{})
+ if !ok {
+ t.Fatalf("data.chunks not array: %v", data["chunks"])
+ }
+ if len(chunks) != 1 {
+ t.Fatalf("data.chunks len=%d want 1", len(chunks))
+ }
+
+ // Empty chunks: still an object with message_id and an empty array.
+ w2 := httptest.NewRecorder()
+ c2, _ := gin.CreateTestContext(w2)
+ respondWithDebugResult(c2, &task.PipelineResult{MessageID: "m-empty"}, nil)
+ var resp2 map[string]interface{}
+ if err := json.Unmarshal(w2.Body.Bytes(), &resp2); err != nil {
+ t.Fatalf("decode empty: %v body=%s", err, w2.Body.String())
+ }
+ data2, ok := resp2["data"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("data not object: %v", resp2)
+ }
+ if data2["message_id"] != "m-empty" {
+ t.Fatalf("data.message_id=%v want m-empty", data2["message_id"])
+ }
+ chunks2, ok := data2["chunks"].([]interface{})
+ if !ok {
+ t.Fatalf("data.chunks not array: %v", data2["chunks"])
+ }
+ if len(chunks2) != 0 {
+ t.Fatalf("data.chunks len=%d want 0", len(chunks2))
+ }
+
+ // Error: must NOT be a success code.
+ w3 := httptest.NewRecorder()
+ c3, _ := gin.CreateTestContext(w3)
+ respondWithDebugResult(c3, nil, errors.New("boom"))
+ var resp3 map[string]interface{}
+ if err := json.Unmarshal(w3.Body.Bytes(), &resp3); err != nil {
+ t.Fatalf("decode err: %v body=%s", err, w3.Body.String())
+ }
+ if code, _ := resp3["code"].(float64); int(code) == int(common.CodeSuccess) {
+ t.Fatalf("error case must not be success: %v", resp3)
+ }
+}
+
+// TestRespondWithDebugResult_ErrorCarriesMessageID pins that a run failure
+// still surfaces the polling key: the response must keep a non-success code
+// (the existing contract forbids masking errors as success) BUT the error
+// envelope's data must carry message_id so the front-end can poll the failure
+// timeline that DebugLogSink.Flush already wrote to Redis.
+func TestRespondWithDebugResult_ErrorCarriesMessageID(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ respondWithDebugResult(c, &task.PipelineResult{MessageID: "m-err"}, errors.New("boom"))
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode: %v body=%s", err, w.Body.String())
+ }
+ if code, _ := resp["code"].(float64); int(code) == int(common.CodeSuccess) {
+ t.Fatalf("error case must not be success: %v", resp)
+ }
+ data, ok := resp["data"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("error envelope data must be an object carrying message_id: %v", resp)
+ }
+ if data["message_id"] != "m-err" {
+ t.Fatalf("error envelope data.message_id=%v want m-err", data["message_id"])
+ }
+}
+
+// TestRunCanvasPipelineDebug_ErrorStillExposesMessageID asserts the full write-side
+// wiring when the executor fails: runCanvasPipelineDebug must (a) still return a
+// non-nil result carrying MessageID even though exec.Execute returned nil, and
+// (b) have already flushed the failure log under "{canvasID}-{messageID}-logs"
+// so the front-end can poll that exact key. This is the scenario that broke
+// before — the failure log was written but unreachable because message_id was
+// dropped on the error path.
+func TestRunCanvasPipelineDebug_ErrorStillExposesMessageID(t *testing.T) {
+ ctx := context.Background()
+ store := &capturedStore{}
+
+ h := NewAgentHandler(service.NewAgentService(), nil).
+ WithRedisStore(store).
+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {
+ return &fakeDebugExecutor{
+ // Simulate an executor that fails and returns no result.
+ result: nil,
+ runErr: errors.New("executor blew up"),
+ }, nil
+ })
+
+ result, err := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))
+ if err == nil {
+ t.Fatalf("expected run error")
+ }
+ if result == nil {
+ t.Fatalf("result is nil; front-end would have no polling key on failure")
+ }
+ if result.MessageID == "" {
+ t.Fatalf("result.MessageID empty on failure; front-end cannot poll the failure log")
+ }
+
+ // The failure log must be written under the composed key.
+ key := "c1-" + result.MessageID + "-logs"
+ raw, ok := store.get(key)
+ if !ok {
+ t.Fatalf("failure log not written under key %q; store keys=%v", key, keysOf(store))
+ }
+ var arr []map[string]interface{}
+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {
+ t.Fatalf("stored failure log not array: %v raw=%s", err, raw)
+ }
+ if !clientConsidersComplete(arr) {
+ t.Fatalf("stored failure log not completion-complete (END last + non-empty msg); arr=%s", raw)
+ }
+}
+
+// TestRunCanvasPipelineDebug_WiresMessageIDAndLog asserts the full write-side wiring
+// of runCanvasPipelineDebug without a live Redis or real canvas: a fake executor
+// emits component progress, the injected DebugLogSink flushes the
+// [{component_id, trace}] array (with the END marker last) to the captured
+// store under the key "{canvasID}-{messageID}-logs", and the returned result
+// carries message_id so the front-end can poll that exact key. The array must
+// satisfy the completion predicate (END last, non-empty END message) so the
+// Log box stops polling.
+func TestRunCanvasPipelineDebug_WiresMessageIDAndLog(t *testing.T) {
+ ctx := context.Background()
+ store := &capturedStore{}
+
+ h := NewAgentHandler(service.NewAgentService(), nil).
+ WithRedisStore(store).
+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {
+ return &fakeDebugExecutor{
+ result: &task.PipelineResult{
+ Chunks: []map[string]any{{"content": "chunk-1"}},
+ },
+ }, nil
+ })
+
+ result, err := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))
+ if err != nil {
+ t.Fatalf("runCanvasPipelineDebug: %v", err)
+ }
+ if result == nil {
+ t.Fatalf("result is nil")
+ }
+ if result.MessageID == "" {
+ t.Fatalf("result.MessageID empty; front-end would have no polling key")
+ }
+
+ // The log array must be written under the composed key.
+ key := "c1-" + result.MessageID + "-logs"
+ raw, ok := store.get(key)
+ if !ok {
+ t.Fatalf("log not written under key %q; store keys=%v", key, keysOf(store))
+ }
+
+ var arr []map[string]interface{}
+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {
+ t.Fatalf("stored log not array: %v raw=%s", err, raw)
+ }
+ if !clientConsidersComplete(arr) {
+ t.Fatalf("stored log not completion-complete (END last + non-empty msg); arr=%s", raw)
+ }
+}
+
+func keysOf(s *capturedStore) []string {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ keys := make([]string, 0, len(s.data))
+ for k := range s.data {
+ keys = append(keys, k)
+ }
+ return keys
+}
+
+// miniredisDebugStore adapts a go-redis client to task.DebugLogStore so the
+// write path can be exercised against a real (in-memory) Redis in tests. It is
+// the production-faithful counterpart of capturedStore: capturedStore proves
+// the key shape, miniredisDebugStore proves the bytes survive a real Redis
+// round-trip that the read path consumes.
+type miniredisDebugStore struct {
+ rdb *goredis.Client
+}
+
+func (s miniredisDebugStore) Set(key, value string, ttl time.Duration) bool {
+ if err := s.rdb.Set(context.Background(), key, value, ttl).Err(); err != nil {
+ return false
+ }
+ return true
+}
+
+// TestRunCanvasPipelineDebug_WriteThenReadViaMiniredis locks the write/read contract
+// end-to-end: runCanvasPipelineDebug writes the debug log array under
+// "{canvasID}-{messageID}-logs" into a real (miniredis) Redis via the injected
+// DebugLogStore, and the SAME handler's GetAgentLogs reads it back through the
+// injected redisGetter — proving the writer's key shape exactly matches the
+// reader's and that the stored bytes round-trip into the [{component_id,
+// trace}] array the front-end polls for (File first, END last, non-empty END
+// message). This is the only test that would catch a key-format drift between
+// the two sides, since the write test uses a capturedStore and the read test
+// seeds Redis directly rather than going through the writer.
+func TestRunCanvasPipelineDebug_WriteThenReadViaMiniredis(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ ctx := context.Background()
+
+ db := setupHandlerAgentsTestDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})
+
+ mr, err := miniredis.Run()
+ if err != nil {
+ t.Fatalf("miniredis.Run: %v", err)
+ }
+ t.Cleanup(mr.Close)
+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ // Both seams point at the same miniredis: the writer stores via
+ // WithRedisStore and the reader fetches via WithRedisGetter, mirroring
+ // production where both hit one Redis.
+ h := NewAgentHandler(service.NewAgentService(), nil).
+ WithRedisStore(miniredisDebugStore{rdb: rdb}).
+ WithRedisGetter(func(key string) (string, error) {
+ return rdb.Get(ctx, key).Result()
+ }).
+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {
+ return &fakeDebugExecutor{
+ result: &task.PipelineResult{Chunks: []map[string]any{{"content": "chunk-1"}}},
+ }, nil
+ })
+
+ // Write side.
+ writeResult, werr := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))
+ if werr != nil {
+ t.Fatalf("runCanvasPipelineDebug: %v", werr)
+ }
+ if writeResult == nil || writeResult.MessageID == "" {
+ t.Fatalf("runCanvasPipelineDebug returned no message_id")
+ }
+ messageID := writeResult.MessageID
+
+ // The key must exist in the same miniredis the reader will query.
+ if !mr.Exists("c1-" + messageID + "-logs") {
+ t.Fatalf("log key c1-%s-logs not present in redis after write", messageID)
+ }
+
+ // Read side: same handler, same miniredis, driven through the real
+ // GetAgentLogs HTTP path.
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)
+ c.Set("user", &entity.User{ID: "u1"})
+ c.Set("user_id", "u1")
+ c.Params = gin.Params{
+ {Key: "canvas_id", Value: "c1"},
+ {Key: "message_id", Value: messageID},
+ }
+ h.GetAgentLogs(c)
+
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode read response: %v body=%s", err, w.Body.String())
+ }
+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {
+ t.Fatalf("read code=%v want 0 body=%s", code, mustJSON(resp))
+ }
+ dataRaw, _ := json.Marshal(resp["data"])
+ trimmed := strings.TrimSpace(string(dataRaw))
+ if len(trimmed) == 0 || trimmed[0] != '[' {
+ t.Fatalf("read response.data must be a JSON array, got %s", string(dataRaw))
+ }
+ var arr []map[string]interface{}
+ if err := json.Unmarshal(dataRaw, &arr); err != nil {
+ t.Fatalf("read data not array: %v body=%s", err, string(dataRaw))
+ }
+ if arr[0]["component_id"] != "File" {
+ t.Errorf("first component_id=%v want File", arr[0]["component_id"])
+ }
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Errorf("last component_id=%v want END", arr[len(arr)-1]["component_id"])
+ }
+ if !clientConsidersComplete(arr) {
+ t.Errorf("stored log not completion-complete (END last + non-empty msg); arr=%s", string(dataRaw))
+ }
+}
diff --git a/internal/handler/agent_pipeline_debug_test.go b/internal/handler/agent_pipeline_debug_test.go
new file mode 100644
index 0000000000..21d961b0a2
--- /dev/null
+++ b/internal/handler/agent_pipeline_debug_test.go
@@ -0,0 +1,92 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package handler
+
+import (
+ "encoding/json"
+ "net/http"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/task"
+)
+
+func dataflowCanvas(id, userID string) *entity.UserCanvas {
+ cv := makeWebhookCanvas(id, userID, "Webhook", nil)
+ cv.CanvasCategory = "dataflow_canvas"
+ return cv
+}
+
+func decodeSuccess(t *testing.T, body []byte) (int, string, []map[string]any, string) {
+ t.Helper()
+ var resp struct {
+ Code int `json:"code"`
+ Data struct {
+ MessageID string `json:"message_id"`
+ Chunks []map[string]any `json:"chunks"`
+ } `json:"data"`
+ Message string `json:"message"`
+ }
+ if err := json.Unmarshal(body, &resp); err != nil {
+ t.Fatalf("decode response: %v (body=%s)", err, body)
+ }
+ return resp.Code, resp.Data.MessageID, resp.Data.Chunks, resp.Message
+}
+
+// TestAgentChatCompletions_DataFlowReturnsChunks pins that the existing
+// chat/completions endpoint serves the pipeline debug (dry-run) for DataFlow
+// canvases: it returns the parsed chunks inline, synchronously, without a
+// dedicated debug route. The webhook entry point is intentionally NOT a
+// debug surface — see TestWebhook_DataFlowRejected in agent_webhook_test.go.
+func TestAgentChatCompletions_DataFlowReturnsChunks(t *testing.T) {
+ h := &AgentHandler{
+ loader: &fakeCanvasLoader{canvas: dataflowCanvas("c1", "u-1")},
+ fileService: &fakeAgentFileService{blob: []byte("file-bytes")},
+ }
+ h.WithNewExecutor(func(_ *task.TaskContext, _ string, _ int) (debugExecutor, error) {
+ return &fakeDebugExecutor{result: &task.PipelineResult{Chunks: []map[string]any{{"text": "dbg-chunk"}}}}, nil
+ })
+ // runCanvasPipelineDebug flushes the debug log via redisStore; inject a no-op
+ // store so the (skipped in this test) log write does not nil-panic.
+ h.WithRedisStore(&capturedStore{})
+
+ // The web contract sends the file list wrapped one level:
+ // `files: [[{id, name}]]` (see use-run-dataflow.ts:34). The pipeline
+ // debug path unwraps both layers and downloads the referenced bytes.
+ c, w := webhookCtx("POST", "/api/v1/agents/c1/chat/completions",
+ `{"agent_id":"c1","files":[[{"id":"f1","name":"doc.txt"}]]}`, "application/json")
+
+ h.AgentChatCompletions(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200 (body=%s)", w.Code, w.Body.String())
+ }
+ code, msgID, data, msg := decodeSuccess(t, w.Body.Bytes())
+ if code != int(common.CodeSuccess) {
+ t.Errorf("code = %d, want %d", code, common.CodeSuccess)
+ }
+ if msg != "success" {
+ t.Errorf("message = %q, want %q", msg, "success")
+ }
+ if msgID == "" {
+ t.Errorf("message_id empty; front-end needs it to poll the debug log")
+ }
+ if len(data) != 1 || data[0]["text"] != "dbg-chunk" {
+ t.Errorf("data.chunks = %#v, want one chunk with text %q", data, "dbg-chunk")
+ }
+}
diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.go
index bfd2d4301c..8e7501fdb1 100644
--- a/internal/handler/agent_test.go
+++ b/internal/handler/agent_test.go
@@ -1228,9 +1228,11 @@ func (s *stubDocService) Accessible(_, _ string) bool {
}
// TestAgentChatCompletions_FilesDeserialized verifies that when the
-// JSON request body contains a `files` field, the
-// agentChatCompletionsRequest struct deserializes it correctly.
-// Mirrors Python's req.get("files", []) at agent_api.py:1313.
+// JSON request body contains the web-contract 2D `files` field
+// (`[[{...}]]`), the agentChatCompletionsRequest struct deserializes it
+// and the inner file list reaches RunAgent. Mirrors Python's
+// agent_api.py:1611 `queue_dataflow(..., files[0], 0)`, where the first
+// inner list is the set of files for the run.
func TestAgentChatCompletions_FilesDeserialized(t *testing.T) {
gin.SetMode(gin.TestMode)
w := httptest.NewRecorder()
@@ -1238,9 +1240,9 @@ func TestAgentChatCompletions_FilesDeserialized(t *testing.T) {
body := `{
"agent_id": "a1",
"query": "hi",
- "files": [
+ "files": [[
{"id": "file-1", "name": "resume.txt", "mime_type": "text/plain", "created_by": "u1"}
- ]
+ ]]
}`
c.Request = httptest.NewRequest("POST", "/api/v1/agents/chat/completions",
strings.NewReader(body))
diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.go
index 2173533300..ef2ded7725 100644
--- a/internal/handler/agent_webhook.go
+++ b/internal/handler/agent_webhook.go
@@ -133,8 +133,11 @@ func (h *AgentHandler) Webhook(c *gin.Context) {
return
}
- // 2. Reject DataFlow.
- if cv.CanvasCategory == "DataFlow" {
+ // 2. Reject DataFlow. DataFlow canvases are ingestion pipelines, not
+ // interactive agents, and must not be triggered by an external webhook.
+ // Mirrors Python agent_api.py:1786, which returns
+ // "Dataflow can not be triggered by webhook." for the same case.
+ if cv.CanvasCategory == "dataflow_canvas" {
common.ResponseWithCodeData(c, common.CodeDataError, nil, "Dataflow can not be triggered by webhook.")
return
}
diff --git a/internal/handler/agent_webhook_test.go b/internal/handler/agent_webhook_test.go
index ff4a768d4d..3f08cad01e 100644
--- a/internal/handler/agent_webhook_test.go
+++ b/internal/handler/agent_webhook_test.go
@@ -153,12 +153,14 @@ func TestWebhook_RejectsUnknownCanvas(t *testing.T) {
}
}
-// TestWebhook_RejectsDataFlowCanvas covers the second guard: the canvas
-// exists but its category is DataFlow (which python
-// agent_api.py:1575 explicitly rejects).
-func TestWebhook_RejectsDataFlowCanvas(t *testing.T) {
+// TestWebhook_DataFlowRejected pins the second guard: a canvas whose category
+// is DataFlow must NOT be triggered by an external webhook. The handler returns
+// CodeDataError with "Dataflow can not be triggered by webhook." — mirroring
+// Python agent_api.py:1786. DataFlow canvases are ingestion pipelines, not
+// interactive agents, and have no chat/debug surface over the webhook entry.
+func TestWebhook_DataFlowRejected(t *testing.T) {
cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)
- cv.CanvasCategory = "DataFlow"
+ cv.CanvasCategory = "dataflow_canvas"
h := &AgentHandler{loader: &fakeCanvasLoader{canvas: cv}}
c, w := webhookCtx("POST", "/api/v1/agents/c1/webhook", `{}`, "application/json")
@@ -166,7 +168,7 @@ func TestWebhook_RejectsDataFlowCanvas(t *testing.T) {
code, msg := errBody(t, w.Body.Bytes())
if code != int(common.CodeDataError) {
- t.Errorf("code = %d, want %d", code, common.CodeDataError)
+ t.Errorf("code = %d, want %d (CodeDataError)", code, common.CodeDataError)
}
if msg != "Dataflow can not be triggered by webhook." {
t.Errorf("message = %q, want %q", msg, "Dataflow can not be triggered by webhook.")
diff --git a/internal/ingestion/component/chunker/image_upload_test.go b/internal/ingestion/component/chunker/image_upload_test.go
index 2a11e05e65..261b105bb1 100644
--- a/internal/ingestion/component/chunker/image_upload_test.go
+++ b/internal/ingestion/component/chunker/image_upload_test.go
@@ -276,3 +276,48 @@ func waitForInflight(t *testing.T, counter *int64, target int64) {
}
}
}
+
+// TestImageUploadDecorator_DebugSkipsUpload verifies that a debug/dry-run run
+// (empty kb_id) does NOT upload the chunk image to storage, while still
+// dropping the raw image bytes. This keeps the debug path free of MinIO side
+// effects and preserves the current memory behaviour (raw bytes discarded, not
+// held until a persist stage). An empty kb_id only occurs in canvas debug
+// (dry-run) mode; production ingestion always supplies a KB.
+func TestImageUploadDecorator_DebugSkipsUpload(t *testing.T) {
+ prev := ChunkImageUploader
+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {
+ t.Fatalf("ChunkImageUploader must not be called in a debug run")
+ return "", nil
+ }
+ t.Cleanup(func() { ChunkImageUploader = prev })
+
+ comp, err := NewOneChunker(nil)
+ if err != nil {
+ t.Fatalf("NewOneChunker: %v", err)
+ }
+ decorated := &imageUploadDecorator{inner: comp}
+
+ inputs := map[string]any{
+ "name": "doc.pdf",
+ "kb_id": "", // debug mode: no KB -> no upload
+ "doc_id": testDocID,
+ "chunks": []map[string]any{
+ {"content_with_weight": "a cropped figure", "image": "data:image/png;base64," + pngBase64},
+ },
+ }
+ out, err := decorated.Invoke(context.Background(), nil, inputs)
+ if err != nil {
+ t.Fatalf("decorated Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) == 0 {
+ t.Fatalf("expected chunks in output, got %#v", out)
+ }
+ ck := chunks[0]
+ if _, stillHas := ck["image"]; stillHas {
+ t.Errorf("raw image should be dropped in debug run, but ck[\"image\"] is still present")
+ }
+ if id, _ := ck["img_id"].(string); id != "" {
+ t.Errorf("img_id should not be set in debug run, got %q", id)
+ }
+}
diff --git a/internal/ingestion/component/chunker/pdfcrop_nocgo.go b/internal/ingestion/component/chunker/pdfcrop_nocgo.go
index 179e199c33..1034dc13e1 100644
--- a/internal/ingestion/component/chunker/pdfcrop_nocgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_nocgo.go
@@ -12,6 +12,8 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/ingestion/component/schema"
+
+ "gorm.io/gorm"
)
func newPDFEngineFromUpstream(_ context.Context, _ *gorm.DB, _ schema.ChunkerFromUpstream) (deepdoctype.PDFEngine, error) {
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index 6b716fa76c..3c28a24223 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -82,8 +82,19 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
ck["id"] = common.ChunkID(docID, text)
}
- if err := uploadChunkImages(ctx, chunks, ChunkImageUploader, kbID); err != nil {
- return nil, err
+ // kb_id is empty only in canvas debug (dry-run) mode; production ingestion
+ // always supplies a KB, so kb_id == "" never occurs in normal operation.
+ // Image bytes are uploaded to MinIO only when a KB is present (i.e. a
+ // persist run); in debug we drop the raw bytes so they are not held in
+ // memory until a persist stage that will never run.
+ if kbID != "" {
+ if err := uploadChunkImages(ctx, chunks, ChunkImageUploader, kbID); err != nil {
+ return nil, err
+ }
+ } else {
+ for _, ck := range chunks {
+ delete(ck, "image")
+ }
}
return out, nil
}
diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.go
index bd037079cb..f6a803a3fc 100644
--- a/internal/ingestion/component/file.go
+++ b/internal/ingestion/component/file.go
@@ -151,6 +151,14 @@ func (c *FileComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stri
if in.fileDesc != nil {
out["file"] = in.fileDesc
}
+ // Pass through in-memory bytes when present (debug / dataflow dry-run
+ // where no doc row exists in storage). The downstream Parser reads
+ // `binary` first and skips the doc_id → storage lookup, so a debug run
+ // can parse the uploaded file without a persisted document. Persist
+ // runs set no `binary` here, so they keep resolving from storage.
+ if len(in.binary) > 0 {
+ out["binary"] = in.binary
+ }
// Publish the resolved run-level metadata into the workflow-wide
// CanvasState.Globals bag so downstream components (Tokenizer,
// Chunker, ...) read it from ctx instead of relying on this output
@@ -169,6 +177,7 @@ type fileInputs struct {
bucket string
path string
fileDesc map[string]any
+ binary []byte
}
// parseFileInputs parses and validates the upstream input map.
@@ -180,6 +189,19 @@ func parseFileInputs(ctx context.Context, inputs map[string]any) (fileInputs, er
}
out := fileInputs{}
+ // Debug / dataflow dry-run fast path: in-memory bytes were supplied
+ // via the graph input `binary` (no persisted document exists). Use
+ // them directly and skip the doc_id → storage resolution so a debug
+ // run parses the uploaded file without a DB/storage round-trip. The
+ // executor only sets `binary` for the non-persist (debug) marker doc.
+ if b, ok := inputs["binary"].([]byte); ok && len(b) > 0 {
+ out.binary = b
+ if n, ok := getString(inputs, "name"); ok && n != "" {
+ out.name = n
+ }
+ return out, nil
+ }
+
if v, ok := getString(inputs, "doc_id"); ok && v != "" {
out.docID = v
out.name = v
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index dacfc6713d..9fef42ab57 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -535,6 +535,12 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
}
+ // 3b. (pages-based page selection is handled upstream via the
+ // component setup: ParserConfig[cpnID][filetype]["pages"] is a
+ // list of page ranges delivered through override_params and
+ // consumed by the deepdoc/pdf parser. No inputs-level handling
+ // here.)
+
// 4. Build the page slice sequentially. Per-page parallelism now
// lives in the parser backends (e.g. internal/deepdoc/parser/pdf
// fans out one worker per page and assembles in page order), so
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 8eea5e131c..05b9f5b042 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -320,6 +320,16 @@ func pythonFamilyName(raw string) string {
return ""
}
+// ParserFileFamily normalises a free-form file-type/extension hint to the
+// python-side family identifier used as the key into a Parser component's
+// setups (e.g. "pdf", "docx", "slides", "text&code"). It is the exported
+// entry point for callers outside this package (e.g. the ingestion task
+// executor that injects the debug page cap into override_params) that need
+// to build the canonical ParserConfig[cpnID][family]["pages"] shape.
+func ParserFileFamily(ext string) string {
+ return pythonFamilyName(ext)
+}
+
// jsonItemsToPages reshapes a parsed JSON payload into the
// schema.Page layout the chunker side consumes. Each JSON item
// becomes one page carrying `text`, `doc_type_kwd`, and any
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 8e8318b796..007d4a3d52 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -360,7 +360,12 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
"chunks": schema.ChunkDocsToMaps(chunks),
}
- if contains(c.param.SearchMethod, "embedding") {
+ // Embedding requires a KB: the embedder (and its embd_id) is configured
+ // on the knowledgebase, so without kb_id there is nothing to resolve
+ // against. A canvas-debug (dry-run) run has kb_id == "" by construction,
+ // so embedding is skipped there — debug only exercises parse+chunk and
+ // must stay side-effect free.
+ if shouldHaveEmbedding(c.param.SearchMethod, kbID) {
chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks)
if err != nil {
return nil, err
@@ -368,7 +373,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
out["embedding_token_consumption"] = tokenCount
out["chunks"] = schema.ChunkDocsToMaps(chunks)
}
- if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields); err != nil {
+ if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields, kbID); err != nil {
return nil, err
}
@@ -776,9 +781,15 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
return b.String()
}
-func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string) error {
+// shouldHaveEmbedding reports whether the tokenizer must attach embedding
+// vectors: the search method requests embedding AND a KB is present.
+func shouldHaveEmbedding(searchMethods []string, kbID string) bool {
+ return contains(searchMethods, "embedding") && kbID != ""
+}
+
+func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string, kbID string) error {
needFullText := contains(searchMethods, "full_text")
- needEmbedding := contains(searchMethods, "embedding")
+ needEmbedding := shouldHaveEmbedding(searchMethods, kbID)
if !needFullText && !needEmbedding {
return nil
}
diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.go
new file mode 100644
index 0000000000..1d9a8c2457
--- /dev/null
+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go
@@ -0,0 +1,109 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Regression guard for the canvas-debug (dry-run) run contract: a debug run
+// carries no KB (kb_id == ""), so the tokenizer must skip embedding rather than
+// resolving an embedder. It lives in the canvas-debug branch (not the
+// ingestion-fixes branch) so the ingestion-fixes branch stays compilable on its
+// own.
+
+package component
+
+import (
+ "context"
+ "testing"
+)
+
+// TestTokenizerComponent_DebugSkipsEmbedding is a regression gate for the
+// canvas-debug (dry-run) contract: a debug run has no KB (kb_id == ""), so
+// embedding MUST be skipped — there is no embedder (and no embd_id) to resolve
+// against, and a debug run must stay side-effect free. The tokenizer must
+// return its chunks without error, without invoking the embedder, and without
+// attaching embedding vectors. Conversely, when a kb_id is present the embedding
+// path must run.
+//
+// It uses embedding-only search_method so it runs without the C++ RAGAnalyzer
+// pool (plain `go test`, no -tags integration), keeping the regression gate
+// executable in every environment.
+func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {
+ stub := newStubEmbedder(4)
+ cIntf, err := NewTokenizerComponentWithResolver(
+ map[string]any{"search_method": []any{"embedding"}, "fields": []any{"text"}},
+ func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ )
+ if err != nil {
+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
+ }
+ c := cIntf.(*TokenizerComponent)
+
+ run := func(kbID string) (int32, map[string]any) {
+ inputs := map[string]any{
+ "name": "doc.pdf",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": "alpha bravo"},
+ {"text": "charlie delta"},
+ },
+ }
+ if kbID != "" {
+ inputs["kb_id"] = kbID
+ }
+ out, err := c.Invoke(context.Background(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke(kb_id=%q): %v", kbID, err)
+ }
+ return stub.calls.Load(), out
+ }
+
+ // Debug (kb_id == ""): embedding must be skipped entirely.
+ stub.calls.Store(0)
+ debugCalls, debugOut := run("")
+ if debugCalls != 0 {
+ t.Fatalf("embedder called under debug (kb_id=\"\"): %d calls; embedding must be skipped", debugCalls)
+ }
+ if got := debugOut["embedding_token_consumption"]; got != nil {
+ t.Errorf("embedding_token_consumption = %v under debug; must be omitted when embedding is skipped", got)
+ }
+ got, ok := debugOut["chunks"].([]map[string]any)
+ if !ok || len(got) != 2 {
+ t.Fatalf("chunks malformed under debug: %v", debugOut["chunks"])
+ }
+ for i, ck := range got {
+ if _, has := ck["q_4_vec"]; has {
+ t.Errorf("chunk[%d] carries q_4_vec under debug; embedding must be skipped", i)
+ }
+ }
+
+ // With a KB: embedding must run.
+ stub.calls.Store(0)
+ kbCalls, kbOut := run("kb-1")
+ if kbCalls != 2 {
+ t.Fatalf("embedder calls = %d with kb_id set; want 2", kbCalls)
+ }
+ if got := kbOut["embedding_token_consumption"]; got == nil {
+ t.Error("embedding_token_consumption missing with kb_id set; embedding accounting must run")
+ }
+ kbChunks, ok := kbOut["chunks"].([]map[string]any)
+ if !ok || len(kbChunks) != 2 {
+ t.Fatalf("chunks malformed with kb_id: %v", kbOut["chunks"])
+ }
+ for i, ck := range kbChunks {
+ vec, ok := ck["q_4_vec"].([]float64)
+ if !ok || len(vec) != 4 {
+ t.Fatalf("chunk[%d] missing q_4_vec with kb_id set: %v", i, ck["q_4_vec"])
+ }
+ }
+}
diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go
index 5dd986e6d2..5ab725a6a1 100644
--- a/internal/ingestion/component/tokenizer_test.go
+++ b/internal/ingestion/component/tokenizer_test.go
@@ -70,6 +70,7 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {
out, err := c.Invoke(context.Background(), nil, map[string]any{
"tenant_id": "t1",
"model_id": "embd-1",
+ "kb_id": "kb-1",
"name": "doc.pdf",
"output_format": "chunks",
"chunks": chunks,
@@ -222,6 +223,7 @@ func TestTokenizerComponent_Invoke_BatchedEmbedding(t *testing.T) {
}
if _, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.txt",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": chunks,
}); err != nil {
@@ -298,6 +300,7 @@ func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {
c, _ := withStubEmbedder(t, 4)
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha bravo"}},
})
@@ -327,6 +330,7 @@ func TestTokenizerComponent_Invoke_EmbedNoResolver(t *testing.T) {
requireTokenizerPool(t)
c, _ := NewTokenizerComponent(nil)
_, err := c.Invoke(context.Background(), nil, map[string]any{
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
@@ -346,6 +350,7 @@ func TestTokenizerComponent_Invoke_EmbedderError(t *testing.T) {
stub.err = errors.New("simulated upstream error")
_, err := c.Invoke(context.Background(), nil, map[string]any{
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
@@ -372,6 +377,7 @@ func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {
c := cIntf.(*TokenizerComponent)
_ = stub
_, err = c.Invoke(context.Background(), nil, map[string]any{
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "a"}, {"text": "b"}, {"text": "c"}},
})
@@ -432,6 +438,7 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
out, err := c.Invoke(context.Background(), nil, map[string]any{
"tenant_id": "tenant-smoke",
"model_id": "embd-smoke",
+ "kb_id": "kb-1",
"name": "smoke.pdf",
"output_format": "chunks",
"chunks": []map[string]any{{"text": chunkText}},
@@ -479,6 +486,7 @@ func TestTokenizerComponent_Embedding_MergesTitleAndContentVectors(t *testing.T)
}
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},
})
@@ -514,6 +522,7 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {
c := cIntf.(*TokenizerComponent)
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
@@ -544,6 +553,7 @@ func TestTokenizerComponent_Embedding_EmptyNameWarnsAndUsesContentVector(t *test
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": " ",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
})
@@ -577,6 +587,7 @@ func TestTokenizerComponent_Embedding_UsesRawNameNotTrimmed(t *testing.T) {
if _, err := c.Invoke(context.Background(), nil, map[string]any{
"name": " report.pdf ",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}},
}); err != nil {
@@ -599,6 +610,7 @@ func TestTokenizerComponent_Embedding_TruncatesByMaxTokensMinus10(t *testing.T)
longText := strings.Repeat("hello world ", 20)
if _, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": longText}},
}); err != nil {
@@ -620,6 +632,7 @@ func TestTokenizerComponent_Embedding_SkipsEmptyCleanedTextsButReturnsZeroWhenAl
c, stub := withStubEmbedder(t, 2)
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{
{"text": ""},
@@ -651,6 +664,7 @@ func TestTokenizerComponent_Embedding_SetsTokenConsumptionIncludingTitleCall(t *
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},
})
@@ -669,6 +683,7 @@ func TestTokenizerComponent_Embedding_BatchesByConfiguredBatchSize(t *testing.T)
if _, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{
{"text": "one"},
@@ -722,11 +737,11 @@ func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testin
compA := compAIntf.(*TokenizerComponent)
compB := compBIntf.(*TokenizerComponent)
- outA, err := compA.Invoke(context.Background(), nil, map[string]any{"name": "docA", "output_format": "chunks", "chunks": []map[string]any{{"text": "alpha"}}})
+ outA, err := compA.Invoke(context.Background(), nil, map[string]any{"kb_id": "kb-1", "name": "docA", "output_format": "chunks", "chunks": []map[string]any{{"text": "alpha"}}})
if err != nil {
t.Fatalf("Invoke A: %v", err)
}
- outB, err := compB.Invoke(context.Background(), nil, map[string]any{"name": "docB", "output_format": "chunks", "chunks": []map[string]any{{"text": "beta"}}})
+ outB, err := compB.Invoke(context.Background(), nil, map[string]any{"kb_id": "kb-1", "name": "docB", "output_format": "chunks", "chunks": []map[string]any{{"text": "beta"}}})
if err != nil {
t.Fatalf("Invoke B: %v", err)
}
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 25dcc1f5f7..2bbd48c199 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -140,6 +140,7 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {
_ = stub
out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{},
})
@@ -190,6 +191,7 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {
}
out, err := cIntf.(*TokenizerComponent).Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{{"text": "alpha bravo"}},
})
@@ -217,6 +219,7 @@ func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *tes
c, stub := withStubEmbedder(t, 2)
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "doc.pdf",
+ "kb_id": "kb-1",
"output_format": "chunks",
"chunks": []map[string]any{},
})
@@ -286,14 +289,14 @@ func TestTokenizerComponent_NewTokenizerComponent_BadParam(t *testing.T) {
}
func TestValidateTokenizerOutputs_FullTextMissingReturnsError(t *testing.T) {
- err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"full_text"}, []string{"text"})
+ err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"full_text"}, []string{"text"}, "")
if err == nil || !strings.Contains(err.Error(), "missing full_text tokens") {
t.Fatalf("err = %v, want missing full_text tokens", err)
}
}
func TestValidateTokenizerOutputs_EmbeddingMissingReturnsError(t *testing.T) {
- err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"embedding"}, []string{"text"})
+ err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"embedding"}, []string{"text"}, "kb-1")
if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {
t.Fatalf("err = %v, want missing embedding vector", err)
}
@@ -301,7 +304,7 @@ func TestValidateTokenizerOutputs_EmbeddingMissingReturnsError(t *testing.T) {
func TestValidateTokenizerOutputs_BothModesFailWhenOneMissing(t *testing.T) {
ck := schema.ChunkDoc{Text: "alpha", ContentLtks: "tok", ContentSmLtks: "sm"}
- err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text", "embedding"}, []string{"text"})
+ err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text", "embedding"}, []string{"text"}, "kb-1")
if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {
t.Fatalf("err = %v, want missing embedding vector", err)
}
@@ -317,7 +320,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.T
ContentLtks: "",
ContentSmLtks: "",
}
- err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text"}, []string{"text"})
+ err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text"}, []string{"text"}, "")
if err == nil || !strings.Contains(err.Error(), "missing full_text tokens") {
t.Fatalf("err = %v, want missing full_text tokens", err)
}
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index 1a5be500e3..bee1d14632 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -39,6 +39,7 @@ import (
"ragflow/internal/common"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
+ "ragflow/internal/ingestion/testutil"
"ragflow/internal/storage"
"github.com/signintech/gopdf"
@@ -89,6 +90,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -180,6 +182,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -268,6 +271,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -366,6 +370,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -463,6 +468,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -545,6 +551,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -625,6 +632,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
attachFixedEmbedderFactory(t, pipe)
out, err := pipe.Run(context.Background(), map[string]any{
"doc_id": docID,
+ "kb_id": "test-kb",
}, nil)
if err != nil {
t.Fatalf("Run: %v", err)
@@ -867,6 +875,15 @@ func withRealTemplateDeps(t *testing.T) storage.Storage {
storage.GetStorageFactory().SetStorage(mem)
t.Cleanup(func() { storage.GetStorageFactory().SetStorage(origStorage) })
+ // The runtime invokes every component with the package-level dao.DB
+ // (see node_body.go). These "real components" tests exercise the
+ // Parser/Extractor model-resolution paths, so wire an in-memory sqlite DB
+ // via the shared testutil helper. It has no tenant/model rows, so
+ // resolution gracefully skips — but the nil-db panic from a headless run
+ // (db never initialized) is avoided without touching production code.
+ db := testutil.SetupTestDB(t)
+ t.Cleanup(testutil.ReplaceDBForTest(t, db))
+
refs := map[string]componentpkg.DocumentStorageRef{}
componentpkg.ResolveDocumentStorageOverride = func(docID string) (*componentpkg.DocumentStorageRef, error) {
ref, ok := refs[docID]
diff --git a/internal/ingestion/task/constants.go b/internal/ingestion/task/constants.go
index 7d9eefc6af..e0450e56f2 100644
--- a/internal/ingestion/task/constants.go
+++ b/internal/ingestion/task/constants.go
@@ -18,9 +18,6 @@ package task
// Special doc_id values used in task messages (mirrors Python constants).
const (
- // CANVAS_DEBUG_DOC_ID marks a canvas debug pipeline task — no persistence.
- CANVAS_DEBUG_DOC_ID = "dataflow_x"
-
// GRAPH_RAPTOR_FAKE_DOC_ID is the fake doc_id used for RAPTOR-generated chunks.
GRAPH_RAPTOR_FAKE_DOC_ID = "graph_raptor_fake_doc"
diff --git a/internal/ingestion/task/debug.go b/internal/ingestion/task/debug.go
new file mode 100644
index 0000000000..b1e6ab5cbc
--- /dev/null
+++ b/internal/ingestion/task/debug.go
@@ -0,0 +1,92 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "strings"
+
+ "github.com/google/uuid"
+ "ragflow/internal/entity"
+)
+
+// NewDebugTaskContext builds an in-memory TaskContext for a canvas-debug
+// (dataflow dry-run) request. The returned context is intentionally
+// side-effect free:
+//
+// - It carries no knowledgebase: KB.ID and Doc.KbID are empty. A canvas
+// debug run has no KB, so kb_id == "" — this is the single debug signal
+// used throughout the ingestion pipeline (the tokenizer skips embedding,
+// the chunker skips image upload, the executor skips the persist stage).
+// kb_id == "" occurs ONLY in debug mode; production ingestion always
+// supplies a KB, so this branch is never reached in normal operation.
+// - Doc.ID is a fresh uuid (a throwaway marker, not a persisted row); it
+// only needs to be non-empty for the executor's input validation.
+// - The parser page cap (debug preview only parses the first few pages)
+// is injected at run time by the executor via Run's override_params
+// channel, keyed by the Parser component's cpnID and the file's family
+// (see injectDebugPageCap). It deliberately does NOT live in
+// ParserConfig as a flat key, because flat keys are dropped by
+// override_params merging and never reach the parser.
+//
+// fileData is the raw bytes of the uploaded debug document (may be nil for a
+// file-less dry-run). It is stored on TaskContext.File and threaded into the
+// pipeline run as the "file" / "binary" input.
+//
+// The caller-supplied kb_id is deliberately ignored: debug never resolves an
+// embedder or writes to a KB, so a debug context has none.
+func NewDebugTaskContext(tenantID, canvasID, fileName string, fileData []byte) *TaskContext {
+ doc := entity.Document{
+ ID: uuid.New().String(),
+ KbID: "",
+ Name: &fileName,
+ }
+ if suffix, docType := deriveDocSuffixAndType(fileName); suffix != "" {
+ doc.Suffix = suffix
+ doc.Type = docType
+ }
+
+ return &TaskContext{
+ Doc: doc,
+ KB: entity.Knowledgebase{ID: ""},
+ Tenant: entity.Tenant{ID: tenantID},
+ PipelineID: canvasID,
+ File: fileData,
+ }
+}
+
+// IsDebug reports whether this context is a canvas-debug (dry-run) context.
+// A debug context carries no knowledgebase (KB.ID == "", forced by
+// NewDebugTaskContext) — the single debug signal used across the ingestion
+// pipeline. Production ingestion always supplies a KB (LoadFromIngestionTask
+// follows doc -> kb), so KB.ID == "" occurs ONLY in debug mode. A debug run
+// must produce no persistent side effect: the executor skips the persist
+// stage, the tokenizer skips embedding, and the chunker skips image upload.
+func (t *TaskContext) IsDebug() bool {
+ return t.KB.ID == ""
+}
+
+// deriveDocSuffixAndType extracts the file extension (e.g. ".pdf") and a
+// lower-cased document type from a file name. It is best-effort: when the
+// name has no usable extension both return "".
+func deriveDocSuffixAndType(fileName string) (suffix, docType string) {
+ dot := strings.LastIndex(fileName, ".")
+ if dot < 0 || dot == len(fileName)-1 {
+ return "", ""
+ }
+ ext := strings.ToLower(fileName[dot+1:])
+ return "." + ext, ext
+}
diff --git a/internal/ingestion/task/debug_log_sink.go b/internal/ingestion/task/debug_log_sink.go
new file mode 100644
index 0000000000..ce55ae406a
--- /dev/null
+++ b/internal/ingestion/task/debug_log_sink.go
@@ -0,0 +1,286 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "sync"
+ "time"
+
+ "ragflow/internal/ingestion/pipeline"
+)
+
+// DebugLogTTL is the Redis expiry for a debug-run log. It mirrors the Python
+// pipeline's 30-minute update TTL (rag/flow/pipeline.py callback: set_obj(...,
+// 60*30)); the Go side writes the log once at the end of the synchronous run.
+const DebugLogTTL = 30 * time.Minute
+
+// DebugLogStore is the minimal Redis surface the debug log sink needs. In
+// production it is satisfied by *redis.Client (which exposes Set(key, value,
+// ttl)); tests pass a capturing closure. Keeping it to a single method keeps the
+// sink free of any redis-package import.
+type DebugLogStore interface {
+ Set(key, value string, ttl time.Duration) bool
+}
+
+// funcStore adapts a plain function to DebugLogStore so tests (and callers that
+// already hold a go-redis client) can wire a sink without a named type.
+type funcStore func(key, value string, ttl time.Duration) bool
+
+// Set implements DebugLogStore.
+func (f funcStore) Set(key, value string, ttl time.Duration) bool { return f(key, value, ttl) }
+
+// component phases mirror runtime.ProgressPhase (Enter=0, Exit=1, Error=2).
+// They are copied locally so the sink stays decoupled from the agent runtime
+// package; the integer values are part of the pipeline's stable event contract.
+const (
+ phaseEnter = 0
+ phaseExit = 1
+ phaseError = 2
+)
+
+// Debug-log size guards. A debug run may emit an unbounded number of component
+// lifecycle events (e.g. a misbehaving component looping), so the sink caps the
+// number of components, the traces per component, and the per-message length.
+// This stops a single run from exhausting memory or writing an oversized entry
+// into the shared Redis. The bounds are far above any legitimate canvas and only
+// catch pathological runs.
+const (
+ maxLogEntries = 1000 // distinct components recorded per run
+ maxTracePerEntry = 200 // progress events kept per component
+ maxMessageRunes = 1024 // rune length cap for a single trace message
+ maxPayloadBytes = 1 << 20 // hard ceiling on the JSON written to Redis (1 MiB)
+)
+
+// debugLogEntry is one component's row in the front-end Log box: a component id
+// plus an ordered list of trace lines. It matches the Python pipeline callback
+// shape exactly (rag/flow/pipeline.py) so GetAgentLogs can return it verbatim
+// and the front-end (useFetchMessageTrace) can read it unchanged.
+type debugLogEntry struct {
+ ComponentID string `json:"component_id"`
+ Trace []debugTrace `json:"trace"`
+}
+
+type debugTrace struct {
+ Progress float64 `json:"progress"`
+ Message string `json:"message"`
+ Datetime string `json:"datetime"`
+ Timestamp float64 `json:"timestamp"`
+ ElapsedTime float64 `json:"elapsed_time"`
+ // Dsl carries the debug-run result (per-component outputs) on the END
+ // marker's single trace entry, so the front-end "View result" page can
+ // render parsed chunks. It mirrors Python's END marker `dsl`
+ // (rag/flow/pipeline.py:98). Nil/empty when unset, in which case
+ // encoding/json omits it (json:"omitempty").
+ Dsl json.RawMessage `json:"dsl,omitempty"`
+}
+
+// DebugLogSink implements pipeline.ProgressSink by accumulating component
+// lifecycle events into the [{component_id, trace}] array the front-end polls
+// for. It deliberately owns no Redis I/O until Flush: all events are buffered
+// in memory, so the sink is fully testable with a fake DebugLogStore and the
+// executor can drive it without a live Redis. A run always ends with an END
+// marker (appended in Flush) carrying a non-empty message, which is the exact
+// signal the front-end uses to consider the run finished.
+type DebugLogSink struct {
+ canvasID string
+ messageID string
+ store DebugLogStore
+ ttl time.Duration
+
+ mu sync.Mutex
+ entries []debugLogEntry
+ lastTS float64
+ // resultDSL is the debug-run result built by BuildDebugResultDSL and handed
+ // to the sink via SetResult (the ResultSink capability). It is written onto
+ // the END marker's trace entry in Flush. Empty until SetResult is called.
+ resultDSL json.RawMessage
+}
+
+// NewDebugLogSink builds a sink that writes to "{canvasID}-{messageID}-logs".
+func NewDebugLogSink(canvasID, messageID string, store DebugLogStore) *DebugLogSink {
+ return &DebugLogSink{
+ canvasID: canvasID,
+ messageID: messageID,
+ store: store,
+ ttl: DebugLogTTL,
+ }
+}
+
+// OnComponentTotal is a no-op for the debug log: the array shape does not use
+// the component-count denominator (it is only consumed by the DB-backed sink
+// for progress aggregation).
+func (s *DebugLogSink) OnComponentTotal(_ context.Context, _ string, _ int) {}
+
+// SetResult receives the debug-run result DSL (built by BuildDebugResultDSL)
+// and stores it for the END marker. It implements the optional ResultSink
+// capability the executor probes for, so the sink stays decoupled from how the
+// DSL is produced. Safe to call at most once per run; a later call replaces the
+// previous value.
+func (s *DebugLogSink) SetResult(dsl map[string]any) {
+ b, err := json.Marshal(dsl)
+ if err != nil {
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.resultDSL = b
+}
+
+// OnComponentProgress maps one component lifecycle event to a trace line and
+// merges it into the running array. Consecutive events for the same component
+// append to that component's trace (matching the Python callback's merge
+// behaviour); a new component opens a fresh array element.
+func (s *DebugLogSink) OnComponentProgress(_ context.Context, ev pipeline.ProgressEvent) {
+ now := time.Now()
+ ts := float64(now.UnixMicro())
+ progress := 0.0
+ switch ev.Phase {
+ case phaseExit:
+ progress = 1.0
+ case phaseError:
+ progress = -1.0
+ }
+
+ message := ev.Message
+ if ev.Phase == phaseError {
+ // Prefix so the front-end timeline renders the line red
+ // (web/src/pages/agent/pipeline-log-sheet/dataflow-timeline.tsx).
+ message = "[ERROR] " + message
+ }
+ // Clamp the message so a runaway component cannot blow up the stored entry.
+ message = truncateRunes(message, maxMessageRunes)
+
+ entry := debugTrace{
+ Progress: progress,
+ Message: message,
+ Datetime: now.Format("15:04:05"),
+ Timestamp: float64(now.UnixNano()) / 1e9,
+ ElapsedTime: 0,
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.lastTS != 0 {
+ // ts and s.lastTS are UnixMicro(); report the delta in SECONDS so the
+ // front-end's verbatim "s" rendering (dataflow-timeline.tsx)
+ // and the "< 0.000001" gates in the other timeline views stay correct.
+ entry.ElapsedTime = (ts - s.lastTS) / 1e6
+ }
+ s.lastTS = ts
+
+ if n := len(s.entries); n > 0 && s.entries[n-1].ComponentID == ev.Component {
+ if len(s.entries[n-1].Trace) >= maxTracePerEntry {
+ // This component already has its full quota of trace lines; drop the
+ // rest so the buffer stays bounded.
+ return
+ }
+ s.entries[n-1].Trace = append(s.entries[n-1].Trace, entry)
+ return
+ }
+ if len(s.entries) >= maxLogEntries {
+ // Component quota reached; ignore further components.
+ return
+ }
+ s.entries = append(s.entries, debugLogEntry{
+ ComponentID: ev.Component,
+ Trace: []debugTrace{entry},
+ })
+}
+
+// Flush serialises the accumulated array (plus a terminal END marker) and
+// persists it to the store. finalErr, when non-nil, is surfaced in the END
+// message so a failed run is still reported and detectable as complete. All
+// Redis I/O is best-effort: a store failure is swallowed (the run must not
+// abort because the log could not be written).
+func (s *DebugLogSink) Flush(ctx context.Context, finalErr error) {
+ s.mu.Lock()
+ entries := append([]debugLogEntry(nil), s.entries...)
+
+ now := time.Now()
+ endTS := float64(now.UnixMicro())
+ elapsed := 0.0
+ if s.lastTS != 0 {
+ // endTS and s.lastTS are UnixMicro(); report the delta in SECONDS
+ // (see OnComponentProgress for the matching conversion).
+ elapsed = (endTS - s.lastTS) / 1e6
+ }
+ endMsg := "Debug run completed"
+ if finalErr != nil {
+ endMsg = "[ERROR] " + finalErr.Error()
+ }
+ entries = append(entries, debugLogEntry{
+ ComponentID: "END",
+ Trace: []debugTrace{{
+ Progress: 1.0,
+ Message: endMsg,
+ Datetime: now.Format("15:04:05"),
+ Timestamp: float64(now.UnixNano()) / 1e9,
+ ElapsedTime: elapsed,
+ Dsl: s.resultDSL,
+ }},
+ })
+ s.mu.Unlock()
+
+ payload, err := json.Marshal(entries)
+ if err != nil {
+ return
+ }
+ // Hard ceiling: if a buggy/abusive run still exceeds it, drop the middle
+ // entries (keeping the first few and the END marker) until under budget.
+ if len(payload) > maxPayloadBytes {
+ entries = trimToPayloadBudget(entries, maxPayloadBytes)
+ if payload, err = json.Marshal(entries); err != nil {
+ return
+ }
+ }
+ key := s.canvasID + "-" + s.messageID + "-logs"
+ s.store.Set(key, string(payload), s.ttl)
+}
+
+// truncateRunes returns s truncated to at most max runes, preserving the original
+// bytes when shorter. Mirrors internal/service/agent_sessions.go's truncateRunes.
+func truncateRunes(s string, max int) string {
+ if max <= 0 {
+ return ""
+ }
+ r := []rune(s)
+ if len(r) <= max {
+ return s
+ }
+ return string(r[:max])
+}
+
+// trimToPayloadBudget collapses the middle of the log (keeping the first few
+// entries plus the END marker at the tail) until the marshaled size fits the
+// byte budget. The END marker is always the last element, so it is preserved.
+// The loop terminates because each pass shrinks the slice; once only the head
+// and END remain the payload is tiny.
+func trimToPayloadBudget(entries []debugLogEntry, budget int) []debugLogEntry {
+ for len(entries) > 2 {
+ if b, err := json.Marshal(entries); err == nil && len(b) <= budget {
+ return entries
+ }
+ keep := len(entries) / 4
+ if keep < 1 {
+ keep = 1
+ }
+ entries = append(append([]debugLogEntry{}, entries[:keep]...), entries[len(entries)-1])
+ }
+ return entries
+}
diff --git a/internal/ingestion/task/debug_log_sink_test.go b/internal/ingestion/task/debug_log_sink_test.go
new file mode 100644
index 0000000000..97388a33d6
--- /dev/null
+++ b/internal/ingestion/task/debug_log_sink_test.go
@@ -0,0 +1,766 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+ "unicode/utf8"
+
+ "gorm.io/gorm"
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/pipeline"
+)
+
+// capturedStore is a fake DebugLogStore that records every write so tests can
+// assert the exact JSON the sink would persist to Redis — no live Redis needed.
+type capturedStore struct {
+ mu sync.Mutex
+ data map[string]string
+}
+
+func (c *capturedStore) Set(key, value string, ttl time.Duration) bool {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.data == nil {
+ c.data = map[string]string{}
+ }
+ c.data[key] = value
+ return true
+}
+
+func (c *capturedStore) get(key string) string {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.data[key]
+}
+
+// clientConsidersComplete mirrors the front-end completion predicate in
+// web/src/pages/agent/hooks/use-fetch-pipeline-log.ts: the run is done only
+// when the LAST array element has component_id == "END" and its first trace
+// message is non-empty.
+func clientConsidersComplete(arr []map[string]any) bool {
+ if len(arr) == 0 {
+ return false
+ }
+ last := arr[len(arr)-1]
+ if last["component_id"] != "END" {
+ return false
+ }
+ trace, ok := last["trace"].([]any)
+ if !ok || len(trace) == 0 {
+ return false
+ }
+ first, ok := trace[0].(map[string]any)
+ if !ok {
+ return false
+ }
+ msg, _ := first["message"].(string)
+ return strings.TrimSpace(msg) != ""
+}
+
+func loadArray(t *testing.T, raw string) []map[string]any {
+ t.Helper()
+ var arr []map[string]any
+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {
+ t.Fatalf("stored value is not a JSON array: %v body=%s", err, raw)
+ }
+ return arr
+}
+
+// TestDebugLogSink_RecordsTraceAndEndMarker covers the core shape contract:
+// same-component events merge into one element's trace, a distinct component
+// opens a new element, and Flush appends an END marker so the front-end can
+// detect completion.
+func TestDebugLogSink_RecordsTraceAndEndMarker(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c1", "m1", store)
+
+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{
+ Component: "File", Message: "File Started", Phase: phaseEnter,
+ })
+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{
+ Component: "File", Message: "File Done", Phase: phaseExit,
+ })
+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{
+ Component: "Chunker", Message: "Chunker Done", Phase: phaseExit,
+ })
+
+ sink.Flush(context.Background(), nil)
+
+ raw := store.get("c1-m1-logs")
+ if raw == "" {
+ t.Fatalf("expected key c1-m1-logs to be written")
+ }
+ arr := loadArray(t, raw)
+ // File (merged, 2 trace entries) + Chunker + END = 3 elements.
+ if len(arr) != 3 {
+ t.Fatalf("want 3 elements, got %d: %s", len(arr), raw)
+ }
+ if arr[0]["component_id"] != "File" {
+ t.Errorf("elem0.component_id=%v want File", arr[0]["component_id"])
+ }
+ fileTrace, _ := arr[0]["trace"].([]any)
+ if len(fileTrace) != 2 {
+ t.Errorf("File trace should merge 2 entries, got %d: %s", len(fileTrace), raw)
+ }
+ if arr[1]["component_id"] != "Chunker" {
+ t.Errorf("elem1.component_id=%v want Chunker", arr[1]["component_id"])
+ }
+ if arr[2]["component_id"] != "END" {
+ t.Errorf("last component_id=%v want END", arr[2]["component_id"])
+ }
+ endTrace, _ := arr[2]["trace"].([]any)
+ endFirst, _ := endTrace[0].(map[string]any)
+ if strings.TrimSpace(endFirst["message"].(string)) == "" {
+ t.Errorf("END message must be non-empty for completion signal: %s", raw)
+ }
+ if !clientConsidersComplete(arr) {
+ t.Errorf("client would NOT consider run complete: %s", raw)
+ }
+}
+
+// TestDebugLogSink_ErrorPrefix verifies that error-phase events get a [ERROR]
+// prefix (so the front-end timeline renders them red) and that Flush with a
+// non-nil error yields an END marker carrying the error, keeping the run
+// detectable as finished.
+func TestDebugLogSink_ErrorPrefix(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c1", "m1", store)
+
+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{
+ Component: "Tokenizer", Message: "Tokenizer: boom", Phase: phaseError,
+ })
+ sink.Flush(context.Background(), errors.New("boom"))
+
+ raw := store.get("c1-m1-logs")
+ arr := loadArray(t, raw)
+
+ // First element is the errored component; its message must carry [ERROR].
+ erroredTrace, _ := arr[0]["trace"].([]any)
+ erroredFirst, _ := erroredTrace[0].(map[string]any)
+ if !strings.HasPrefix(erroredFirst["message"].(string), "[ERROR] ") {
+ t.Errorf("error trace message should be prefixed [ERROR] , got %q", erroredFirst["message"])
+ }
+
+ // Last element is END with the error surfaced, so the run still completes.
+ lastTrace, _ := arr[len(arr)-1]["trace"].([]any)
+ lastFirst, _ := lastTrace[0].(map[string]any)
+ if !strings.HasPrefix(lastFirst["message"].(string), "[ERROR] ") {
+ t.Errorf("END message on failure should be prefixed [ERROR] , got %q", lastFirst["message"])
+ }
+ if !clientConsidersComplete(arr) {
+ t.Errorf("failed run must still be detectable as complete: %s", raw)
+ }
+}
+
+// TestDebugLogSink_OnComponentTotalIsNoOp ensures the denominator callback
+// (used by the DB-backed sink for progress aggregation) is a safe no-op here:
+// the debug log array shape does not use it.
+func TestDebugLogSink_OnComponentTotalIsNoOp(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c1", "m1", store)
+ // Must not panic and must not write.
+ sink.OnComponentTotal(context.Background(), "task-1", 5)
+ if len(store.data) != 0 {
+ t.Errorf("OnComponentTotal must not write to the store, wrote %v", store.data)
+ }
+}
+
+// TestPipelineExecutor_DebugRunWritesLogViaSink drives the real
+// PipelineExecutor debug branch (KB.ID == "") with an injected fake pipeline
+// that emits progress through the sink. It proves the end-to-end wiring inside
+// the executor: a ProgressSink attached via WithProgressSink is forwarded to the
+// pipeline and its events land in the persisted debug-log array (with the END
+// marker), without touching the DB/index persist path.
+func TestPipelineExecutor_DebugRunWritesLogViaSink(t *testing.T) {
+ ctx := context.Background()
+ taskCtx := NewDebugTaskContext("tenant-1", "c1", "doc.pdf", nil)
+ exec, err := NewPipelineExecutor(taskCtx, "c1", 0)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor: %v", err)
+ }
+
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c1", "m1", store)
+ exec.WithProgressSink(sink)
+
+ // Fake the canvas/DSL boundary: loadDSLFunc returns a dummy DSL and
+ // runPipelineFunc emits progress through the (now attached) sink instead of
+ // running a real pipeline.
+ exec.loadDSLFunc = func(_ context.Context, _ string) (string, string, error) {
+ return "dummy-dsl", "", nil
+ }
+ exec.runPipelineFunc = func(ctx context.Context, _ string) (map[string]any, string, error) {
+ exec.progressSink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "File", Message: "File Done", Phase: phaseExit,
+ })
+ exec.progressSink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "Chunker", Message: "Chunker Done", Phase: phaseExit,
+ })
+ return map[string]any{}, "", nil
+ }
+
+ if _, err := exec.Execute(ctx); err != nil {
+ t.Fatalf("Execute: %v", err)
+ }
+ // Mirrors runCanvasPipelineDebug: flush (incl. END marker) after the run.
+ sink.Flush(ctx, nil)
+
+ raw := store.get("c1-m1-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c1-m1-logs")
+ }
+ arr := loadArray(t, raw)
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Errorf("last element must be END marker, got %s", raw)
+ }
+ if !clientConsidersComplete(arr) {
+ t.Errorf("client would not consider run complete: %s", raw)
+ }
+}
+
+// traceStubComponent is a no-op runtime.Component used to exercise the REAL
+// pipeline → TrackProgress → DebugLogSink wiring (the enter→exit pairing) without
+// a live DB or any real ingestion work.
+type traceStubComponent struct{}
+
+func (traceStubComponent) Invoke(_ context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {
+ return map[string]any{"ok": true}, nil
+}
+
+// traceChunkComponent emits a real `chunks` output so the end-to-end wiring can
+// be asserted to populate the END-marker DSL's params.outputs — the exact
+// contract the front-end "View result" needs and the one the nested-state bug
+// broke (empty Result tabs).
+type traceChunkComponent struct{}
+
+func (traceChunkComponent) Invoke(_ context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {
+ // Use the real component output shape ([]map[string]any as produced by
+ // ChunkDocsToMaps) so the end-to-end strip/recurse path for []map[string]any
+ // is actually exercised — a []any stub hides the deepCopyStrip type gap.
+ return map[string]any{
+ "chunks": []map[string]any{
+ {"text": "real chunk", "vector": []float64{0.5}},
+ },
+ }, nil
+}
+
+// TestDebugLogSink_RealPipeline_EachComponentTraceHasStartedAndDone drives a
+// REAL pipeline (not the fake runPipelineFunc used elsewhere) through the same
+// wiring a dataflow debug run uses: NewPipelineFromDSL + WithProgressSink
+// (DebugLogSink) + Run. The canvas framework wraps each component in
+// realComponentBody → runtime.TrackProgress, which MUST emit BOTH PhaseEnter
+// (" Started", progress 0) and PhaseExit (" Done", progress 1) on
+// success.
+//
+// This locks the contract the front-end depends on: every component ends with a
+// Done line, not just a Started line. It is the regression guard for the
+// observation that a debug run's polled log appeared to stay at progress 0 —
+// per the code, each component's trace must contain [Started, Done].
+func TestDebugLogSink_RealPipeline_EachComponentTraceHasStartedAndDone(t *testing.T) {
+ const (
+ compA = "trace.RealStubA"
+ compB = "trace.RealStubB"
+ )
+ runtime.MustRegister(compA, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+ runtime.MustRegister(compB, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c-trace", "m-trace", store)
+
+ dsl := []byte(`{"dsl":{"components":{
+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["a"]},
+ "a":{"obj":{"component_name":"` + compA + `","params":{}},"upstream":["begin"],"downstream":["b"]},
+ "b":{"obj":{"component_name":"` + compB + `","params":{}},"upstream":["a"]}
+ },"path":["begin","a","b"],"graph":{"nodes":[]}}}`)
+
+ pipe, err := pipeline.NewPipelineFromDSL(dsl, "task-trace",
+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-trace"))
+ if err != nil {
+ t.Fatalf("NewPipelineFromDSL: %v", err)
+ }
+ if _, err := pipe.Run(context.Background(), map[string]any{"name": "doc-trace"}, nil); err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+ sink.Flush(context.Background(), nil)
+
+ raw := store.get("c-trace-m-trace-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c-trace-m-trace-logs")
+ }
+ arr := loadArray(t, raw)
+
+ // Completion predicate must hold: END is last with a non-empty message.
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Fatalf("last element must be END marker: %s", raw)
+ }
+ if !clientConsidersComplete(arr) {
+ t.Fatalf("client would not consider run complete: %s", raw)
+ }
+
+ // Every real component (a, b) must show BOTH Started (progress 0) and Done
+ // (progress 1) within its own trace — proving TrackProgress emitted the full
+ // enter→exit lifecycle, not just enter.
+ for _, comp := range []string{"a", "b"} {
+ var gotStart, gotDone bool
+ for _, el := range arr {
+ if el["component_id"] != comp {
+ continue
+ }
+ traces, _ := el["trace"].([]any)
+ for _, tr := range traces {
+ tm, _ := tr.(map[string]any)
+ if tm == nil {
+ continue
+ }
+ prog, _ := tm["progress"].(float64)
+ msg, _ := tm["message"].(string)
+ switch {
+ case prog == 0 && msg == comp+" Started":
+ gotStart = true
+ case prog == 1 && msg == comp+" Done":
+ gotDone = true
+ }
+ }
+ }
+ if !gotStart || !gotDone {
+ t.Fatalf("component %q missing Started/Done in trace: %s", comp, raw)
+ }
+ }
+}
+
+// TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL locks the end-to-end
+// contract the front-end "View result" depends on: the debug log's END marker
+// trace[0] MUST carry a `dsl` (built by BuildDebugResultDSL) whose `components`
+// map mirrors the canvas. The front-end `useFetchPipelineResult` picks
+// `latest.trace.at(0)` where `latest.component_id == "END"` and reads
+// `dsl.components[].obj` to render parsed chunks (parser.tsx:41). Without
+// this, "View result" shows only the file preview — the bug this change fixes.
+//
+// It drives a REAL pipeline (same wiring as a dataflow debug run) so the `dsl`
+// is built from an actual run output map, not a hand-built one.
+func TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL(t *testing.T) {
+ const (
+ compC = "trace.RealStubC"
+ compD = "trace.RealStubD"
+ )
+ runtime.MustRegister(compC, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+ runtime.MustRegister(compD, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c-dsl", "m-dsl", store)
+
+ dsl := `{"dsl":{"components":{
+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["c"]},
+ "c":{"obj":{"component_name":"` + compC + `","params":{}},"upstream":["begin"],"downstream":["d"]},
+ "d":{"obj":{"component_name":"` + compD + `","params":{}},"upstream":["c"]}
+ },"path":["begin","c","d"],"graph":{"nodes":[{"id":"begin","data":{"name":"开始"}},{"id":"c","data":{"name":"解析"}},{"id":"d","data":{"name":"分词"}}]}}}`
+
+ pipe, err := pipeline.NewPipelineFromDSL([]byte(dsl), "task-dsl",
+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-dsl"))
+ if err != nil {
+ t.Fatalf("NewPipelineFromDSL: %v", err)
+ }
+ output, err := pipe.Run(context.Background(), map[string]any{"name": "doc-dsl"}, nil)
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ // The executor would call this after Run; here we invoke it directly.
+ resultDSL, err := BuildDebugResultDSL(dsl, output)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL: %v", err)
+ }
+ sink.SetResult(resultDSL)
+ sink.Flush(context.Background(), nil)
+
+ raw := store.get("c-dsl-m-dsl-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c-dsl-m-dsl-logs")
+ }
+ arr := loadArray(t, raw)
+
+ // END must be the last element.
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Fatalf("last element must be END marker: %s", raw)
+ }
+ endTrace, _ := arr[len(arr)-1]["trace"].([]any)
+ if len(endTrace) == 0 {
+ t.Fatalf("END trace is empty: %s", raw)
+ }
+ endFirst, _ := endTrace[0].(map[string]any)
+ dslVal := endFirst["dsl"]
+ if dslVal == nil {
+ t.Fatalf("END marker trace[0] must carry a non-empty dsl: %s", raw)
+ }
+
+ // json.RawMessage decodes into a generic map as map[string]any (not a
+ // string), so accept either form.
+ var dslDoc map[string]any
+ switch v := dslVal.(type) {
+ case string:
+ if err := json.Unmarshal([]byte(v), &dslDoc); err != nil {
+ t.Fatalf("END dsl is not valid JSON: %v body=%s", err, v)
+ }
+ case map[string]any:
+ dslDoc = v
+ default:
+ t.Fatalf("END dsl unexpected type %T: %s", v, raw)
+ }
+ components, ok := dslDoc["components"].(map[string]any)
+ if !ok || len(components) == 0 {
+ t.Fatalf("END dsl.components missing or empty: %s", raw)
+ }
+ // The dsl must mirror every component in the canvas (begin, c, d).
+ for _, id := range []string{"begin", "c", "d"} {
+ if _, exists := components[id]; !exists {
+ t.Errorf("END dsl.components missing component %q: %s", id, raw)
+ }
+ }
+}
+
+// TestDebugLogSink_RealPipeline_EndMarkerDSLShowsChunks is the strongest
+// regression guard for the nested-state bug: it drives a REAL pipeline whose
+// component "c" emits actual `chunks`, then asserts the END-marker DSL carries
+// a non-empty params.outputs for that component. With the bug (reading
+// output[id] at the top level instead of output["state"][id]), the run output
+// is nested and params.outputs stays empty — the front-end "View result" shows
+// blank tabs. The companion TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL
+// only checks component keys exist (its stubs return {"ok":true}, no chunks),
+// so it would not catch this. This test closes that end-to-end gap.
+//
+// It also pins the inverse: component "d" (stub returning {"ok":true}) has NO
+// recognized output key, so its params.outputs must be absent — the safe-empty
+// contract the front-end renders as a blank step.
+func TestDebugLogSink_RealPipeline_EndMarkerDSLShowsChunks(t *testing.T) {
+ const (
+ compC = "trace.RealStubChunks"
+ compD = "trace.RealStubD2"
+ )
+ runtime.MustRegister(compC, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceChunkComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+ runtime.MustRegister(compD, runtime.CategoryIngestion,
+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
+ runtime.Metadata{Version: "1.0.0"})
+
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c-chunk", "m-chunk", store)
+
+ dsl := `{"dsl":{"components":{
+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["c"]},
+ "c":{"obj":{"component_name":"` + compC + `","params":{"setups":{"pdf":{"parse_method":"general"}}}},"upstream":["begin"],"downstream":["d"]},
+ "d":{"obj":{"component_name":"` + compD + `","params":{}},"upstream":["c"]}
+ },"path":["begin","c","d"],"graph":{"nodes":[{"id":"begin","data":{"name":"开始"}},{"id":"c","data":{"name":"解析"}},{"id":"d","data":{"name":"分词"}}]}}}`
+
+ pipe, err := pipeline.NewPipelineFromDSL([]byte(dsl), "task-chunk",
+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-chunk"))
+ if err != nil {
+ t.Fatalf("NewPipelineFromDSL: %v", err)
+ }
+ output, err := pipe.Run(context.Background(), map[string]any{"name": "doc-chunk"}, nil)
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ resultDSL, err := BuildDebugResultDSL(dsl, output)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL: %v", err)
+ }
+ sink.SetResult(resultDSL)
+ sink.Flush(context.Background(), nil)
+
+ raw := store.get("c-chunk-m-chunk-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written")
+ }
+ arr := loadArray(t, raw)
+ if arr[len(arr)-1]["component_id"] != "END" {
+ t.Fatalf("last element must be END marker: %s", raw)
+ }
+ endTrace, _ := arr[len(arr)-1]["trace"].([]any)
+ endFirst, _ := endTrace[0].(map[string]any)
+
+ var dslDoc map[string]any
+ switch v := endFirst["dsl"].(type) {
+ case string:
+ if err := json.Unmarshal([]byte(v), &dslDoc); err != nil {
+ t.Fatalf("END dsl not valid JSON: %v", err)
+ }
+ case map[string]any:
+ dslDoc = v
+ default:
+ t.Fatalf("END dsl unexpected type %T", v)
+ }
+ components, _ := dslDoc["components"].(map[string]any)
+
+ // Component "c" emits chunks -> params.outputs MUST be populated.
+ cParams := components["c"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
+ cOutputs, ok := cParams["outputs"].(map[string]any)
+ if !ok {
+ t.Fatalf("REGRESSION: chunk-emitting component c has empty params.outputs; "+
+ "the front-end 'View result' would render blank tabs. params=%#v", cParams)
+ }
+ if of, _ := cOutputs["output_format"].(map[string]any); of["value"] != "chunks" {
+ t.Errorf("c output_format=%#v want {value:\"chunks\"}", cOutputs["output_format"])
+ }
+ chunksVal := cOutputs["chunks"].(map[string]any)["value"].([]any)
+ if len(chunksVal) != 1 {
+ t.Fatalf("c chunks.value len=%d want 1", len(chunksVal))
+ }
+ chunk0 := chunksVal[0].(map[string]any)
+ if chunk0["text"] != "real chunk" {
+ t.Errorf("chunk text=%v want 'real chunk'", chunk0["text"])
+ }
+ if _, exists := chunk0["vector"]; exists {
+ t.Errorf("vector must be stripped from chunk payload")
+ }
+ if _, ok := cParams["setups"].(map[string]any); !ok {
+ t.Errorf("c params.setups must be carried from DSL: %#v", cParams)
+ }
+
+ // Component "d" emits {"ok":true} -> no recognized format -> outputs absent.
+ dParams := components["d"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
+ if _, exists := dParams["outputs"]; exists {
+ t.Errorf("d has no recognized output, outputs must be absent: %#v", dParams)
+ }
+}
+
+// TestDebugLogSink_ElapsedTimeIsInSeconds locks the unit contract for
+// `elapsed_time`: it MUST be expressed in SECONDS, not the raw UnixMicro delta.
+// This is the regression guard for the dataflow timeline showing values like
+// "19951s" — the sink stored microseconds while the front-end renders the number
+// verbatim with an "s" suffix (dataflow-timeline.tsx) and the rest of the app
+// (webhook/workflow timelines) gates on `elapsed_time < 0.000001`, i.e. seconds.
+//
+// The test drives two progress events ~20ms apart and asserts the recorded
+// elapsed_time is close to the measured wall-clock gap in seconds (not 1e6x
+// larger, which is what an unfixed microsecond delta would produce).
+func TestDebugLogSink_ElapsedTimeIsInSeconds(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c-elapsed", "m-elapsed", store)
+ ctx := context.Background()
+
+ t0 := time.Now()
+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "A", Message: "A Started", Phase: phaseEnter,
+ })
+ time.Sleep(20 * time.Millisecond)
+ t1 := time.Now()
+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "A", Message: "A Done", Phase: phaseExit,
+ })
+ sink.Flush(ctx, nil)
+
+ raw := store.get("c-elapsed-m-elapsed-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c-elapsed-m-elapsed-logs")
+ }
+ arr := loadArray(t, raw)
+ if len(arr) < 2 {
+ t.Fatalf("expected at least A + END elements, got %d: %s", len(arr), raw)
+ }
+ aTrace, _ := arr[0]["trace"].([]any)
+ if len(aTrace) < 2 {
+ t.Fatalf("A trace should have [Started, Done], got %d entries: %s", len(aTrace), raw)
+ }
+ doneTrace, _ := aTrace[1].(map[string]any)
+ elapsed, ok := doneTrace["elapsed_time"].(float64)
+ if !ok {
+ t.Fatalf("elapsed_time missing or wrong type: %s", raw)
+ }
+
+ gapSeconds := t1.Sub(t0).Seconds()
+ // Tolerance absorbs call-overhead jitter; the key is that elapsed is in the
+ // same SECONDS scale as gapSeconds, not 1e6x larger (microseconds).
+ // With an unfixed microsecond delta, elapsed would be ~gapSeconds*1e6 and
+ // this assertion fails.
+ if math.Abs(elapsed-gapSeconds) > math.Max(gapSeconds*0.5, 0.01) {
+ t.Fatalf("elapsed_time=%v want ~%v seconds (sink must divide the UnixMicro delta by 1e6)", elapsed, gapSeconds)
+ }
+}
+
+// TestDebugLogSink_TimestampIsInSeconds locks the unit contract for the
+// `timestamp` field: it MUST be a Unix epoch in SECONDS, not the raw UnixMicro
+// value. Although the dataflow timeline currently renders the `datetime` string
+// rather than `timestamp`, the field is part of the stable log contract the
+// front-end can rely on; emitting microseconds would put it 1e6 off from any
+// consumer expecting a Unix timestamp. This matches the SECONDS convention
+// already established by elapsed_time and the other (webhook/workflow) timeline
+// views.
+func TestDebugLogSink_TimestampIsInSeconds(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c-ts", "m-ts", store)
+ ctx := context.Background()
+
+ before := time.Now().Unix()
+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "A", Message: "A Started", Phase: phaseEnter,
+ })
+ sink.Flush(ctx, nil)
+
+ raw := store.get("c-ts-m-ts-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c-ts-m-ts-logs")
+ }
+ arr := loadArray(t, raw)
+ aTrace, _ := arr[0]["trace"].([]any)
+ if len(aTrace) == 0 {
+ t.Fatalf("A trace is empty: %s", raw)
+ }
+ firstTrace, _ := aTrace[0].(map[string]any)
+ ts, ok := firstTrace["timestamp"].(float64)
+ if !ok {
+ t.Fatalf("timestamp missing or wrong type: %s", raw)
+ }
+
+ // If the sink emitted UnixMicro, ts would be ~1.7e15; a Unix-seconds value
+ // is ~1.7e9 and must be within a few seconds of `before`.
+ diff := math.Abs(ts - float64(before))
+ if diff > 5 {
+ t.Fatalf("timestamp=%v want ~%d (Unix seconds, not UnixMicro)", ts, before)
+ }
+}
+
+// TestDebugLogSink_RespectsCaps pins the size guards: a pathological run that
+// emits far more components / trace lines / over-long messages than any
+// legitimate canvas ever would must be clamped, while the END completion marker
+// and the front-end's completion predicate stay intact. This is the regression
+// guard for the unbounded-memory / oversized-Redis-entry DoS surface.
+//
+// Counts are deliberately bounded so the test stays fast: we only need to exceed
+// each cap by a small margin to prove clamping, not replay millions of events.
+func TestDebugLogSink_RespectsCaps(t *testing.T) {
+ ctx := context.Background()
+
+ // Component cap: more distinct components than maxLogEntries (each one trace,
+ // one over-long message that must be truncated to the rune cap).
+ storeA := &capturedStore{}
+ sinkA := NewDebugLogSink("c1", "mA", storeA)
+ for i := 0; i < maxLogEntries+200; i++ {
+ sinkA.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: fmt.Sprintf("Comp-%d", i),
+ Message: strings.Repeat("x", 5000), // truncated to maxMessageRunes
+ Phase: phaseExit,
+ })
+ }
+ sinkA.Flush(ctx, nil)
+
+ arrA := loadArray(t, storeA.get("c1-mA-logs"))
+ // Flush always appends the terminal END marker, so count only real components.
+ var compCountA int
+ for _, el := range arrA {
+ if el["component_id"] == "END" {
+ continue
+ }
+ compCountA++
+ }
+ if compCountA > maxLogEntries {
+ t.Fatalf("components=%d want <= %d", compCountA, maxLogEntries)
+ }
+ for _, el := range arrA {
+ traces, _ := el["trace"].([]any)
+ for _, tr := range traces {
+ tm, _ := tr.(map[string]any)
+ if tm == nil {
+ continue
+ }
+ msg, _ := tm["message"].(string)
+ if utf8.RuneCountInString(msg) > maxMessageRunes {
+ t.Fatalf("message rune len=%d want <= %d", utf8.RuneCountInString(msg), maxMessageRunes)
+ }
+ }
+ }
+ if !clientConsidersComplete(arrA) {
+ t.Errorf("clamped log must still satisfy completion predicate; raw=%s", storeA.get("c1-mA-logs"))
+ }
+
+ // Trace cap: a single component with more traces than maxTracePerEntry.
+ storeB := &capturedStore{}
+ sinkB := NewDebugLogSink("c1", "mB", storeB)
+ for j := 0; j < maxTracePerEntry+50; j++ {
+ sinkB.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: "Busy",
+ Message: "ok",
+ Phase: phaseExit,
+ })
+ }
+ sinkB.Flush(ctx, nil)
+
+ arrB := loadArray(t, storeB.get("c1-mB-logs"))
+ // Busy (the only real component) + the END completion marker = 2 elements.
+ if len(arrB) != 2 {
+ t.Fatalf("Busy + END expected, got %d elements: %s", len(arrB), storeB.get("c1-mB-logs"))
+ }
+ busyTrace, _ := arrB[0]["trace"].([]any)
+ if len(busyTrace) > maxTracePerEntry {
+ t.Fatalf("Busy trace len=%d want <= %d", len(busyTrace), maxTracePerEntry)
+ }
+ if !clientConsidersComplete(arrB) {
+ t.Errorf("trace-capped log must still satisfy completion predicate; raw=%s", storeB.get("c1-mB-logs"))
+ }
+}
+
+// TestDebugLogSink_PayloadHardCap pins the absolute byte ceiling on the JSON
+// written to Redis. It fills to the per-entry caps (maxLogEntries components x
+// maxTracePerEntry traces) — which pre-trim is ~16 MiB — and asserts the stored
+// value never exceeds maxPayloadBytes, and still ends with the END marker.
+func TestDebugLogSink_PayloadHardCap(t *testing.T) {
+ store := &capturedStore{}
+ sink := NewDebugLogSink("c1", "m1", store)
+ ctx := context.Background()
+
+ for i := 0; i < maxLogEntries; i++ {
+ comp := fmt.Sprintf("Comp-%d", i)
+ for j := 0; j < maxTracePerEntry; j++ {
+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{
+ Component: comp,
+ Message: "short",
+ Phase: phaseExit,
+ })
+ }
+ }
+ sink.Flush(ctx, nil)
+
+ raw := store.get("c1-m1-logs")
+ if raw == "" {
+ t.Fatalf("expected debug log written to c1-m1-logs")
+ }
+ if len(raw) > maxPayloadBytes {
+ t.Fatalf("stored payload bytes=%d want <= %d", len(raw), maxPayloadBytes)
+ }
+ arr := loadArray(t, raw)
+ if !clientConsidersComplete(arr) {
+ t.Errorf("hard-capped log must still satisfy completion predicate; arr=%s", raw)
+ }
+}
diff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.go
new file mode 100644
index 0000000000..6d38d2dbc2
--- /dev/null
+++ b/internal/ingestion/task/debug_pages_integration_test.go
@@ -0,0 +1,213 @@
+//go:build integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/component"
+ "ragflow/internal/ingestion/pipeline"
+ "ragflow/internal/storage"
+)
+
+// TestExecute_DebugViaEntry_HonorsPagesCap_Integration is the end-to-end
+// proof that the canvas-debug page cap travels through Run's override_params
+// channel (NOT pipeline inputs) and is honoured by the deepdoc/pdf parser.
+//
+// It parses a multi-page PDF twice through a real debug pipeline (real DSL,
+// real pdfium parser, in-memory sqlite + storage):
+//
+// - Uncapped baseline: an explicit cpnID+family page cap of [[1, 1000000]]
+// is supplied in ParserConfig. injectDebugPageCap must RESPECT it, so the
+// parser reads every page.
+// - Capped: no page cap is supplied, so injectDebugPageCap injects the
+// debug default [[1, debugPageCapPages]], and the parser reads only the
+// leading pages.
+//
+// The capped run must therefore yield strictly fewer chunk characters than
+// the uncapped run. This exercises the full path the production code uses
+// (Run's 3rd-arg override_params → applyOverrideParams → c.Setups →
+// ConfigureFromSetup → deepdoc pdf Config.Pages).
+//
+// This is gated by //go:build integration because it depends on the native
+// pdfium library, the 03_multipage.pdf fixture, and (optionally) the
+// tokenizer resource dict.
+func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
+ requireTokenizerPool(t)
+ mustLoadTaskTestConfig(t)
+
+ origDB := dao.DB
+ realDB := mustOpenTaskTestDB(t)
+ dao.DB = realDB
+ t.Cleanup(func() { dao.DB = origDB })
+
+ realStorage := storage.NewMemoryStorage()
+ origStorage := storage.GetStorageFactory().GetStorage()
+ storage.GetStorageFactory().SetStorage(realStorage)
+ t.Cleanup(func() { storage.GetStorageFactory().SetStorage(origStorage) })
+
+ templatePath := filepath.Join(taskRepoRoot(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_general.json")
+ templateBytes, err := os.ReadFile(templatePath)
+ if err != nil {
+ t.Fatalf("read template: %v", err)
+ }
+ templateBytes = disableTokenizerEmbeddingForTaskTemplate(t, templateBytes)
+
+ // envelopeDSL is stored verbatim on the canvas (this is what production
+ // persists), so loadDSLFromCanvas marshals the envelope and Run receives
+ // it. injectDebugPageCap must unwrap it to find the Parser cpnID.
+ var envelope struct {
+ DSL json.RawMessage `json:"dsl"`
+ }
+ if err := json.Unmarshal(templateBytes, &envelope); err != nil {
+ t.Fatalf("unmarshal template envelope: %v", err)
+ }
+ var canvasDSL entity.JSONMap
+ if err := json.Unmarshal(templateBytes, &canvasDSL); err != nil {
+ t.Fatalf("unmarshal template into canvas dsl: %v", err)
+ }
+
+ // Discover the Parser cpnID from the inner DSL.
+ schemas, err := pipeline.ExtractAllComponentParams(envelope.DSL)
+ if err != nil {
+ t.Fatalf("ExtractAllComponentParams: %v", err)
+ }
+ var parserCpnID string
+ for _, s := range schemas {
+ if s.ComponentName == component.ComponentNameParser {
+ parserCpnID = s.CpnID
+ }
+ }
+ if parserCpnID == "" {
+ t.Fatal("template has no Parser component")
+ }
+
+ pdfPath := filepath.Join(taskRepoRoot(t), "internal", "deepdoc", "parser", "pdf", "testdata", "pdfs", "03_multipage.pdf")
+ pdfBytes, err := os.ReadFile(pdfPath)
+ if err != nil {
+ t.Skipf("read pdf fixture %s: %v", pdfPath, err)
+ return
+ }
+
+ suffix := fmt.Sprintf("%d", time.Now().UnixNano())
+ tenantID := taskLimit32("it_tenant_" + suffix)
+ canvasID := taskLimit32("it_canvas_" + suffix)
+
+ if err := realDB.Create(&entity.UserCanvas{
+ ID: canvasID,
+ UserID: tenantID,
+ Permission: "me",
+ CanvasCategory: "agent_canvas",
+ DSL: canvasDSL,
+ }).Error; err != nil {
+ t.Fatalf("create user canvas: %v", err)
+ }
+ t.Cleanup(func() { _ = realDB.Where("id = ?", canvasID).Delete(&entity.UserCanvas{}).Error })
+
+ // Explicit "parse all pages" cap (JSON-decoded []any form, the shape the
+ // parser actually consumes). injectDebugPageCap must respect it and leave
+ // it untouched, so the parser reads every page of the PDF.
+ allPages := map[string]any{
+ parserCpnID: map[string]any{
+ "pdf": map[string]any{
+ "pages": []any{[]any{1, 1000000}},
+ },
+ },
+ }
+
+ newDebugCtx := func(parserConfig map[string]any) *TaskContext {
+ // A canvas-debug (dry-run) context carries no KB: KB.ID == "" is the
+ // single debug signal. The executor then skips the persist stage and
+ // injects the debug page cap (see injectDebugPageCap, gated on
+ // KB.ID == "").
+ return &TaskContext{
+ Doc: entity.Document{
+ ID: "debug-run-1",
+ KbID: "",
+ Name: taskStrPtr("03_multipage.pdf"),
+ Type: "pdf",
+ ParserConfig: parserConfig,
+ },
+ KB: entity.Knowledgebase{
+ ID: "",
+ TenantID: tenantID,
+ EmbdID: "embd-1",
+ },
+ Tenant: entity.Tenant{ID: tenantID},
+ File: pdfBytes,
+ PipelineID: canvasID,
+ }
+ }
+
+ // Uncapped baseline: explicit pages=all → executor respects override.
+ uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor (uncapped): %v", err)
+ }
+ uncapped, err := uncappedExec.Execute(context.Background())
+ if err != nil {
+ t.Fatalf("Execute (uncapped): %v", err)
+ }
+
+ // Capped: no pages override → executor injects [[1, debugPageCapPages]].
+ cappedExec, err := NewPipelineExecutor(newDebugCtx(nil), canvasID, 0)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor (capped): %v", err)
+ }
+ capped, err := cappedExec.Execute(context.Background())
+ if err != nil {
+ t.Fatalf("Execute (capped): %v", err)
+ }
+
+ uncappedLen := len(joinedChunks(uncapped.Chunks))
+ cappedLen := len(joinedChunks(capped.Chunks))
+ if uncappedLen == 0 {
+ t.Fatal("uncapped baseline produced no chunks; fixture/pipeline misconfigured")
+ }
+ if cappedLen == 0 {
+ t.Fatal("pages cap produced no chunks; pages wiring broken")
+ }
+ if cappedLen >= uncappedLen {
+ t.Errorf("pages cap had no effect: capped text len %d >= uncapped %d (pages not honoured via override_params)", cappedLen, uncappedLen)
+ }
+}
+
+// joinedChunks concatenates the text of every chunk into a single string so
+// the page-cap assertion does not depend on how the chunker splits pages.
+func joinedChunks(chunks []map[string]any) string {
+ var b strings.Builder
+ for _, ck := range chunks {
+ for _, key := range []string{"content_with_weight", "content", "text"} {
+ if v, ok := ck[key].(string); ok && v != "" {
+ b.WriteString(v)
+ b.WriteString("\n")
+ break
+ }
+ }
+ }
+ return b.String()
+}
diff --git a/internal/ingestion/task/debug_result_dsl.go b/internal/ingestion/task/debug_result_dsl.go
new file mode 100644
index 0000000000..284bec3f34
--- /dev/null
+++ b/internal/ingestion/task/debug_result_dsl.go
@@ -0,0 +1,253 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This file is the Go analogue of Python's `Graph.__str__`
+// (agent/canvas.py:119) used by the dataflow debug "View result" page.
+// Python attaches `dsl = json.loads(str(self))` to the END debug-log marker
+// (rag/flow/pipeline.py:98); the front-end reads `dsl.components[].obj`
+// to render each component's parsed output. Go has no per-component `__str__`
+// serialization, so BuildDebugResultDSL combines the STATIC DSL structure
+// (component_name / downstream / params / graph.nodes) with the RUN output map
+// (state.go:284 `out[] = `) to emit the identical JSON
+// shape. Result is a strict subset (UI-relevant fields only) of Python's, so
+// the front-end renders chunks with parity and a smaller payload.
+
+package task
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+// ResultSink is an OPTIONAL capability a ProgressSink may implement to receive
+// the debug-run result DSL. The pipeline executor probes for it via a type
+// assertion, so the ProgressSink contract stays unchanged and non-debug
+// (DB-backed) sinks simply ignore it — keeping the coupling one-directional.
+type ResultSink interface {
+ SetResult(dsl map[string]any)
+}
+
+// outputFormats is the priority order used to pick a component's payload key,
+// mirroring NormalizeChunks (chunk_utils.go:27).
+var outputFormats = []string{"chunks", "json", "text", "html", "markdown"}
+
+// vectorKeys are dropped from payloads while copying. The front-end never
+// renders raw vectors and Python's serialized component obj excludes them;
+// stripping keeps the Redis-stored debug log at Python-scale size.
+var vectorKeys = map[string]struct{}{
+ "vector": {},
+ "embedding": {},
+ "q_vec": {},
+ "feature": {},
+}
+
+// isVectorKey reports whether k is a raw embedding-vector key that must be
+// stripped from debug payloads. It matches the fixed legacy keys
+// (vector/embedding/feature) AND the dimension-scoped pattern q__vec that
+// the tokenizer actually emits (see hasEmbeddingVector, tokenizer.go:828, e.g.
+// q_4_vec, q_1024_vec). The literal "q_vec" entry never matches those, so a
+// bare map lookup would let real vectors leak into the Redis log.
+func isVectorKey(k string) bool {
+ if _, ok := vectorKeys[k]; ok {
+ return true
+ }
+ return strings.HasPrefix(k, "q_") && strings.HasSuffix(k, "_vec")
+}
+
+// BuildDebugResultDSL builds the `dsl` object the debug-log END marker carries
+// so the front-end "View result" page can render each component's output.
+//
+// dsl is the raw canvas DSL JSON (optionally wrapped as {"dsl": {...}}). output
+// is the pipeline run output keyed by component id (output[] is that
+// component's outputs map, which may carry chunks/json/text/html/markdown).
+func BuildDebugResultDSL(dsl string, output map[string]any) (map[string]any, error) {
+ var tpl map[string]any
+ if err := json.Unmarshal([]byte(dsl), &tpl); err != nil {
+ return nil, fmt.Errorf("BuildDebugResultDSL: unmarshal dsl: %w", err)
+ }
+ root := tpl
+ if nested, ok := tpl["dsl"].(map[string]any); ok {
+ root = nested
+ }
+
+ components, ok := root["components"].(map[string]any)
+ if !ok {
+ return nil, fmt.Errorf("BuildDebugResultDSL: dsl missing components map")
+ }
+
+ built := make(map[string]any, len(components))
+ for id, raw := range components {
+ comp, _ := raw.(map[string]any)
+ if comp == nil {
+ comp = map[string]any{}
+ }
+
+ // component_name: prefer the nested obj.component_name (Python shape),
+ // fall back to a top-level component_name.
+ name := ""
+ var staticParams map[string]any
+ if obj, _ := comp["obj"].(map[string]any); obj != nil {
+ name, _ = obj["component_name"].(string)
+ if p, _ := obj["params"].(map[string]any); p != nil {
+ staticParams = p
+ }
+ }
+ if name == "" {
+ name, _ = comp["component_name"].(string)
+ }
+
+ down := comp["downstream"] // preserve as-is (string or []any)
+
+ // Build obj.params: start from a deep copy of the static DSL params
+ // (setups/field_name/...), then inject the runtime outputs wrapper.
+ mergedParams := map[string]any{}
+ for k, v := range staticParams {
+ mergedParams[k] = deepCopy(v)
+ }
+ if format, payload := detectFormat(lookupComponentOutput(output, id)); format != "" {
+ mergedParams["outputs"] = map[string]any{
+ format: map[string]any{
+ "value": deepCopyStrip(payload),
+ "type": "",
+ },
+ "output_format": map[string]any{"value": format},
+ }
+ }
+
+ built[id] = map[string]any{
+ "obj": map[string]any{
+ "component_name": name,
+ "params": mergedParams,
+ },
+ "downstream": down,
+ "component_name": name,
+ }
+ }
+
+ result := map[string]any{
+ "components": built,
+ "graph": deepCopy(root["graph"]),
+ }
+ return result, nil
+}
+
+// lookupComponentOutput resolves a single component's runtime output map from
+// the pipeline run result.
+//
+// The production `pipe.Run` return value nests every component's outputs under
+// output["state"][] — finalizeResult (pipeline.go:636) attaches
+// runState.Snapshot() (state.go:296, map[string]map[string]any keyed by cpn
+// id), and statePost (scheduler.go:229) writes each component's top-level
+// output keys there via SetVar(cpnID, k, v). So the canonical lookup is
+// output["state"][id].
+//
+// A flat keyed-by-id shape (output[id] directly) is accepted as a fallback so
+// this builder stays usable for hand-built outputs in tests and any
+// non-Snapshot callers; it is never produced by the real pipeline.
+func lookupComponentOutput(output map[string]any, id string) any {
+ // The production run output nests each component under
+ // output["state"][] (finalizeResult → runState.Snapshot(), state.go:296).
+ // Snapshot returns map[string]map[string]any, but some callers build a
+ // map[string]any-shaped state, so accept BOTH concrete types — a
+ // single-type assertion would silently fail the real shape and fall through
+ // to the (usually empty) top-level lookup.
+ switch state := output["state"].(type) {
+ case map[string]map[string]any:
+ if v, ok := state[id]; ok {
+ return v
+ }
+ case map[string]any:
+ if v, ok := state[id]; ok {
+ return v
+ }
+ }
+ // Fallback: flat shape (tests / non-Snapshot producers).
+ return output[id]
+}
+
+// detectFormat returns the output key (chunks/json/text/html/markdown) present
+// in a component's output map, by priority, plus the raw payload under it.
+// Returns ("", nil) when the component produced no recognized output — the
+// front-end then renders that step empty (matching Python's empty obj).
+func detectFormat(out any) (string, any) {
+ m, ok := out.(map[string]any)
+ if !ok || m == nil {
+ return "", nil
+ }
+ for _, f := range outputFormats {
+ if v, exists := m[f]; exists && v != nil {
+ return f, v
+ }
+ }
+ return "", nil
+}
+
+// deepCopy returns a JSON-compatible deep copy of v (maps/slices/primitives),
+// preserving structure but sharing nothing mutable with the source.
+func deepCopy(v any) any {
+ switch val := v.(type) {
+ case map[string]any:
+ cp := make(map[string]any, len(val))
+ for k, vv := range val {
+ cp[k] = deepCopy(vv)
+ }
+ return cp
+ case []map[string]any:
+ cp := make([]any, len(val))
+ for i, vv := range val {
+ cp[i] = deepCopy(vv)
+ }
+ return cp
+ case []any:
+ cp := make([]any, len(val))
+ for i, vv := range val {
+ cp[i] = deepCopy(vv)
+ }
+ return cp
+ default:
+ return v
+ }
+}
+
+// deepCopyStrip is deepCopy plus dropping vectorKeys from every map it visits.
+// Used for component payloads so raw embedding vectors never reach the log.
+func deepCopyStrip(v any) any {
+ switch val := v.(type) {
+ case map[string]any:
+ cp := make(map[string]any, len(val))
+ for k, vv := range val {
+ if isVectorKey(k) {
+ continue
+ }
+ cp[k] = deepCopyStrip(vv)
+ }
+ return cp
+ case []map[string]any:
+ cp := make([]any, len(val))
+ for i, vv := range val {
+ cp[i] = deepCopyStrip(vv)
+ }
+ return cp
+ case []any:
+ cp := make([]any, len(val))
+ for i, vv := range val {
+ cp[i] = deepCopyStrip(vv)
+ }
+ return cp
+ default:
+ return v
+ }
+}
diff --git a/internal/ingestion/task/debug_result_dsl_test.go b/internal/ingestion/task/debug_result_dsl_test.go
new file mode 100644
index 0000000000..f443fba339
--- /dev/null
+++ b/internal/ingestion/task/debug_result_dsl_test.go
@@ -0,0 +1,430 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "testing"
+)
+
+// TestBuildDebugResultDSL locks the contract that the Go debug log's END-marker
+// `dsl` must satisfy so the front-end "View result" page renders parsed chunks.
+//
+// The shape is the faithful Go analogue of Python's `Graph.__str__`
+// (agent/canvas.py:119) + the END-marker `dsl` attachment (rag/flow/pipeline.py:98):
+// for every component in the DSL we emit `dsl.components[].obj` carrying
+// `component_name` and `params.outputs[].value`, plus `downstream` and
+// `graph.nodes[].data.name`. The front-end reads exactly these keys
+// (web/src/pages/dataflow-result/parser.tsx:41, hooks.ts:215).
+func TestBuildDebugResultDSL(t *testing.T) {
+ const (
+ compA = "my.Parser"
+ compB = "my.Tokenizer"
+ )
+ dsl := `{
+ "dsl": {
+ "components": {
+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
+ "a": {"obj": {"component_name": "` + compA + `", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "upstream": ["begin"], "downstream": ["b"]},
+ "b": {"obj": {"component_name": "` + compB + `", "params": {"field_name": "content"}}, "upstream": ["a"]}
+ },
+ "graph": {"nodes": [
+ {"id": "begin", "data": {"name": "开始"}},
+ {"id": "a", "data": {"name": "解析"}},
+ {"id": "b", "data": {"name": "分词"}}
+ ]}
+ }
+ }`
+
+ // Run output: keyed by component id (state.go:284 `out[ref]=v`). `a` emits
+ // chunks, `b` emits plain text, `begin` has no recognized output key.
+ output := map[string]any{
+ "a": map[string]any{
+ "chunks": []any{
+ map[string]any{"text": "hello", "vector": []float64{0.1, 0.2}},
+ },
+ },
+ "b": map[string]any{"text": "plain"},
+ }
+
+ result, err := BuildDebugResultDSL(dsl, output)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL: %v", err)
+ }
+
+ components, ok := result["components"].(map[string]any)
+ if !ok {
+ t.Fatalf("result.components missing or wrong type: %#v", result["components"])
+ }
+ if len(components) != 3 {
+ t.Fatalf("want 3 components (begin,a,b), got %d: %#v", len(components), components)
+ }
+
+ // --- component "a": must carry chunks under params.outputs ---
+ a, ok := components["a"].(map[string]any)
+ if !ok {
+ t.Fatalf("components.a wrong type: %#v", components["a"])
+ }
+ aObj, ok := a["obj"].(map[string]any)
+ if !ok {
+ t.Fatalf("components.a.obj wrong type: %#v", a["obj"])
+ }
+ if name, _ := aObj["component_name"].(string); name != compA {
+ t.Errorf("components.a.obj.component_name=%v want %q", name, compA)
+ }
+ // downstream preserved
+ if down, _ := a["downstream"].(string); down != "b" {
+ // accept []any form too
+ if dl, _ := a["downstream"].([]any); len(dl) != 1 || dl[0] != "b" {
+ t.Errorf("components.a.downstream=%v want [b]", a["downstream"])
+ }
+ }
+ aParams, ok := aObj["params"].(map[string]any)
+ if !ok {
+ t.Fatalf("components.a.obj.params wrong type: %#v", aObj["params"])
+ }
+ aOutputs, ok := aParams["outputs"].(map[string]any)
+ if !ok {
+ t.Fatalf("components.a.obj.params.outputs wrong type: %#v", aParams["outputs"])
+ }
+ // output_format selects "chunks"
+ of, ok := aOutputs["output_format"].(map[string]any)
+ if !ok || of["value"] != "chunks" {
+ t.Errorf("components.a output_format=%#v want {value:\"chunks\"}", aOutputs["output_format"])
+ }
+ chunksOut, ok := aOutputs["chunks"].(map[string]any)
+ if !ok {
+ t.Fatalf("components.a outputs.chunks wrong type: %#v", aOutputs["chunks"])
+ }
+ chunksVal, _ := chunksOut["value"].([]any)
+ if len(chunksVal) != 1 {
+ t.Fatalf("components.a outputs.chunks.value len=%d want 1", len(chunksVal))
+ }
+ chunk0, _ := chunksVal[0].(map[string]any)
+ if chunk0["text"] != "hello" {
+ t.Errorf("chunk text=%v want hello", chunk0["text"])
+ }
+ // VECTOR STRIPPING: Python's serialized obj excludes raw vectors; verify gone.
+ if _, exists := chunk0["vector"]; exists {
+ t.Errorf("vector key must be stripped from chunk payload, but it is present: %#v", chunk0)
+ }
+ // static params (setups) carried through, matching Python's cpn["params"] copy.
+ if setups, _ := aParams["setups"].(map[string]any); setups == nil {
+ t.Errorf("static params.setups must be carried from DSL, got %#v", aParams)
+ }
+
+ // --- component "b": plain text format ---
+ b, _ := components["b"].(map[string]any)
+ bObj, _ := b["obj"].(map[string]any)
+ bParams, _ := bObj["params"].(map[string]any)
+ bOutputs, _ := bParams["outputs"].(map[string]any)
+ if bOutputs["output_format"].(map[string]any)["value"] != "text" {
+ t.Errorf("components.b output_format=%#v want {value:\"text\"}", bOutputs["output_format"])
+ }
+ // static params.field_name carried (used by front-end chunk text key).
+ if fn, _ := bParams["field_name"].(string); fn != "content" {
+ t.Errorf("components.b params.field_name=%v want content", fn)
+ }
+
+ // --- component "begin": no recognized output -> outputs empty, safe ---
+ begin, _ := components["begin"].(map[string]any)
+ beginObj, _ := begin["obj"].(map[string]any)
+ beginParams, _ := beginObj["params"].(map[string]any)
+ if _, exists := beginParams["outputs"]; exists {
+ t.Errorf("begin has no run output, outputs must be absent: %#v", beginParams)
+ }
+
+ // --- graph.nodes copied through ---
+ graph, ok := result["graph"].(map[string]any)
+ if !ok {
+ t.Fatalf("result.graph missing: %#v", result["graph"])
+ }
+ nodes, _ := graph["nodes"].([]any)
+ if len(nodes) != 3 {
+ t.Fatalf("graph.nodes len=%d want 3", len(nodes))
+ }
+}
+
+// TestBuildDebugResultDSL_NestedState locks the REAL pipeline output shape.
+//
+// Unlike TestBuildDebugResultDSL (which uses a flat keyed-by-id map for
+// readability), the production `pipe.Run` return value nests each component's
+// outputs under output["state"][] — see finalizeResult
+// (pipeline.go:636, `merged["state"] = runState.Snapshot()`) and
+// CanvasState.Snapshot() (state.go:296), which returns
+// map[string]map[string]any keyed by cpn id. statePost (scheduler.go:229)
+// writes each component's top-level output keys there via SetVar(cpnID, k, v).
+//
+// A prior bug read output[id] at the TOP level, which is always nil for the
+// real run, so every component's params.outputs stayed empty and the front-end
+// "View result" rendered blank tabs. This test pins the nested lookup so that
+// regression cannot recur.
+func TestBuildDebugResultDSL_NestedState(t *testing.T) {
+ const (
+ compA = "my.Parser"
+ compB = "my.Tokenizer"
+ )
+ dsl := `{
+ "dsl": {
+ "components": {
+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
+ "a": {"obj": {"component_name": "` + compA + `", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "upstream": ["begin"], "downstream": ["b"]},
+ "b": {"obj": {"component_name": "` + compB + `", "params": {"field_name": "content"}}, "upstream": ["a"]}
+ },
+ "graph": {"nodes": [
+ {"id": "begin", "data": {"name": "开始"}},
+ {"id": "a", "data": {"name": "解析"}},
+ {"id": "b", "data": {"name": "分词"}}
+ ]}
+ }
+ }`
+
+ // REAL shape: Snapshot() returns map[string]map[string]any (state.go:296),
+ // so state is typed map[string]map[string]any at runtime — NOT
+ // map[string]any. Using the real type here is what makes this test a true
+ // regression guard: a single-type assertion in lookupComponentOutput would
+ // pass a map[string]any-shaped test yet fail the production output.
+ output := map[string]any{
+ "state": map[string]map[string]any{
+ "a": {
+ "chunks": []any{
+ map[string]any{"text": "hello", "vector": []float64{0.1, 0.2}},
+ },
+ },
+ "b": {"text": "plain"},
+ },
+ }
+
+ result, err := BuildDebugResultDSL(dsl, output)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL: %v", err)
+ }
+ components, ok := result["components"].(map[string]any)
+ if !ok {
+ t.Fatalf("result.components missing: %#v", result["components"])
+ }
+
+ // "a" must carry chunks under params.outputs even though the run output is
+ // nested under output["state"]["a"].
+ aObj := components["a"].(map[string]any)["obj"].(map[string]any)
+ aParams := aObj["params"].(map[string]any)
+ aOutputs, ok := aParams["outputs"].(map[string]any)
+ if !ok {
+ t.Fatalf("NESTED-SHAPE REGRESSION: components.a params.outputs missing; "+
+ "BuildDebugResultDSL must read output[\"state\"][id], got params=%#v", aParams)
+ }
+ if of, _ := aOutputs["output_format"].(map[string]any); of["value"] != "chunks" {
+ t.Errorf("components.a output_format=%#v want {value:\"chunks\"}", aOutputs["output_format"])
+ }
+ chunksVal := aOutputs["chunks"].(map[string]any)["value"].([]any)
+ if len(chunksVal) != 1 {
+ t.Fatalf("components.a outputs.chunks.value len=%d want 1", len(chunksVal))
+ }
+ chunk0 := chunksVal[0].(map[string]any)
+ if chunk0["text"] != "hello" {
+ t.Errorf("chunk text=%v want hello", chunk0["text"])
+ }
+ if _, exists := chunk0["vector"]; exists {
+ t.Errorf("vector key must be stripped, but present: %#v", chunk0)
+ }
+
+ // "b": plain text nested under output["state"]["b"].
+ bParams := components["b"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
+ bOutputs, ok := bParams["outputs"].(map[string]any)
+ if !ok {
+ t.Fatalf("NESTED-SHAPE REGRESSION: components.b params.outputs missing; "+
+ "got params=%#v", bParams)
+ }
+ if of, _ := bOutputs["output_format"].(map[string]any); of["value"] != "text" {
+ t.Errorf("components.b output_format=%#v want {value:\"text\"}", bOutputs["output_format"])
+ }
+}
+
+// TestLookupComponentOutput locks the resolution rules of
+// lookupComponentOutput: the production run output nests each component under
+// output["state"][] (finalizeResult + Snapshot), so that is the primary
+// lookup; a flat keyed-by-id shape at the top level is the documented fallback
+// for tests and any non-Snapshot caller. These branches are defensive and must
+// keep behaving if the run-output envelope changes.
+func TestLookupComponentOutput(t *testing.T) {
+ const id = "a"
+
+ // 1. nested present -> returns the nested value (primary path).
+ out1 := map[string]any{"state": map[string]any{id: map[string]any{"chunks": "x"}}}
+ if got := lookupComponentOutput(out1, id); got == nil {
+ t.Errorf("case1: nested present, want non-nil, got nil")
+ }
+
+ // 2. nested present but NOT a map -> must fall back to top-level.
+ out2 := map[string]any{"state": "not-a-map", id: map[string]any{"text": "y"}}
+ if got := lookupComponentOutput(out2, id); got == nil {
+ t.Errorf("case2: state wrong type, must fall back to top-level, got nil")
+ } else if m, _ := got.(map[string]any); m["text"] != "y" {
+ t.Errorf("case2: fallback got %#v want {text:\"y\"}", got)
+ }
+
+ // 3. nested present, id missing in state but present at top-level -> fallback.
+ out3 := map[string]any{"state": map[string]any{"other": map[string]any{}}, id: map[string]any{"text": "z"}}
+ if got := lookupComponentOutput(out3, id); got == nil {
+ t.Errorf("case3: id missing in state, must fall back to top-level, got nil")
+ } else if m, _ := got.(map[string]any); m["text"] != "z" {
+ t.Errorf("case3: fallback got %#v want {text:\"z\"}", got)
+ }
+
+ // 4. nested present, id missing everywhere -> nil (safe empty).
+ out4 := map[string]any{"state": map[string]any{"other": map[string]any{}}}
+ if got := lookupComponentOutput(out4, id); got != nil {
+ t.Errorf("case4: id missing everywhere, want nil, got %#v", got)
+ }
+
+ // 5. no "state" key at all -> flat top-level lookup.
+ out5 := map[string]any{id: map[string]any{"json": "j"}}
+ if got := lookupComponentOutput(out5, id); got == nil {
+ t.Errorf("case5: flat-only, want non-nil, got nil")
+ }
+}
+
+// TestDeepCopyStrip_StripsVectorsFromMapSlice locks the regression for review
+// comment #5: real component chunk output is []map[string]any (as produced by
+// ChunkDocsToMaps), which the type switch MUST recurse into AND strip vectorKeys
+// from. Previously []map[string]any fell into the default branch — no recursion,
+// no stripping — so embedding vectors leaked into the Redis debug log and the
+// copy shared mutable state with the source.
+func TestDeepCopyStrip_StripsVectorsFromMapSlice(t *testing.T) {
+ src := []map[string]any{
+ {
+ "text": "real chunk",
+ "vector": []float64{0.1, 0.2},
+ "embedding": []float64{0.3},
+ "q_1024_vec": []float64{0.7},
+ "nested": map[string]any{"q_vec": []float64{0.9}, "q_4_vec": []float64{0.8}, "keep": "x"},
+ },
+ {"text": "second", "feature": []float64{0.4}},
+ }
+
+ got := deepCopyStrip(src)
+
+ // Returned slice must be a deep copy (new []any holding new maps), not the
+ // original slice/map identity.
+ cp, ok := got.([]any)
+ if !ok {
+ t.Fatalf("deepCopyStrip returned %T, want []any", got)
+ }
+ if len(cp) != 2 {
+ t.Fatalf("len=%d want 2", len(cp))
+ }
+
+ // First chunk: vector & embedding dropped, text kept, nested vector dropped.
+ c0, ok := cp[0].(map[string]any)
+ if !ok {
+ t.Fatalf("element 0 type %T, want map[string]any", cp[0])
+ }
+ if _, exists := c0["vector"]; exists {
+ t.Errorf("vector must be stripped, but present: %#v", c0)
+ }
+ if _, exists := c0["embedding"]; exists {
+ t.Errorf("embedding must be stripped, but present: %#v", c0)
+ }
+ if _, exists := c0["q_1024_vec"]; exists {
+ t.Errorf("q_1024_vec (dimension-scoped vector key) must be stripped, but present: %#v", c0)
+ }
+ if c0["text"] != "real chunk" {
+ t.Errorf("text=%v want 'real chunk'", c0["text"])
+ }
+ nested, _ := c0["nested"].(map[string]any)
+ if _, exists := nested["q_vec"]; exists {
+ t.Errorf("nested q_vec must be stripped, but present: %#v", nested)
+ }
+ if _, exists := nested["q_4_vec"]; exists {
+ t.Errorf("nested q_4_vec (dimension-scoped vector key) must be stripped, but present: %#v", nested)
+ }
+ if nested["keep"] != "x" {
+ t.Errorf("nested.keep=%v want x", nested["keep"])
+ }
+
+ // Second chunk: feature (a vectorKey) stripped, text kept.
+ c1, ok := cp[1].(map[string]any)
+ if !ok {
+ t.Fatalf("element 1 type %T, want map[string]any", cp[1])
+ }
+ if _, exists := c1["feature"]; exists {
+ t.Errorf("feature must be stripped, but present: %#v", c1)
+ }
+ if c1["text"] != "second" {
+ t.Errorf("text=%v want 'second'", c1["text"])
+ }
+
+ // Mutation isolation: mutating the copy must not touch the source.
+ c0["text"] = "mutated"
+ if src[0]["text"] != "real chunk" {
+ t.Errorf("copy mutation leaked into source: %#v", src[0])
+ }
+}
+
+// TestDeepCopy_MapSliceIsDeep locks that deepCopy also recurses []map[string]any
+// (structure preservation, no vector-strip concern for plain deepCopy).
+func TestDeepCopy_MapSliceIsDeep(t *testing.T) {
+ src := []map[string]any{{"text": "a", "n": map[string]any{"v": 1}}}
+ got := deepCopy(src)
+ cp, ok := got.([]any)
+ if !ok {
+ t.Fatalf("deepCopy returned %T, want []any", got)
+ }
+ m, ok := cp[0].(map[string]any)
+ if !ok {
+ t.Fatalf("element type %T, want map[string]any", cp[0])
+ }
+ if m["text"] != "a" {
+ t.Errorf("text=%v want a", m["text"])
+ }
+ if n, _ := m["n"].(map[string]any); n["v"] != 1 {
+ t.Errorf("nested not copied: %#v", m["n"])
+ }
+ // isolate
+ m["text"] = "mut"
+ if src[0]["text"] != "a" {
+ t.Errorf("deepCopy shares mutable state: %#v", src[0])
+ }
+}
+
+// TestDetectFormat_Priority locks the format selection order
+// (chunks > json > text > html > markdown) used to pick a component's payload
+// key. A component that emits multiple recognized keys must surface the
+// highest-priority one, matching NormalizeChunks and the front-end tab render.
+func TestDetectFormat_Priority(t *testing.T) {
+ cases := []struct {
+ name string
+ in map[string]any
+ want string
+ }{
+ {"chunks beats text", map[string]any{"text": "t", "chunks": "c"}, "chunks"},
+ {"json beats text", map[string]any{"text": "t", "json": "j"}, "json"},
+ {"text beats html", map[string]any{"html": "h", "text": "t"}, "text"},
+ {"html beats markdown", map[string]any{"markdown": "m", "html": "h"}, "html"},
+ {"markdown alone", map[string]any{"markdown": "m"}, "markdown"},
+ {"only unrecognized -> empty", map[string]any{"ok": true}, ""},
+ {"empty map -> empty", map[string]any{}, ""},
+ {"nil -> empty", nil, ""},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got, _ := detectFormat(c.in)
+ if got != c.want {
+ t.Errorf("detectFormat(%#v) = %q, want %q", c.in, got, c.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.go
new file mode 100644
index 0000000000..d39681f097
--- /dev/null
+++ b/internal/ingestion/task/debug_test.go
@@ -0,0 +1,254 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "gorm.io/gorm"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/component"
+ "ragflow/internal/ingestion/pipeline"
+)
+
+// TestNewDebugTaskContext_InjectsDebugID asserts the debug context carries the
+// side-effect-free markers: a fresh non-empty Doc.ID (a throwaway uuid, not a
+// persisted row) and an empty KB (debug has no knowledgebase, so kb_id == "" is
+// the debug signal used across the pipeline). The parser page cap is no longer
+// stored as a flat ParserConfig key here — flat keys are dropped by the
+// override_params merge and never reach the parser. It is injected at run time
+// by injectDebugPageCap via Run's override_params channel (see
+// TestInjectDebugPageCap).
+func TestNewDebugTaskContext_InjectsDebugID(t *testing.T) {
+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))
+
+ if taskCtx.Doc.ID == "" {
+ t.Errorf("Doc.ID = %q, want non-empty (uuid)", taskCtx.Doc.ID)
+ }
+ if taskCtx.Doc.ParserConfig != nil {
+ t.Errorf("Doc.ParserConfig = %v, want nil (debug page cap is injected via override_params, not a flat ParserConfig key)", taskCtx.Doc.ParserConfig)
+ }
+ if taskCtx.Doc.KbID != "" {
+ t.Errorf("Doc.KbID = %q, want empty (debug has no KB)", taskCtx.Doc.KbID)
+ }
+ if taskCtx.KB.ID != "" {
+ t.Errorf("KB.ID = %q, want empty (debug has no KB)", taskCtx.KB.ID)
+ }
+ if taskCtx.Tenant.ID != "t1" {
+ t.Errorf("Tenant.ID = %q, want t1", taskCtx.Tenant.ID)
+ }
+ if taskCtx.PipelineID != "canvas-1" {
+ t.Errorf("PipelineID = %q, want canvas-1", taskCtx.PipelineID)
+ }
+}
+
+// TestExecute_DebugViaEntry proves the entry-point constructor produces a
+// valid debug TaskContext that routes through PipelineExecutor.Execute and
+// returns the pipeline's chunks WITHOUT persisting (no index insert, no
+// pipeline log).
+func TestExecute_DebugViaEntry(t *testing.T) {
+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))
+
+ logCalled := false
+ insertCalled := false
+
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor: %v", err)
+ }
+ exec.
+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return "dsl", "canvas-1", nil
+ }).
+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
+ return map[string]any{
+ "chunks": []map[string]any{
+ {"text": "c1"},
+ {"text": "c2"},
+ {"text": "c3"},
+ {"text": "c4"},
+ },
+ }, dsl, nil
+ }).
+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
+ logCalled = true
+ return nil
+ }).
+ WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
+ insertCalled = true
+ return nil, nil
+ })
+
+ result, err := exec.Execute(context.Background())
+ if err != nil {
+ t.Fatalf("Execute: %v", err)
+ }
+ if result == nil {
+ t.Fatal("result is nil")
+ }
+ if len(result.Chunks) != 4 {
+ t.Errorf("len(result.Chunks) = %d, want 4", len(result.Chunks))
+ }
+ if logCalled {
+ t.Error("pipeline log should NOT be created in a debug (kb_id == \"\") run")
+ }
+ if insertCalled {
+ t.Error("chunk insert should NOT be called in a debug (kb_id == \"\") run")
+ }
+}
+
+// TestInjectDebugPageCap verifies the canvas-debug page cap is delivered
+// through Run's override_params channel (the existing ParserConfig shape),
+// NOT through pipeline inputs. The cap must land at
+// ParserConfig[cpnID][family]["pages"], expressed as the JSON-decoded
+// []any{[]any{1, N}} form (a list of [from,to] pairs) — cpnID is the Parser
+// component's instance id from the DSL and family is the document's filetype
+// family. This is the exact shape NormalizeParserConfigPages produces after a
+// storage JSON round-trip and the shape the deepdoc pdf parser consumes
+// (NormalizePDFPages requires []any, not a Go [][]int).
+//
+// It also pins the regression: a flat top-level "pages" key would be dropped
+// by the override_params merge and never reach the parser, so the cap must be
+// nested under the cpnID.
+func TestInjectDebugPageCap(t *testing.T) {
+ templatePath := filepath.Join(taskRepoRoot(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_general.json")
+ raw, err := os.ReadFile(templatePath)
+ if err != nil {
+ t.Fatalf("read template: %v", err)
+ }
+ var envelope struct {
+ DSL json.RawMessage `json:"dsl"`
+ }
+ if err := json.Unmarshal(raw, &envelope); err != nil {
+ t.Fatalf("unmarshal template envelope: %v", err)
+ }
+ dsl := string(envelope.DSL)
+
+ // Discover the Parser cpnID the same way the executor does.
+ schemas, err := pipeline.ExtractAllComponentParams(envelope.DSL)
+ if err != nil {
+ t.Fatalf("ExtractAllComponentParams: %v", err)
+ }
+ var parserCpnID string
+ for _, s := range schemas {
+ if s.ComponentName == component.ComponentNameParser {
+ parserCpnID = s.CpnID
+ }
+ }
+ if parserCpnID == "" {
+ t.Fatal("template has no Parser component")
+ }
+
+ t.Run("pdf injects [1,2] under cpnID+family", func(t *testing.T) {
+ parserConfig := map[string]any{}
+ injectDebugPageCap(dsl, parserConfig, "pdf")
+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])
+ }
+ pdf, ok := famEntry["pdf"].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q][\"pdf\"] = %T, want map[string]any", parserCpnID, famEntry["pdf"])
+ }
+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {
+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, pdf["pages"], debugPageCapPages)
+ }
+ })
+
+ t.Run("docx injects under docx family", func(t *testing.T) {
+ parserConfig := map[string]any{}
+ injectDebugPageCap(dsl, parserConfig, "docx")
+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])
+ }
+ docx, ok := famEntry["docx"].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q][\"docx\"] = %T, want map[string]any", parserCpnID, famEntry["docx"])
+ }
+ if !reflect.DeepEqual(docx["pages"], []any{[]any{1, debugPageCapPages}}) {
+ t.Errorf("parserConfig[%q][\"docx\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, docx["pages"], debugPageCapPages)
+ }
+ })
+
+ t.Run("empty docType is a no-op", func(t *testing.T) {
+ parserConfig := map[string]any{}
+ injectDebugPageCap(dsl, parserConfig, "")
+ if len(parserConfig) != 0 {
+ t.Errorf("parserConfig = %v, want empty (no family derivable from empty docType)", parserConfig)
+ }
+ })
+
+ t.Run("does not clobber existing parser params", func(t *testing.T) {
+ parserConfig := map[string]any{
+ parserCpnID: map[string]any{
+ "pdf": map[string]any{"parse_method": "deepdoc"},
+ },
+ }
+ injectDebugPageCap(dsl, parserConfig, "pdf")
+ famEntry := parserConfig[parserCpnID].(map[string]any)
+ pdf := famEntry["pdf"].(map[string]any)
+ if pdf["parse_method"] != "deepdoc" {
+ t.Errorf("parserConfig[%q][\"pdf\"][\"parse_method\"] = %v, want deepdoc (existing params must be preserved)", parserCpnID, pdf["parse_method"])
+ }
+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {
+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, pdf["pages"], debugPageCapPages)
+ }
+ })
+
+ t.Run("envelope dsl form (production shape)", func(t *testing.T) {
+ // In production dsl is the canvas envelope {"dsl": {"components": ...}},
+ // not the bare components map. injectDebugPageCap must still find the
+ // Parser cpnID after unwrapping.
+ wrapped := fmt.Sprintf(`{"dsl":%s}`, string(envelope.DSL))
+ parserConfig := map[string]any{}
+ injectDebugPageCap(wrapped, parserConfig, "pdf")
+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q] = %T, want map[string]any (envelope dsl must unwrap)", parserCpnID, parserConfig[parserCpnID])
+ }
+ pdf, ok := famEntry["pdf"].(map[string]any)
+ if !ok {
+ t.Fatalf("parserConfig[%q][\"pdf\"] = %T, want map[string]any", parserCpnID, famEntry["pdf"])
+ }
+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {
+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]] (envelope dsl not unwrapped?)", parserCpnID, pdf["pages"], debugPageCapPages)
+ }
+ })
+
+ t.Run("respects explicit caller-supplied cap", func(t *testing.T) {
+ // When the document already carries an explicit cpnID+family page cap,
+ // the debug default must NOT override it (so a wider/narrower cap wins).
+ parserConfig := map[string]any{
+ parserCpnID: map[string]any{
+ "pdf": map[string]any{"pages": []any{[]any{1, 1000000}}},
+ },
+ }
+ injectDebugPageCap(dsl, parserConfig, "pdf")
+ famEntry := parserConfig[parserCpnID].(map[string]any)
+ pdf := famEntry["pdf"].(map[string]any)
+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, 1000000}}) {
+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, 1000000]] (explicit cap must be respected, not overridden by debug default)", parserCpnID, pdf["pages"])
+ }
+ })
+}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index dfd187a550..f8a3fbd5b5 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -28,6 +28,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/knowledge_compile"
pipelinepkg "ragflow/internal/ingestion/pipeline"
@@ -41,9 +42,14 @@ type PipelineResult struct {
DocID string
KbID string
Metadata map[string]any
+ Chunks []map[string]any // populated only in debug (dry-run) mode
ChunkCount int
TokenConsumption int
Duration float64 // pipeline wall-clock seconds
+ // MessageID is the polling key for the debug-run log. The front-end reads
+ // it from the run response and polls GET /agents/:id/logs/:message_id to
+ // render progress; it is empty for non-debug (persist) runs.
+ MessageID string
}
type PipelineExecutor struct {
@@ -66,15 +72,14 @@ func validateTaskContext(taskCtx *TaskContext) error {
if taskCtx.Doc.ID == "" {
return fmt.Errorf("pipeline executor: empty document id")
}
- if taskCtx.Doc.KbID == "" {
+ // A debug (dry-run) context carries no knowledgebase (see
+ // TaskContext.IsDebug), so it must not be required to supply one.
+ if !taskCtx.IsDebug() && taskCtx.Doc.KbID == "" {
return fmt.Errorf("pipeline executor: empty document knowledgebase id")
}
if taskCtx.Doc.Name == nil || *taskCtx.Doc.Name == "" {
return fmt.Errorf("pipeline executor: empty document name")
}
- if taskCtx.KB.ID == "" {
- return fmt.Errorf("pipeline executor: empty knowledgebase id")
- }
if taskCtx.Tenant.ID == "" {
return fmt.Errorf("pipeline executor: empty tenant id")
}
@@ -170,9 +175,10 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
return nil, err
}
- if s.taskCtx.Doc.ID == CANVAS_DEBUG_DOC_ID {
- s.recordPipelineLog(ctx, dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "done")
- return nil, nil
+ // A debug (dry-run) run produces no persistent side effect (no MinIO
+ // image upload, no index insert, no pipeline log); see TaskContext.IsDebug.
+ if s.taskCtx.IsDebug() {
+ return s.collectDebugOutput(ctx, pipelineOutput, start)
}
result, err := s.processOutput(ctx, pipelineOutput, start)
@@ -187,6 +193,22 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
return result, nil
}
+// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.
+// It surfaces the pipeline's chunks so a debug endpoint can render them, but
+// performs no DB/index writes — the embedding vectors already computed by the
+// pipeline run are left on the chunks. This keeps debug runs side-effect free.
+func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
+ chunks := NormalizeChunks(pipelineOutput)
+ return &PipelineResult{
+ DocID: s.taskCtx.Doc.ID,
+ KbID: s.taskCtx.Doc.KbID,
+ Chunks: chunks,
+ ChunkCount: countDistinctChunkIDs(chunks),
+ TokenConsumption: GetEmbeddingTokenConsumption(pipelineOutput),
+ Duration: time.Since(start).Seconds(),
+ }, nil
+}
+
func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
if pipelineOutput == nil {
return nil, nil
@@ -375,6 +397,12 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (
}
parserConfig := map[string]interface{}(s.taskCtx.Doc.ParserConfig)
+ if parserConfig == nil {
+ // Debug (dataflow dry-run) contexts intentionally carry no
+ // ParserConfig; start from an empty map so the debug page cap can be
+ // injected in place below without a nil-map assignment panic.
+ parserConfig = map[string]interface{}{}
+ }
common.InjectExtractorLLMID(parserConfig, s.taskCtx.Tenant.LLMID)
// When the dataset enables auto-metadata, ensure the Extractor node(s)
// carry the enable_metadata mode + field schema so the LLM extraction fires
@@ -404,15 +432,51 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (
if s.taskCtx.Doc.ID != "" {
inputs["doc_id"] = s.taskCtx.Doc.ID
}
- if s.taskCtx.File != nil {
- inputs["file"] = s.taskCtx.File
- }
+ // Run-level metadata shared by both persist and debug (dataflow
+ // dry-run) runs. In debug the KB is absent (NewDebugTaskContext forces
+ // KB.ID == ""); in persist it is the document's own KB. Either way the
+ // Tokenizer reads kb_id from CanvasState.Globals.
inputs["tenant_id"] = s.taskCtx.Tenant.ID
inputs["kb_id"] = s.taskCtx.KB.ID
if s.taskCtx.KB.Language != nil {
inputs["lang"] = *s.taskCtx.KB.Language
}
+ // File delivery and doc metadata differ between the two run modes.
+ debug := s.taskCtx.IsDebug()
+ if debug {
+ // A debug (dry-run) run has no DB document row, so the parser
+ // cannot resolve its bytes via doc_id → storage. Deliver the
+ // uploaded bytes directly as `binary` (what the parser actually
+ // reads) and surface the doc name/type so family detection works.
+ if s.taskCtx.File != nil {
+ inputs["file"] = s.taskCtx.File
+ inputs["binary"] = s.taskCtx.File
+ }
+ if s.taskCtx.Doc.Name != nil && *s.taskCtx.Doc.Name != "" {
+ inputs["name"] = *s.taskCtx.Doc.Name
+ }
+ if s.taskCtx.Doc.Type != "" {
+ inputs["file_type"] = s.taskCtx.Doc.Type
+ }
+ } else {
+ if s.taskCtx.File != nil {
+ inputs["file"] = s.taskCtx.File
+ }
+ }
+
+ // A canvas-debug (dataflow dry-run) must return a fast preview, so it
+ // caps the parser to the first few pages. The cap is delivered through
+ // override_params (Run's 3rd argument) — the SAME channel the
+ // production ParserConfig uses — keyed by the Parser component's cpnID
+ // and the document's filetype family. It is NOT passed through pipeline
+ // inputs: the parser selects pages from ParserConfig[cpnID][family]
+ // ["pages"] (a list of 1-indexed inclusive ranges), exactly mirroring
+ // NormalizeParserConfigPages / pdf_pages_test.go. See injectDebugPageCap.
+ if debug {
+ injectDebugPageCap(dsl, parserConfig, s.taskCtx.Doc.Type)
+ }
+
// Component params from Doc.ParserConfig — including the tenant LLM id
// injected into Extractor components above — are passed to Run as
// override_params, keyed by cpnID with override-wins. The DSL itself is
@@ -421,9 +485,105 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (
if err != nil {
return nil, dsl, err
}
+
+ // Surface the debug-run result DSL to any sink that implements ResultSink
+ // (the DebugLogSink used by canvas-debug runs). This mirrors Python's
+ // END-marker `dsl` attachment (rag/flow/pipeline.py:98) so the front-end
+ // "View result" page can render parsed chunks. The probe is an optional
+ // capability: non-debug (DB-backed) sinks ignore it and the ProgressSink
+ // contract is unchanged, keeping the coupling one-directional.
+ if rs, ok := s.progressSink.(ResultSink); ok {
+ if resultDSL, e := BuildDebugResultDSL(dsl, output); e == nil {
+ rs.SetResult(resultDSL)
+ }
+ }
+
payload, err := pipelinepkg.ExtractPayload(dsl, output)
if err != nil {
return nil, dsl, err
}
return payload, dsl, nil
}
+
+// debugPageCapPages is the number of leading pages a canvas-debug
+// (dataflow dry-run) parses. The debug preview must return fast, so we cap
+// the parser to the first few pages. The cap is expressed as the 1-indexed
+// inclusive range [1, debugPageCapPages], matching the production
+// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).
+const debugPageCapPages = 2
+
+// injectDebugPageCap wires the canvas-debug page cap into parserConfig using
+// the SAME channel the production ParserConfig travels through: Run's
+// override_params (the 3rd argument), keyed by the Parser component's cpnID
+// and the document's filetype family. It must NOT be passed as a pipeline
+// input — the parser selects pages from ParserConfig[cpnID][family]["pages"],
+// which the deepdoc/pdf parser consumes as a list of page ranges.
+//
+// dsl is the raw pipeline DSL (used only to discover the Parser component's
+// cpnID); docType is the uploaded file's extension (e.g. "pdf"). When no
+// Parser component can be found, or the docType yields no known family, the
+// call is a no-op (the run parses everything, which is safe).
+//
+// dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in
+// production (loadDSLFromCanvas marshals canvas.DSL), so it is unwrapped the
+// same way NewPipelineFromDSL does before ExtractAllComponentParams runs.
+// An explicit page cap already present in parserConfig (keyed by cpnID +
+// family) is respected and left untouched — the debug default is only a
+// fallback, so a debug run can honour a narrower or wider caller-supplied cap.
+func injectDebugPageCap(dsl string, parserConfig map[string]any, docType string) {
+ // Unwrap the canvas envelope to the inner components map.
+ var raw map[string]any
+ if err := json.Unmarshal([]byte(dsl), &raw); err != nil {
+ return
+ }
+ if env, ok := raw["dsl"].(map[string]any); ok && len(env) > 0 {
+ raw = env
+ }
+ inner, err := json.Marshal(raw)
+ if err != nil {
+ return
+ }
+ schemas, err := pipelinepkg.ExtractAllComponentParams(inner)
+ if err != nil {
+ return
+ }
+ var parserCpnID string
+ for _, s := range schemas {
+ if s.ComponentName == component.ComponentNameParser {
+ parserCpnID = s.CpnID
+ break
+ }
+ }
+ if parserCpnID == "" {
+ return
+ }
+ family := component.ParserFileFamily(docType)
+ if family == "" {
+ return
+ }
+ // Respect an explicit page cap already present under cpnID + family.
+ if cpnEntry, ok := parserConfig[parserCpnID].(map[string]any); ok {
+ if famEntry, ok := cpnEntry[family].(map[string]any); ok {
+ if _, has := famEntry["pages"]; has {
+ return
+ }
+ }
+ }
+ cpnEntry, ok := parserConfig[parserCpnID].(map[string]any)
+ if !ok {
+ cpnEntry = map[string]any{}
+ parserConfig[parserCpnID] = cpnEntry
+ }
+ famEntry, ok := cpnEntry[family].(map[string]any)
+ if !ok {
+ famEntry = map[string]any{}
+ cpnEntry[family] = famEntry
+ }
+ // pages is delivered as a JSON-decoded map — the very shape a ParserConfig
+ // arrives in from the API/storage JSON round-trip: a []any of [from,to]
+ // pairs. The deepdoc/pdf parser's NormalizePDFPages requires this
+ // []any-of-[]any form (not a Go [][]int), so it is built explicitly here.
+ // The shallow override_params merge in applyOverrideParams preserves this
+ // shape all the way to ConfigureFromSetup, so the cap is honoured.
+ famEntry["pages"] = []any{[]any{1, debugPageCapPages}}
+}
diff --git a/internal/ingestion/task/pipeline_executor_persist_test.go b/internal/ingestion/task/pipeline_executor_persist_test.go
new file mode 100644
index 0000000000..aeca8a05e4
--- /dev/null
+++ b/internal/ingestion/task/pipeline_executor_persist_test.go
@@ -0,0 +1,151 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "context"
+ "testing"
+
+ "gorm.io/gorm"
+ "ragflow/internal/entity"
+)
+
+func strptr(s string) *string { return &s }
+
+// TestExecute_DebugSkipsPipelineLog asserts that a debug run
+// (KB.ID == "") returns before recording the pipeline operation
+// log and before inserting chunks into the index. A debug run
+// must produce no persistent side effects: no pipeline_log and no
+// index insert.
+func TestExecute_DebugSkipsPipelineLog(t *testing.T) {
+ taskCtx := &TaskContext{
+ Ctx: context.Background(),
+ Doc: entity.Document{
+ ID: "debug-run-1",
+ KbID: "",
+ Name: strptr("doc.pdf"),
+ ParserID: "parser-1",
+ Suffix: "pdf",
+ Type: "pdf",
+ SourceType: "local",
+ },
+ KB: entity.Knowledgebase{ID: ""},
+ Tenant: entity.Tenant{ID: "tenant-1"},
+ }
+
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor: %v", err)
+ }
+ exec.WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return `"dsl"`, "", nil
+ })
+ exec.WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
+ return map[string]any{"chunks": []map[string]any{}}, "dsl", nil
+ })
+
+ var logCalled bool
+ exec.WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
+ logCalled = true
+ return nil
+ })
+ var insertCalled bool
+ exec.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
+ insertCalled = true
+ return nil, nil
+ })
+
+ if _, err := exec.Execute(context.Background()); err != nil {
+ t.Fatalf("Execute: %v", err)
+ }
+ if logCalled {
+ t.Errorf("recordPipelineLog must not be called in a debug (kb_id == \"\") run")
+ }
+ if insertCalled {
+ t.Errorf("index insert must not be called in a debug (kb_id == \"\") run")
+ }
+}
+
+// TestExecute_DebugReturnsChunks asserts that a debug (kb_id == "") run
+// returns the pipeline's chunks in the result instead of discarding them. The
+// chunks are surfaced for a debug HTTP endpoint to render; no DB/index writes
+// happen and the pipeline operation log is never recorded.
+func TestExecute_DebugReturnsChunks(t *testing.T) {
+ taskCtx := &TaskContext{
+ Ctx: context.Background(),
+ Doc: entity.Document{
+ ID: "debug-run-1",
+ KbID: "",
+ Name: strptr("doc.pdf"),
+ ParserID: "parser-1",
+ Suffix: "pdf",
+ Type: "pdf",
+ SourceType: "local",
+ },
+ KB: entity.Knowledgebase{ID: ""},
+ Tenant: entity.Tenant{ID: "tenant-1"},
+ }
+
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor: %v", err)
+ }
+ exec.WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return "dsl", "canvas-1", nil
+ })
+ exec.WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
+ return map[string]any{
+ "chunks": []map[string]any{
+ {"id": "c1", "content": "hello", "text": "hello"},
+ {"id": "c2", "content": "world", "text": "world"},
+ {"id": "c3", "content": "debug", "text": "debug"},
+ },
+ "embedding_token_consumption": 12,
+ }, "dsl", nil
+ })
+
+ var logCalled bool
+ exec.WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
+ logCalled = true
+ return nil
+ })
+ var insertCalled bool
+ exec.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
+ insertCalled = true
+ return nil, nil
+ })
+
+ result, err := exec.Execute(context.Background())
+ if err != nil {
+ t.Fatalf("Execute: %v", err)
+ }
+ if result == nil {
+ t.Fatalf("Execute returned nil result in debug (kb_id == \"\") mode; expected chunks")
+ }
+ if len(result.Chunks) != 3 {
+ t.Errorf("expected 3 chunks in result, got %d", len(result.Chunks))
+ }
+ if result.ChunkCount != 3 {
+ t.Errorf("expected ChunkCount 3, got %d", result.ChunkCount)
+ }
+ if logCalled {
+ t.Errorf("recordPipelineLog must not be called in a debug (kb_id == \"\") run")
+ }
+ if insertCalled {
+ t.Errorf("index insert must not be called in a debug (kb_id == \"\") run")
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 296063fc01..1fce515841 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -139,7 +139,6 @@ func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {
{name: "missing doc id", mutate: func(ctx *TaskContext) { ctx.Doc.ID = "" }},
{name: "missing kb id", mutate: func(ctx *TaskContext) { ctx.Doc.KbID = "" }},
{name: "missing doc name", mutate: func(ctx *TaskContext) { ctx.Doc.Name = nil }},
- {name: "missing knowledgebase id", mutate: func(ctx *TaskContext) { ctx.KB.ID = "" }},
{name: "missing tenant id", mutate: func(ctx *TaskContext) { ctx.Tenant.ID = "" }},
}
@@ -155,6 +154,19 @@ func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {
}
}
+// TestNewPipelineExecutor_AcceptsDebugTaskContext verifies the canvas-debug
+// (dry-run) contract: a TaskContext with an empty KB.ID is valid because debug
+// mode carries no knowledgebase. KB.ID == "" never occurs in production
+// ingestion, which always supplies a KB.
+func TestNewPipelineExecutor_AcceptsDebugTaskContext(t *testing.T) {
+ ctx := makeTaskCtx()
+ ctx.KB = entity.Knowledgebase{ID: ""}
+ ctx.Doc.KbID = ""
+ if _, err := NewPipelineExecutor(ctx, "flow-1", 0); err != nil {
+ t.Fatalf("debug TaskContext rejected: %v", err)
+ }
+}
+
func TestNewPipelineExecutor_DocBulkSize(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 128)
if svc.docBulkSize != 128 {
diff --git a/web/src/pages/agent/hooks/use-run-dataflow.ts b/web/src/pages/agent/hooks/use-run-dataflow.ts
index 6dac58acb9..da74fc8865 100644
--- a/web/src/pages/agent/hooks/use-run-dataflow.ts
+++ b/web/src/pages/agent/hooks/use-run-dataflow.ts
@@ -44,6 +44,13 @@ export function useRunDataflow({
return msgId;
} else {
+ // Even on a run failure the debug log (with the [ERROR] END marker) is
+ // already written; surface its polling key so the log sheet can render
+ // the failure timeline instead of leaving it stuck on an empty state.
+ const msgId = get(res, 'data.data.message_id');
+ if (msgId) {
+ setMessageId(msgId);
+ }
message.error(get(res, 'data.message', ''));
}
},