Files
ragflow/internal/agent/canvas/compile_test.go
Zhichang Yu 12787996d1 feat(agent): Go ingestion pipeline progress mirroring and DeepDOC parser hardening (#16795)
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.

feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.

refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.

refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.

feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).

refactor: translate Chinese prose comments to English in non-harness Go
files.

chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
2026-07-10 10:36:10 +08:00

218 lines
8.2 KiB
Go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package canvas
import (
"bytes"
"context"
"sort"
"strings"
"testing"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"ragflow/internal/common"
)
// TestCompile_LogsWhenLegacyNodesPresent exercises the
// decoder-bypass guard in Compile: a Canvas that carries
// LoopItem/IterationItem entries in `Components` (i.e. one that
// never went through dsl.NormalizeForCanvas) must produce a
// visible warning through common.Logger. The guard is
// intentionally a log, not a panic, so internal drivers / legacy
// fixtures can still drive Compile; the log makes the regression
// observable.
//
// The test swaps common.Logger for a buffer-backed encoder so
// we can assert on the structured log message. We don't fail on
// `Compile` itself failing — the legacy fixture graph is
// intentionally minimal and may not compile end-to-end without a
// Begin node; the assertion is strictly about the log surface.
func TestCompile_LogsWhenLegacyNodesPresent(t *testing.T) {
var buf bytes.Buffer
prev := common.Logger
common.Logger = zap.New(
zapcore.NewCore(
zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()),
zapcore.AddSync(&buf),
zapcore.InfoLevel,
),
)
t.Cleanup(func() { common.Logger = prev })
c := &Canvas{
Components: map[string]CanvasComponent{
"Loop:abc": {
Obj: CanvasComponentObj{ComponentName: "Loop", Params: map[string]any{}},
},
"LoopItem:def": {
Obj: CanvasComponentObj{ComponentName: "LoopItem", Params: map[string]any{}},
Downstream: []string{"Body:1"},
},
"Body:1": {
Obj: CanvasComponentObj{ComponentName: "Message", Params: map[string]any{}},
},
},
}
// Compile may return an error from downstream BuildWorkflow —
// we ignore it; the assertion is on the log line.
_, _ = Compile(context.Background(), c)
got := buf.String()
if !strings.Contains(got, "LoopItem/IterationItem") {
t.Errorf("expected legacy-node log warning, got %q", got)
}
if !strings.Contains(got, "bypassed dsl.NormalizeForCanvas") {
t.Errorf("expected bypass warning, got %q", got)
}
}
// TestCompile_NoLogOnCleanCanvas is the negative case: a Canvas
// whose components carry only modern names must NOT trip the
// guard. This guards against an over-eager regex that fires on
// every Compile.
func TestCompile_NoLogOnCleanCanvas(t *testing.T) {
var buf bytes.Buffer
prev := common.Logger
common.Logger = zap.New(
zapcore.NewCore(
zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()),
zapcore.AddSync(&buf),
zapcore.InfoLevel,
),
)
t.Cleanup(func() { common.Logger = prev })
c := &Canvas{
Components: map[string]CanvasComponent{
"begin": {
Obj: CanvasComponentObj{ComponentName: "Begin", Params: map[string]any{}},
},
"llm:0": {
Obj: CanvasComponentObj{ComponentName: "LLM", Params: map[string]any{}},
Downstream: []string{},
},
},
}
// We don't fail on Compile's own error (it may fail for many
// reasons unrelated to legacy names); the assertion is on the
// absence of the legacy log line.
_, _ = Compile(context.Background(), c)
got := buf.String()
if strings.Contains(got, "LoopItem/IterationItem") {
t.Errorf("unexpected legacy-node log on clean canvas: %q", got)
}
}
// TestWithCheckPointID_OptionSetsField verifies the new R2 option records
// the stable id on CompileOptions. (Compile cannot apply eino's
// compose.WithCheckPointID directly — it's a run-time Option, not a
// GraphCompileOption — so the id is carried on CompiledCanvas for the
// caller to thread to Workflow.Invoke.)
func TestWithCheckPointID_OptionSetsField(t *testing.T) {
o := CompileOptions{}
WithCheckPointID("task-42")(&o)
if o.CheckPointID != "task-42" {
t.Errorf("WithCheckPointID did not set field: got %q", o.CheckPointID)
}
}
// TestWithInterruptAfterNonTerminalCpn_OptionSetsField verifies the no-arg
// option flips the internal-compute flag.
func TestWithInterruptAfterNonTerminalCpn_OptionSetsField(t *testing.T) {
o := CompileOptions{}
WithInterruptAfterNonTerminalCpn()(&o)
if !o.InterruptAfterNonTerminal {
t.Error("WithInterruptAfterNonTerminalCpn did not set flag")
}
}
// TestComputeNonTerminalCpnIDs covers the core selection rule for the
// no-arg WithInterruptAfterNonTerminalCpn: include every component with
// out-degree > 0, exclude terminal nodes (no downstream) and UserFillUp
// nodes (§4.2.b — they carry their own compose.Interrupt).
func TestComputeNonTerminalCpnIDs(t *testing.T) {
c := &Canvas{
Components: map[string]CanvasComponent{
"n1": {Obj: CanvasComponentObj{ComponentName: "Parser"}, Downstream: []string{"n2"}},
"n2": {Obj: CanvasComponentObj{ComponentName: "LLM"}, Downstream: []string{"n3"}},
"n3": {Obj: CanvasComponentObj{ComponentName: "Answer"}, Downstream: nil}, // terminal
"uf": {Obj: CanvasComponentObj{ComponentName: "UserFillUp"}, Downstream: []string{"n4"}}, // excluded
"n4": {Obj: CanvasComponentObj{ComponentName: "Answer"}, Downstream: nil}, // terminal
},
}
got := computeNonTerminalCpnIDs(c)
sort.Strings(got)
want := []string{"n1", "n2"}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("computeNonTerminalCpnIDs = %v, want %v", got, want)
}
}
// TestCompile_RejectsUserFillUpInResumeMode covers S3 (plan §4.2.b 方案 A):
// when WithInterruptAfterNonTerminalCpn is set (ingestion resume mode), a DSL
// containing a UserFillUp node must be rejected at compile time. A UserFillUp
// node emits its own compose.Interrupt (wait-for-user); the pipeline resume
// loop would catch it via IsInterruptError and auto-resume with nil data,
// silently skipping the human interaction. The guard fires before BuildWorkflow
// so it is independent of whether the rest of the graph compiles.
func TestCompile_RejectsUserFillUpInResumeMode(t *testing.T) {
c := &Canvas{
Components: map[string]CanvasComponent{
"begin": {Obj: CanvasComponentObj{ComponentName: "Begin", Params: map[string]any{}}},
"uf": {Obj: CanvasComponentObj{ComponentName: "UserFillUp"}, Downstream: []string{"end"}},
"end": {Obj: CanvasComponentObj{ComponentName: "Answer", Params: map[string]any{}}},
},
}
_, err := Compile(context.Background(), c, WithInterruptAfterNonTerminalCpn())
if err == nil {
t.Fatal("expected Compile to reject UserFillUp in resume mode, got nil error")
}
if !strings.Contains(err.Error(), "UserFillUp") {
t.Errorf("expected UserFillUp reject message, got %v", err)
}
}
// TestCompile_PropagatesCheckPointID verifies Compile stores the stable id
// on the returned CompiledCanvas. The assertion only fires when the canvas
// actually compiles end-to-end in this environment (component factories /
// DB may be unavailable in unit scope) — matching the repo convention in
// the other Compile tests that ignore compile errors. The option-side
// contract is independently covered by TestWithCheckPointID_OptionSetsField.
func TestCompile_PropagatesCheckPointID(t *testing.T) {
c := &Canvas{
Components: map[string]CanvasComponent{
"begin": {Obj: CanvasComponentObj{ComponentName: "Begin", Params: map[string]any{}}},
"llm:0": {Obj: CanvasComponentObj{ComponentName: "LLM", Params: map[string]any{}}, Downstream: []string{"answer:0"}},
"answer:0": {Obj: CanvasComponentObj{ComponentName: "Answer", Params: map[string]any{}}},
},
}
compiled, err := Compile(context.Background(), c, WithCheckPointID("task-9"))
if err != nil {
t.Skipf("skipping propagation assertion: canvas did not compile in unit scope: %v", err)
}
if compiled == nil {
t.Skip("skipping propagation assertion: nil CompiledCanvas")
}
if compiled.CheckPointID != "task-9" {
t.Errorf("CompiledCanvas.CheckPointID = %q, want %q", compiled.CheckPointID, "task-9")
}
}