Files
ragflow/internal/agent/canvas/run_tracker.go
Zhichang Yu 12787996d1 feat(agent): Go ingestion pipeline progress mirroring and DeepDOC parser hardening (#16795)
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.

feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.

refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.

refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.

feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).

refactor: translate Chinese prose comments to English in non-harness Go
files.

chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
2026-07-10 10:36:10 +08:00

208 lines
7.8 KiB
Go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// run_tracker.go persists canvas-run business metadata to a Redis Hash.
// See plan §2.6 (Key 2: "agent:run:{run_id}"). This is the *business*
// channel — checkpoint payload (eino bytes) lives in checkpoint_store.go.
//
// Status code mapping (stored as int under the "status" field):
//
// 0 = running, 1 = succeeded, 2 = failed, 3 = cancelled.
package canvas
import (
"context"
"errors"
redis2 "ragflow/internal/engine/redis"
"time"
"github.com/redis/go-redis/v9"
)
// runKeyPrefix is the Redis Hash key namespace for run metadata.
// The full key is "agent:run:{run_id}".
const runKeyPrefix = "agent:run:"
// runStatus values for the "status" hash field.
const (
runStatusRunning = "0"
runStatusSucceeded = "1"
runStatusFailed = "2"
runStatusCancelled = "3"
)
// runFieldInterruptID is the hash field that holds the eino interrupt id a
// run is paused on (plan §4.4, M3). Reuses the run hash — no dedicated key.
const runFieldInterruptID = "interrupt_id"
func runKey(runID string) string { return runKeyPrefix + runID }
// RunTracker manages canvas-run metadata (canvas_id, status, checkpoint
// link, resume chain, ...) on a Redis Hash. Operations are explicit — the
// eino CheckPointStore does NOT write these fields, so callers (HTTP
// handler, cancel watcher) must invoke Start/Mark* at the right points.
type RunTracker struct {
client *redis.Client
ttl time.Duration
}
// NewRunTracker returns a tracker wired to the global Redis client. When
// the cache is uninitialized, client is nil; methods error in that case
// rather than panicking, and tests can inject a client via struct-literal
// construction.
func NewRunTracker(ttl time.Duration) *RunTracker {
var client *redis.Client
if rc := redis2.Get(); rc != nil {
client = rc.GetClient()
}
return &RunTracker{client: client, ttl: ttl}
}
// NewRunTrackerWithClient returns a tracker wired to a caller-supplied
// redis.Client. The intended use is tests that want to drive the
// RunTracker against an in-memory miniredis without touching the
// global Redis cache, but the helper is exported so non-test callers
// (multi-tenant deployments, custom Redis pools) can inject a
// dedicated client without going through the global cache singleton.
func NewRunTrackerWithClient(client *redis.Client, ttl time.Duration) *RunTracker {
return &RunTracker{client: client, ttl: ttl}
}
// Start records a new run as in-progress. canvasID and tenantID identify
// the source DSL and tenant; parentRunID may be empty for fresh runs and
// carries the source run-id for resume chains (R1 in plan §2.6).
//
// The HSet + Expire are sent through a pipeline so a TTL is set on the
// first write — without that, the key would have no expiry and a crashed
// run would leak the hash.
func (t *RunTracker) Start(ctx context.Context, runID, canvasID, tenantID, parentRunID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
now := time.Now().UnixMilli()
key := runKey(runID)
pipe := t.client.Pipeline()
pipe.HSet(ctx, key, map[string]any{
"canvas_id": canvasID,
"tenant_id": tenantID,
"parent_run_id": parentRunID,
"status": runStatusRunning,
"cancel_requested": 0,
"started_at": now,
})
pipe.Expire(ctx, key, t.ttl)
_, err := pipe.Exec(ctx)
return err
}
// AttachCheckpoint writes the latest checkpoint id for this run. It is the
// ONLY writer of the "checkpoint_id" field; every W1/W2/W3/W4 path (plan
// §2.6) must call this once before the run goroutine returns.
func (t *RunTracker) AttachCheckpoint(ctx context.Context, runID, checkpointID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HSet(ctx, runKey(runID), "checkpoint_id", checkpointID).Err()
}
// AttachInterrupt persists the eino interrupt id that paused this run
// (plan §4.4, M3). The id is written to the SAME hash key as the other run
// metadata — no new Redis key — so a process that crashes between "Invoke
// returned an interrupt" and "the next loop iteration resumes" can recover:
// the next process reads GetInterruptID at the top of its resume loop and
// calls compose.ResumeWithData with it. The ONLY writer of "interrupt_id".
func (t *RunTracker) AttachInterrupt(ctx context.Context, runID, interruptID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HSet(ctx, runKey(runID), runFieldInterruptID, interruptID).Err()
}
// GetInterruptID returns the persisted interrupt id for runID. The bool is
// false when no interrupt is pending (field absent or empty). A nil error
// with (id, false) means "no pending interrupt" — callers treat that as
// "fresh run, do a normal Invoke".
func (t *RunTracker) GetInterruptID(ctx context.Context, runID string) (string, bool, error) {
if t == nil || t.client == nil {
return "", false, errors.New("run tracker: redis client not initialized")
}
v, err := t.client.HGet(ctx, runKey(runID), runFieldInterruptID).Result()
if errors.Is(err, redis.Nil) {
return "", false, nil
}
if err != nil {
return "", false, err
}
if v == "" {
return "", false, nil
}
return v, true, nil
}
// ClearInterruptID removes the persisted interrupt id. Called when a run
// completes (no longer paused) or is cancelled (the checkpoint is wiped).
func (t *RunTracker) ClearInterruptID(ctx context.Context, runID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HDel(ctx, runKey(runID), runFieldInterruptID).Err()
}
// MarkSucceeded transitions the run to status=1 and stamps finished_at.
func (t *RunTracker) MarkSucceeded(ctx context.Context, runID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HSet(ctx, runKey(runID),
"status", runStatusSucceeded,
"finished_at", time.Now().UnixMilli(),
).Err()
}
// MarkFailed transitions the run to status=2 and records the reason.
func (t *RunTracker) MarkFailed(ctx context.Context, runID, reason string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HSet(ctx, runKey(runID),
"status", runStatusFailed,
"finished_at", time.Now().UnixMilli(),
"failure_reason", reason,
).Err()
}
// MarkCancelled transitions the run to status=3 and sets the cancel flag.
func (t *RunTracker) MarkCancelled(ctx context.Context, runID string) error {
if t == nil || t.client == nil {
return errors.New("run tracker: redis client not initialized")
}
return t.client.HSet(ctx, runKey(runID),
"status", runStatusCancelled,
"finished_at", time.Now().UnixMilli(),
"cancel_requested", 1,
).Err()
}
// Get returns all hash fields for a run. The empty map (not nil) plus a
// nil error means "no such run" — callers can detect this with len(map)==0
// if they need to distinguish from a key that exists with no fields.
func (t *RunTracker) Get(ctx context.Context, runID string) (map[string]string, error) {
if t == nil || t.client == nil {
return nil, errors.New("run tracker: redis client not initialized")
}
return t.client.HGetAll(ctx, runKey(runID)).Result()
}