mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-10 17:31:29 +08:00
refactor(task): sink parser page-cap override into pipeline package (#17905)
## Summary
Moves the canvas-debug parser page-cap injection out of the `task`
orchestrator and into a **debug-agnostic** `pipeline` helper, so
`PipelineExecutor` keeps only the orchestration skeleton (resolving one
of the P1 review findings: the executor was overloaded with
DSL/parser-param assembly).
### Changes
- **`pipeline/parser_page_cap.go`** (new):
- `BuildParserPageCapOverride(parserConfig, dsl, docType, capPages int,
parserComponentName string, familyOf)` — injects the
`ParserConfig[cpnID][family]["pages"]` cap through the same
`override_params` channel production uses. The cap value and family
resolution are injected by the caller, so the function carries no debug
semantics and is reusable for any page-cap scenario.
- `ExtractParserCpnID(dsl, parserComponentName)` — shared Parser cpnID
discovery from (optionally enveloped) DSL.
- `UnwrapCanvasDSL(raw []byte)` — exported single source of truth for
stripping the `{"dsl": {...}}` canvas envelope.
- `pipeline` does **not** import `component` (no reverse dependency);
callers inject `component.ComponentNameParser` /
`component.ParserFileFamily`.
- **`task/pipeline_executor.go`**: removed `injectDebugPageCap` (the
`debugPageCapPages = 2` constant stays in the task package). The debug
branch now calls `pipeline.BuildParserPageCapOverride(...)`.
- **`task/pipeline_executor.go` `warnUnknownComponentParams`**: fixed a
production no-op bug — it passed the enveloped DSL straight to
`ExtractAllComponentParams`, which silently errored and disabled the
unknown-cpnID guard. It now unwraps the envelope first.
- **`task/debug_result_dsl.go`**: reuses `pipeline.UnwrapCanvasDSL`
instead of a third inline envelope-unwrap copy.
### Behavior
No external debug-preview behavior changes. The three original
invariants are preserved exactly:
1. explicit `pages` caps under `cpnID+family` are respected (not
overwritten),
2. an empty family (unknown docType) is a no-op,
3. the injected shape is `[]any{[]any{1, capPages}}` (the
`[]any`-of-`[]any` form `NormalizePDFPages` requires).
## Test plan
- New `pipeline/parser_page_cap_test.go`: `BuildParserPageCapOverride`
(inject / respect-existing / unknown-family no-op / no-Parser no-op),
`ExtractParserCpnID` (enveloped + raw), `UnwrapCanvasDSL`.
- `task/debug_test.go`: `TestInjectDebugPageCap` migrated to the new
helper; new
`TestWarnUnknownComponentParamsDetectsUnknownCPNFromEnvelope` captures
the warning via `zaptest/observer` to prove the envelope no-op bug is
fixed.
- Both `internal/ingestion/pipeline` and `internal/ingestion/task` pass
`build.sh --test` (unit tier).
## Notes
- `TOKEN_CHUNKER_HANDOFF.md` is an unrelated untracked file and was
deliberately **not** included in this PR.
This commit is contained in:
@@ -27,6 +27,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/ingestion/component"
|
||||
@@ -42,9 +45,9 @@ import (
|
||||
// real pdfium parser, in-memory sqlite + storage):
|
||||
//
|
||||
// - Uncapped baseline: an explicit cpnID+family page cap of [[1, 1000000]]
|
||||
// is supplied in ParserConfig. injectDebugPageCap must RESPECT it, so the
|
||||
// is supplied in ParserConfig. pipeline.BuildParserPageCapOverride must RESPECT it, so the
|
||||
// parser reads every page.
|
||||
// - Capped: no page cap is supplied, so injectDebugPageCap injects the
|
||||
// - Capped: no page cap is supplied, so pipeline.BuildParserPageCapOverride injects the
|
||||
// debug default [[1, debugPageCapPages]], and the parser reads only the
|
||||
// leading pages.
|
||||
//
|
||||
@@ -79,7 +82,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
|
||||
|
||||
// envelopeDSL is stored verbatim on the canvas (this is what production
|
||||
// persists), so loadDSLFromCanvas marshals the envelope and Run receives
|
||||
// it. injectDebugPageCap must unwrap it to find the Parser cpnID.
|
||||
// it. pipeline.BuildParserPageCapOverride must unwrap it to find the Parser cpnID.
|
||||
var envelope struct {
|
||||
DSL json.RawMessage `json:"dsl"`
|
||||
}
|
||||
@@ -129,7 +132,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
|
||||
t.Cleanup(func() { _ = realDB.Where("id = ?", canvasID).Delete(&entity.UserCanvas{}).Error })
|
||||
|
||||
// Explicit "parse all pages" cap (JSON-decoded []any form, the shape the
|
||||
// parser actually consumes). injectDebugPageCap must respect it and leave
|
||||
// parser actually consumes). pipeline.BuildParserPageCapOverride must respect it and leave
|
||||
// it untouched, so the parser reads every page of the PDF.
|
||||
allPages := map[string]any{
|
||||
parserCpnID: map[string]any{
|
||||
@@ -142,7 +145,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
|
||||
newDebugCtx := func(parserConfig map[string]any) *TaskContext {
|
||||
// A canvas-debug (dry-run) context carries no KB: KB.ID == "" is the
|
||||
// single debug signal. The executor then skips the persist stage and
|
||||
// injects the debug page cap (see injectDebugPageCap, gated on
|
||||
// injects the debug page cap (see pipeline.BuildParserPageCapOverride, gated on
|
||||
// KB.ID == "").
|
||||
return &TaskContext{
|
||||
Doc: entity.Document{
|
||||
@@ -163,6 +166,15 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Capture warnings during the runs so we assert the envelope-unwrap fix
|
||||
// did not regress into noisy warnings: with a well-formed template the
|
||||
// parserConfig cpnID matches the DSL, so warnUnknownComponentParams must
|
||||
// NOT emit its "not present in the pipeline DSL" warning.
|
||||
core, recorded := observer.New(zap.NewAtomicLevelAt(zap.DebugLevel))
|
||||
oldLogger := common.Logger
|
||||
common.Logger = zap.New(core)
|
||||
defer func() { common.Logger = oldLogger }()
|
||||
|
||||
// Uncapped baseline: explicit pages=all → executor respects override.
|
||||
uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0)
|
||||
if err != nil {
|
||||
@@ -183,6 +195,15 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
|
||||
t.Fatalf("Execute (capped): %v", err)
|
||||
}
|
||||
|
||||
// A well-formed template's parserConfig cpnID matches the DSL, so the
|
||||
// unknown-cpnID guard must stay silent. If it warned here, the envelope
|
||||
// was not unwrapped (the pre-fix no-op regression).
|
||||
for _, e := range recorded.All() {
|
||||
if strings.Contains(e.Message, "not present in the pipeline DSL") {
|
||||
t.Errorf("warnUnknownComponentParams emitted an unknown-cpnID warning during a well-formed debug run (envelope-unwrap regression?): %s", e.Message)
|
||||
}
|
||||
}
|
||||
|
||||
uncappedLen := len(joinedChunks(uncapped.Chunks))
|
||||
cappedLen := len(joinedChunks(capped.Chunks))
|
||||
if uncappedLen == 0 {
|
||||
|
||||
Reference in New Issue
Block a user