[go] Merged memory extractor with ingestion task handler (#17730)

Routes async memory extraction through the shared ingestion NATS
pipeline via task_type=memory, removing the dedicated Redis consumer.
Memory failures ack terminally, transient errors nack for redelivery.
This commit is contained in:
Zhichang Yu
2026-08-03 17:08:14 +08:00
committed by GitHub
parent 8ac047d11a
commit c69a2d4798
10 changed files with 430 additions and 82 deletions

View File

@@ -16,14 +16,32 @@
package common
import "encoding/json"
const (
// TaskSubject is the NATS subject on which ingestion and memory tasks are
// published and consumed. Producer and consumer must reference this single
// symbol so the routing contract cannot diverge (mirrors the RAGFLOW_TASKS
// JetStream subject in internal/engine/nats).
TaskSubject = "tasks.RAGFLOW"
TaskTypeIngestionTask = "ingestion_task"
TaskTypeIngestionTest = "ingestion_test"
// TaskTypeMemory is the async memory-extraction task type. Memory tasks
// share the tasks.RAGFLOW subject and the Ingestor's consumer + worker
// pool with ingestion tasks; processMessage dispatches them by TaskType.
// The memory-specific payload (message_dict/memory_id/source_id) is
// carried in TaskMessage.Payload.
TaskTypeMemory = "memory"
)
type TaskMessage struct {
TaskID string `json:"task_id" binding:"required"`
TaskType string `json:"task_type" binding:"required"`
// Payload carries the task-specific body for non-ingestion task types
// (e.g. the memory extraction payload). It is left empty for ingestion
// tasks and old messages, so existing consumers are unaffected.
Payload json.RawMessage `json:"payload,omitempty"`
}
type TaskHandle interface {