mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-09-08 10:14:35 +08:00
fix(agent): collect CodeExec artifacts from ReAct tool responses (#16609)
### Summary The Go backend Agent component was not returning artifacts produced by the CodeExec tool. While the Python agent collects the "`_ARTIFACTS`" envelope from tool responses and appends artifact markdown to the final content, the Go agent only returned the assistant text, so generated images were missing from the chat output. ### Changes - Wire `react.WithMessageFuture()` in `runEinoReActAgent` and store the resulting `MessageFuture` in the invocation context. - After the ReAct loop finishes, drain the future and extract ``_ARTIFACTS`` entries from every tool response message. - Support reading the tool payload from both `msg.Content` and `msg.UserInputMultiContent` to match eino's tool contract. - De-duplicate artifacts by URL and render images as `!` and other files as download links. - Add `agent_artifact_test.go` with a regression test that simulates a CodeExec-style tool response carrying an image artifact and verifies it is collected and formatted. ### Verification - `go test ./internal/agent/component/... -run TestAgent_ReActAgent_CollectsArtifactsFromCodeExecTool` passes. - `go test ./internal/agent/component/... -count=1` compiles; the only failure is an unrelated DNS-pinning timeout test (`TestInvoke_ProxyDNSPin`). - `gofmt` clean for modified files. ### Related Fixes the behavior shown in the screenshot where the Go agent ignored the CodeExec-generated PNG artifact.
This commit is contained in:
@@ -13,6 +13,7 @@ package component
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -160,7 +161,13 @@ func runEinoReActAgent(ctx context.Context, p AgentParam) (*schema.Message, erro
|
||||
}
|
||||
|
||||
input := []*schema.Message{schema.UserMessage(p.UserPrompt)}
|
||||
return agent.Generate(ctx, input)
|
||||
opt, future := react.WithMessageFuture()
|
||||
ctx = setArtifactCollector(ctx, future)
|
||||
msg, err := agent.Generate(ctx, input, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// addToolCallMemory summarizes the tool calls observed in msg via
|
||||
@@ -530,7 +537,7 @@ func (c *AgentComponent) Invoke(ctx context.Context, inputs map[string]any) (map
|
||||
}
|
||||
}
|
||||
}
|
||||
artifacts := collectArtifactsFromToolCalls(msg)
|
||||
artifacts := collectArtifactsFromToolCalls(ctx, msg)
|
||||
artifactMD := formatArtifactMarkdown(artifacts, content)
|
||||
out := map[string]any{
|
||||
"content": content + artifactMD,
|
||||
@@ -653,24 +660,132 @@ type artifactEntry struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// collectArtifactsFromToolCalls extracts artifact entries from a
|
||||
// ReAct final message. Today, eino's react.Agent.Generate returns only
|
||||
// the final message — the tool-response state lives inside the agent
|
||||
// runtime and is not directly accessible. So this v1 returns an empty
|
||||
// slice; the wiring lives in a follow-up that hoists the eino state
|
||||
// into a place AgentComponent can read.
|
||||
//
|
||||
// When the tool response state becomes accessible (a future phase),
|
||||
// the entry point to wire it is here: scan the eino conversation
|
||||
// messages for entries whose `Extra["_ARTIFACTS"]` carries the
|
||||
// per-tool artifact metadata, decode the JSON, and append to the
|
||||
// returned slice. The shape expected from each tool is:
|
||||
//
|
||||
// { "name": "report.pdf", "url": "https://..." }
|
||||
func collectArtifactsFromToolCalls(_ *schema.Message) []artifactEntry {
|
||||
// artifactCollectorKey is the context key used to stash the
|
||||
// MessageFuture from react.WithMessageFuture() so the AgentComponent
|
||||
// can collect artifacts after the ReAct loop finishes. The collector
|
||||
// is created per-invocation in runEinoReActAgent.
|
||||
type artifactCollectorKey struct{}
|
||||
|
||||
// setArtifactCollector registers the MessageFuture for this agent run
|
||||
// in the context. It is called from runEinoReActAgent after
|
||||
// react.WithMessageFuture() returns a future.
|
||||
func setArtifactCollector(ctx context.Context, future react.MessageFuture) context.Context {
|
||||
return context.WithValue(ctx, artifactCollectorKey{}, future)
|
||||
}
|
||||
|
||||
// getArtifactCollector retrieves the MessageFuture registered for the
|
||||
// current agent run. Returns nil when no collector was registered
|
||||
// (e.g., tests that stub agentRunner).
|
||||
func getArtifactCollector(ctx context.Context) react.MessageFuture {
|
||||
v := ctx.Value(artifactCollectorKey{})
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
if f, ok := v.(react.MessageFuture); ok {
|
||||
return f
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectArtifactsFromToolCalls drains the MessageFuture stored in
|
||||
// ctx (if any) and extracts artifact entries from every tool response
|
||||
// message that carries a `_ARTIFACTS` payload in its Extra field.
|
||||
// The final message is ignored because it is an assistant message and
|
||||
// does not contain tool results. Returns a de-duplicated list ordered
|
||||
// by first appearance.
|
||||
//
|
||||
// The expected payload shape in each tool response is:
|
||||
//
|
||||
// { "_ARTIFACTS": [{ "name": "report.pdf", "url": "https://..." }, ...] }
|
||||
func collectArtifactsFromToolCalls(ctx context.Context, _ *schema.Message) []artifactEntry {
|
||||
future := getArtifactCollector(ctx)
|
||||
if future == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
var out []artifactEntry
|
||||
|
||||
iter := future.GetMessages()
|
||||
for {
|
||||
msg, ok, err := iter.Next()
|
||||
if err != nil {
|
||||
common.Debug("agent: artifact collection iterator error", zap.Error(err))
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
if msg == nil || msg.Role != schema.Tool {
|
||||
continue
|
||||
}
|
||||
rawArtifacts := extractArtifactsFromToolMessage(msg)
|
||||
for _, a := range rawArtifacts {
|
||||
if a.URL == "" || a.Name == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[a.URL]; exists {
|
||||
continue
|
||||
}
|
||||
seen[a.URL] = struct{}{}
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// extractArtifactsFromToolMessage parses the JSON payload of a tool
|
||||
// response message and returns the `_ARTIFACTS` list. The payload is
|
||||
// read from msg.Content when it is non-empty; otherwise the first text
|
||||
// element of msg.UserInputMultiContent is used. This matches the eino
|
||||
// tool contract where tool results are delivered as a string.
|
||||
func extractArtifactsFromToolMessage(msg *schema.Message) []artifactEntry {
|
||||
payload := msg.Content
|
||||
if payload == "" && len(msg.UserInputMultiContent) > 0 {
|
||||
payload = toolMessageTextContent(msg)
|
||||
}
|
||||
if payload == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal([]byte(payload), &envelope); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
raw, ok := envelope["_ARTIFACTS"].([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]artifactEntry, 0, len(raw))
|
||||
for _, item := range raw {
|
||||
m, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, _ := m["name"].(string)
|
||||
url, _ := m["url"].(string)
|
||||
if name == "" || url == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, artifactEntry{Name: name, URL: url})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// toolMessageTextContent returns the first text content part of a tool
|
||||
// message, or an empty string if no text part is found.
|
||||
func toolMessageTextContent(msg *schema.Message) string {
|
||||
for i := range msg.UserInputMultiContent {
|
||||
part := &msg.UserInputMultiContent[i]
|
||||
if part.Type == schema.ChatMessagePartTypeText && part.Text != "" {
|
||||
return part.Text
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// formatArtifactMarkdown renders a slice of artifacts as markdown
|
||||
// links, omitting URLs already present in the existing text (Python's
|
||||
// `_collect_tool_artifact_markdown` does the same de-duplication).
|
||||
|
||||
Reference in New Issue
Block a user