fix(agent): collect CodeExec artifacts from ReAct tool responses (#16609)

### Summary

The Go backend Agent component was not returning artifacts produced by
the CodeExec tool. While the Python agent collects the "`_ARTIFACTS`"
envelope from tool responses and appends artifact markdown to the final
content, the Go agent only returned the assistant text, so generated
images were missing from the chat output.

### Changes

- Wire `react.WithMessageFuture()` in `runEinoReActAgent` and store the
resulting `MessageFuture` in the invocation context.
- After the ReAct loop finishes, drain the future and extract
``_ARTIFACTS`` entries from every tool response message.
- Support reading the tool payload from both `msg.Content` and
`msg.UserInputMultiContent` to match eino's tool contract.
- De-duplicate artifacts by URL and render images as `!` and other files
as download links.
- Add `agent_artifact_test.go` with a regression test that simulates a
CodeExec-style tool response carrying an image artifact and verifies it
is collected and formatted.

### Verification

- `go test ./internal/agent/component/... -run
TestAgent_ReActAgent_CollectsArtifactsFromCodeExecTool` passes.
- `go test ./internal/agent/component/... -count=1` compiles; the only
failure is an unrelated DNS-pinning timeout test
(`TestInvoke_ProxyDNSPin`).
- `gofmt` clean for modified files.

### Related

Fixes the behavior shown in the screenshot where the Go agent ignored
the CodeExec-generated PNG artifact.
This commit is contained in:
euvre
2026-07-05 20:53:43 +08:00
committed by GitHub
parent 1d3c100acb
commit 8b065d3ddd
2 changed files with 381 additions and 17 deletions
+132 -17
View File
@@ -13,6 +13,7 @@ package component
import (
"context"
"encoding/json"
"fmt"
"strings"
@@ -160,7 +161,13 @@ func runEinoReActAgent(ctx context.Context, p AgentParam) (*schema.Message, erro
}
input := []*schema.Message{schema.UserMessage(p.UserPrompt)}
return agent.Generate(ctx, input)
opt, future := react.WithMessageFuture()
ctx = setArtifactCollector(ctx, future)
msg, err := agent.Generate(ctx, input, opt)
if err != nil {
return nil, err
}
return msg, nil
}
// addToolCallMemory summarizes the tool calls observed in msg via
@@ -530,7 +537,7 @@ func (c *AgentComponent) Invoke(ctx context.Context, inputs map[string]any) (map
}
}
}
artifacts := collectArtifactsFromToolCalls(msg)
artifacts := collectArtifactsFromToolCalls(ctx, msg)
artifactMD := formatArtifactMarkdown(artifacts, content)
out := map[string]any{
"content": content + artifactMD,
@@ -653,24 +660,132 @@ type artifactEntry struct {
URL string `json:"url"`
}
// collectArtifactsFromToolCalls extracts artifact entries from a
// ReAct final message. Today, eino's react.Agent.Generate returns only
// the final message — the tool-response state lives inside the agent
// runtime and is not directly accessible. So this v1 returns an empty
// slice; the wiring lives in a follow-up that hoists the eino state
// into a place AgentComponent can read.
//
// When the tool response state becomes accessible (a future phase),
// the entry point to wire it is here: scan the eino conversation
// messages for entries whose `Extra["_ARTIFACTS"]` carries the
// per-tool artifact metadata, decode the JSON, and append to the
// returned slice. The shape expected from each tool is:
//
// { "name": "report.pdf", "url": "https://..." }
func collectArtifactsFromToolCalls(_ *schema.Message) []artifactEntry {
// artifactCollectorKey is the context key used to stash the
// MessageFuture from react.WithMessageFuture() so the AgentComponent
// can collect artifacts after the ReAct loop finishes. The collector
// is created per-invocation in runEinoReActAgent.
type artifactCollectorKey struct{}
// setArtifactCollector registers the MessageFuture for this agent run
// in the context. It is called from runEinoReActAgent after
// react.WithMessageFuture() returns a future.
func setArtifactCollector(ctx context.Context, future react.MessageFuture) context.Context {
return context.WithValue(ctx, artifactCollectorKey{}, future)
}
// getArtifactCollector retrieves the MessageFuture registered for the
// current agent run. Returns nil when no collector was registered
// (e.g., tests that stub agentRunner).
func getArtifactCollector(ctx context.Context) react.MessageFuture {
v := ctx.Value(artifactCollectorKey{})
if v == nil {
return nil
}
if f, ok := v.(react.MessageFuture); ok {
return f
}
return nil
}
// collectArtifactsFromToolCalls drains the MessageFuture stored in
// ctx (if any) and extracts artifact entries from every tool response
// message that carries a `_ARTIFACTS` payload in its Extra field.
// The final message is ignored because it is an assistant message and
// does not contain tool results. Returns a de-duplicated list ordered
// by first appearance.
//
// The expected payload shape in each tool response is:
//
// { "_ARTIFACTS": [{ "name": "report.pdf", "url": "https://..." }, ...] }
func collectArtifactsFromToolCalls(ctx context.Context, _ *schema.Message) []artifactEntry {
future := getArtifactCollector(ctx)
if future == nil {
return nil
}
seen := make(map[string]struct{})
var out []artifactEntry
iter := future.GetMessages()
for {
msg, ok, err := iter.Next()
if err != nil {
common.Debug("agent: artifact collection iterator error", zap.Error(err))
break
}
if !ok {
break
}
if msg == nil || msg.Role != schema.Tool {
continue
}
rawArtifacts := extractArtifactsFromToolMessage(msg)
for _, a := range rawArtifacts {
if a.URL == "" || a.Name == "" {
continue
}
if _, exists := seen[a.URL]; exists {
continue
}
seen[a.URL] = struct{}{}
out = append(out, a)
}
}
return out
}
// extractArtifactsFromToolMessage parses the JSON payload of a tool
// response message and returns the `_ARTIFACTS` list. The payload is
// read from msg.Content when it is non-empty; otherwise the first text
// element of msg.UserInputMultiContent is used. This matches the eino
// tool contract where tool results are delivered as a string.
func extractArtifactsFromToolMessage(msg *schema.Message) []artifactEntry {
payload := msg.Content
if payload == "" && len(msg.UserInputMultiContent) > 0 {
payload = toolMessageTextContent(msg)
}
if payload == "" {
return nil
}
var envelope map[string]any
if err := json.Unmarshal([]byte(payload), &envelope); err != nil {
return nil
}
raw, ok := envelope["_ARTIFACTS"].([]any)
if !ok {
return nil
}
out := make([]artifactEntry, 0, len(raw))
for _, item := range raw {
m, ok := item.(map[string]any)
if !ok {
continue
}
name, _ := m["name"].(string)
url, _ := m["url"].(string)
if name == "" || url == "" {
continue
}
out = append(out, artifactEntry{Name: name, URL: url})
}
return out
}
// toolMessageTextContent returns the first text content part of a tool
// message, or an empty string if no text part is found.
func toolMessageTextContent(msg *schema.Message) string {
for i := range msg.UserInputMultiContent {
part := &msg.UserInputMultiContent[i]
if part.Type == schema.ChatMessagePartTypeText && part.Text != "" {
return part.Text
}
}
return ""
}
// formatArtifactMarkdown renders a slice of artifacts as markdown
// links, omitting URLs already present in the existing text (Python's
// `_collect_tool_artifact_markdown` does the same de-duplication).