fix(go-agent): forward sys.files uploads to vision LLMs (#18246)

internal/agent/component/agent.go

@@ -276,19 +276,49 @@ func scanAllStreamForToolCall(_ context.Context, stream *schema.StreamReader[*sc
 // buildAgentInputMessages assembles the Python-compatible Agent prompt: the
 // configured history window followed by the current user prompt. The current
 // in-flight user entry is excluded through SnapshotPriorHistory, because the
-// canvas service appends it to state before invoking the workflow.
+// canvas service appends it to state before invoking the workflow. Uploaded
+// files from sys.files are folded into that user prompt (file texts merged,
+// images attached as multi-modal content parts).
 func buildAgentInputMessages(ctx context.Context, p AgentParam) []*schema.Message {
-	current := schema.Message{Role: schema.User, Content: p.UserPrompt}
-	messages := []schema.Message{}
-	if p.MessageHistoryWindowSize > 0 {
-		if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {
-			// Python takes the last 2*N entries from history, which already
-			// contains the current user input, and then removes that final
-			// entry before formatting the configured prompt.
-			priorLimit := p.MessageHistoryWindowSize*2 - 1
-			messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)
+	var state *runtime.CanvasState
+	if s, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && s != nil {
+		state = s
+	}
+	// Inject sys.files uploads into the current user message, mirroring
+	// the LLM component (llm.go) and Python's Agent._prepare_prompt_variables
+	// delegation to LLMBundle. Uploaded files land in state.Sys["files"]
+	// (service/agent.go) as data:image URIs / parsed text; without this
+	// step a vision agent never sees the attached image. File texts merge
+	// into the user prompt; images become multi-modal content parts.
+	// The {sys.files} placeholder, when present, has already been resolved
+	// by ResolveTemplate upstream in invokeNow, so injection here is
+	// unconditional — same effective behavior as the LLM component.
+	userText := p.UserPrompt
+	var images []string
+	if state != nil {
+		var texts []string
+		texts, images = collectSysFiles(state)
+		if len(texts) > 0 {
+			joined := strings.Join(texts, "\n\n")
+			if userText != "" {
+				userText += "\n\n" + joined
+			} else {
+				userText = joined
+			}
 		}
 	}
+	current := schema.Message{Role: schema.User, Content: userText}
+	if len(images) > 0 {
+		current = userMessageWithImages(userText, images)
+	}
+	messages := []schema.Message{}
+	if p.MessageHistoryWindowSize > 0 && state != nil {
+		// Python takes the last 2*N entries from history, which already
+		// contains the current user input, and then removes that final
+		// entry before formatting the configured prompt.
+		priorLimit := p.MessageHistoryWindowSize*2 - 1
+		messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)
+	}
 	if len(messages) > 0 && messages[len(messages)-1].Role == current.Role {
 		messages[len(messages)-1] = current
 	} else {

internal/agent/component/llm.go

@@ -887,6 +887,15 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []
 		return out
 	}
 
+	out = append(out, userMessageWithImages(user, images))
+	return out
+}
+
+// userMessageWithImages builds a user message carrying the text plus the
+// given data-image URIs as eino multi-modal content parts. Shared by the
+// LLM component (buildMessagesWithImages) and the Agent component
+// (buildAgentInputMessages) so both produce the exact same part shape.
+func userMessageWithImages(user string, images []string) schema.Message {
 	parts := make([]schema.MessageInputPart, 0, 1+len(images))
 	if user != "" {
 		parts = append(parts, schema.MessageInputPart{
@@ -903,11 +912,10 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []
 			},
 		})
 	}
-	out = append(out, schema.Message{
+	return schema.Message{
 		Role:                  schema.User,
 		UserInputMultiContent: parts,
-	})
-	return out
+	}
 }
 
 // mergeLLMParam layers raw inputs over the receiver's default param set.

internal/agent/component/vision_test.go

@@ -373,3 +373,58 @@ func TestLLM_Invoke_VisualFilesAsString(t *testing.T) {
 			user.UserInputMultiContent[1])
 	}
 }
+
+// TestBuildAgentInputMessagesInjectsSysFiles guards the agent-side half of
+// the upload fix: buildAgentInputMessages must fold sys.files uploads into
+// the current user message (file text merged, images as multi-modal parts).
+// Without it a vision agent (e.g. qwen3-vl-plus) reports it cannot see the
+// attached image.
+func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) {
+	uri := "data:image/png;base64,iVBORw0KGgo="
+	state := runtime.NewCanvasState("run-agent-files", "task-agent-files")
+	state.Sys["files"] = []string{"parsed document text", uri}
+	ctx := runtime.WithState(t.Context(), state)
+
+	messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "描述图片内容"})
+	if len(messages) != 1 {
+		t.Fatalf("message count = %d, want 1", len(messages))
+	}
+	msg := messages[0]
+	if msg.Role != schema.User {
+		t.Fatalf("role = %v, want user", msg.Role)
+	}
+	if len(msg.UserInputMultiContent) != 2 {
+		t.Fatalf("parts = %d, want 2 (text + image)", len(msg.UserInputMultiContent))
+	}
+	textPart := msg.UserInputMultiContent[0]
+	if textPart.Type != schema.ChatMessagePartTypeText ||
+		textPart.Text != "描述图片内容\n\nparsed document text" {
+		t.Errorf("text part = %+v, want merged prompt+file text", textPart)
+	}
+	imagePart := msg.UserInputMultiContent[1]
+	if imagePart.Type != schema.ChatMessagePartTypeImageURL ||
+		imagePart.Image == nil || imagePart.Image.URL == nil || *imagePart.Image.URL != uri {
+		t.Errorf("image part = %+v, want image_url part carrying the upload URI", imagePart)
+	}
+}
+
+// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into
+// the plain string Content without creating multi-modal parts.
+func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) {
+	state := runtime.NewCanvasState("run-agent-text", "task-agent-text")
+	state.Sys["files"] = []string{"first file text", "second file text"}
+	ctx := runtime.WithState(t.Context(), state)
+
+	messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "summarize"})
+	if len(messages) != 1 {
+		t.Fatalf("message count = %d, want 1", len(messages))
+	}
+	want := "summarize\n\nfirst file text\n\nsecond file text"
+	if messages[0].Content != want {
+		t.Errorf("Content = %q, want %q", messages[0].Content, want)
+	}
+	if len(messages[0].UserInputMultiContent) != 0 {
+		t.Errorf("text-only files must not create multi-modal parts, got %d",
+			len(messages[0].UserInputMultiContent))
+	}
+}

internal/entity/models/llm.go

@@ -91,6 +91,11 @@ func toInternalMessages(msgs []*schema.Message) []Message {
 			role = "user"
 		}
 		msg := Message{Role: role, Content: mm.Content}
+		if len(mm.UserInputMultiContent) > 0 {
+			if blocks := openAIContentBlocksFromEino(mm.UserInputMultiContent); len(blocks) > 0 {
+				msg.Content = blocks
+			}
+		}
 		if len(mm.ToolCalls) > 0 {
 			msg.ToolCalls = toolCallsToInternal(mm.ToolCalls)
 		}
@@ -102,6 +107,62 @@ func toInternalMessages(msgs []*schema.Message) []Message {
 	return out
 }
 
+// openAIContentBlocksFromEino converts eino multi-modal input parts into
+// OpenAI-style content blocks ("text" / "image_url"). Message.Content is
+// interface{} and every driver already understands this block shape: the
+// generic OpenAI-compatible request builder marshals it verbatim
+// (buildChatMessages in base_model.go), while the native anthropic /
+// google converters type-switch on []interface{} (anthropicContent /
+// googleMessageParts). The slice MUST therefore be []interface{}, not
+// []map[string]interface{}, or googleMessageParts misses it. Unsupported
+// part types are skipped; a nil return tells the caller to fall back to
+// the plain string Content.
+func openAIContentBlocksFromEino(parts []schema.MessageInputPart) []interface{} {
+	blocks := make([]interface{}, 0, len(parts))
+	for _, part := range parts {
+		switch part.Type {
+		case schema.ChatMessagePartTypeText:
+			if part.Text == "" {
+				continue
+			}
+			blocks = append(blocks, map[string]interface{}{"type": "text", "text": part.Text})
+		case schema.ChatMessagePartTypeImageURL:
+			url := einoImagePartURL(part.Image)
+			if url == "" {
+				continue
+			}
+			blocks = append(blocks, map[string]interface{}{
+				"type":      "image_url",
+				"image_url": map[string]interface{}{"url": url},
+			})
+		}
+	}
+	if len(blocks) == 0 {
+		return nil
+	}
+	return blocks
+}
+
+// einoImagePartURL resolves an image part to a single URL string: either
+// the direct URL (the agent component carries data URIs this way) or a
+// reassembled data URI from Base64Data + MIMEType.
+func einoImagePartURL(img *schema.MessageInputImage) string {
+	if img == nil {
+		return ""
+	}
+	if img.URL != nil && *img.URL != "" {
+		return *img.URL
+	}
+	if img.Base64Data != nil && *img.Base64Data != "" {
+		mime := img.MIMEType
+		if mime == "" {
+			mime = "image/png"
+		}
+		return "data:" + mime + ";base64," + *img.Base64Data
+	}
+	return ""
+}
+
 // fromInternalResponse converts a *ChatResponse to *schema.Message. The
 // existing ChatResponse only carries answer text (+ optional reasoning), so
 // the resulting Message has Role=Assistant and Content=answer.

internal/entity/models/llm_test.go

@@ -301,3 +301,95 @@ func (d *captureToolDriver) ListTasks(ctx context.Context, _ *APIConfig) ([]List
 func (d *captureToolDriver) ShowTask(ctx context.Context, _ string, _ *APIConfig) (*TaskResponse, error) {
 	return nil, nil
 }
+
+// TestToInternalMessagesConvertsMultiModalContent guards the eino→driver
+// boundary: UserInputMultiContent must become OpenAI-style content blocks
+// ([]interface{} of {type:text} / {type:image_url}) on Message.Content,
+// otherwise image parts produced by the component layer are silently
+// dropped before the request reaches any driver.
+func TestToInternalMessagesConvertsMultiModalContent(t *testing.T) {
+	uri := "data:image/png;base64,iVBORw0KGgo="
+	internal := toInternalMessages([]*schema.Message{
+		{
+			Role: schema.User,
+			UserInputMultiContent: []schema.MessageInputPart{
+				{Type: schema.ChatMessagePartTypeText, Text: "describe the image"},
+				{Type: schema.ChatMessagePartTypeImageURL,
+					Image: &schema.MessageInputImage{
+						MessagePartCommon: schema.MessagePartCommon{URL: &uri},
+					}},
+			},
+		},
+	})
+	if len(internal) != 1 {
+		t.Fatalf("len(internal) = %d, want 1", len(internal))
+	}
+	blocks, ok := internal[0].Content.([]interface{})
+	if !ok {
+		t.Fatalf("Content type = %T, want []interface{} content blocks", internal[0].Content)
+	}
+	if len(blocks) != 2 {
+		t.Fatalf("len(blocks) = %d, want 2", len(blocks))
+	}
+	textBlock, ok := blocks[0].(map[string]interface{})
+	if !ok || textBlock["type"] != "text" || textBlock["text"] != "describe the image" {
+		t.Fatalf("text block = %#v, want {type:text, text:describe the image}", blocks[0])
+	}
+	imageBlock, ok := blocks[1].(map[string]interface{})
+	if !ok || imageBlock["type"] != "image_url" {
+		t.Fatalf("image block = %#v, want type image_url", blocks[1])
+	}
+	imageURL, ok := imageBlock["image_url"].(map[string]interface{})
+	if !ok || imageURL["url"] != uri {
+		t.Fatalf("image_url = %#v, want url %q", imageBlock["image_url"], uri)
+	}
+}
+
+// TestToInternalMessagesReassemblesBase64Image: parts that carry Base64Data
+// instead of a URL are reassembled into a data URI.
+func TestToInternalMessagesReassemblesBase64Image(t *testing.T) {
+	b64 := "aGVsbG8="
+	internal := toInternalMessages([]*schema.Message{
+		{
+			Role: schema.User,
+			UserInputMultiContent: []schema.MessageInputPart{
+				{Type: schema.ChatMessagePartTypeImageURL,
+					Image: &schema.MessageInputImage{
+						MessagePartCommon: schema.MessagePartCommon{
+							Base64Data: &b64,
+							MIMEType:   "image/jpeg",
+						},
+					}},
+			},
+		},
+	})
+	blocks, ok := internal[0].Content.([]interface{})
+	if !ok || len(blocks) != 1 {
+		t.Fatalf("Content = %#v, want one content block", internal[0].Content)
+	}
+	imageBlock, ok := blocks[0].(map[string]interface{})
+	if !ok {
+		t.Fatalf("block = %#v, want map", blocks[0])
+	}
+	imageURL, ok := imageBlock["image_url"].(map[string]interface{})
+	if !ok || imageURL["url"] != "data:image/jpeg;base64,aGVsbG8=" {
+		t.Fatalf("image_url = %#v, want reassembled data URI", imageBlock["image_url"])
+	}
+}
+
+// TestToInternalMessagesUnsupportedPartsFallBackToString: when every part is
+// of an unsupported type, Content stays the plain string.
+func TestToInternalMessagesUnsupportedPartsFallBackToString(t *testing.T) {
+	internal := toInternalMessages([]*schema.Message{
+		{
+			Role:    schema.User,
+			Content: "plain",
+			UserInputMultiContent: []schema.MessageInputPart{
+				{Type: schema.ChatMessagePartTypeAudioURL},
+			},
+		},
+	})
+	if content, ok := internal[0].Content.(string); !ok || content != "plain" {
+		t.Fatalf("Content = %#v, want string %q", internal[0].Content, "plain")
+	}
+}