diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.go index 60542569e5..f3e707cc3c 100644 --- a/internal/agent/component/agent.go +++ b/internal/agent/component/agent.go @@ -276,19 +276,49 @@ func scanAllStreamForToolCall(_ context.Context, stream *schema.StreamReader[*sc // buildAgentInputMessages assembles the Python-compatible Agent prompt: the // configured history window followed by the current user prompt. The current // in-flight user entry is excluded through SnapshotPriorHistory, because the -// canvas service appends it to state before invoking the workflow. +// canvas service appends it to state before invoking the workflow. Uploaded +// files from sys.files are folded into that user prompt (file texts merged, +// images attached as multi-modal content parts). func buildAgentInputMessages(ctx context.Context, p AgentParam) []*schema.Message { - current := schema.Message{Role: schema.User, Content: p.UserPrompt} - messages := []schema.Message{} - if p.MessageHistoryWindowSize > 0 { - if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil { - // Python takes the last 2*N entries from history, which already - // contains the current user input, and then removes that final - // entry before formatting the configured prompt. - priorLimit := p.MessageHistoryWindowSize*2 - 1 - messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit) + var state *runtime.CanvasState + if s, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && s != nil { + state = s + } + // Inject sys.files uploads into the current user message, mirroring + // the LLM component (llm.go) and Python's Agent._prepare_prompt_variables + // delegation to LLMBundle. Uploaded files land in state.Sys["files"] + // (service/agent.go) as data:image URIs / parsed text; without this + // step a vision agent never sees the attached image. File texts merge + // into the user prompt; images become multi-modal content parts. + // The {sys.files} placeholder, when present, has already been resolved + // by ResolveTemplate upstream in invokeNow, so injection here is + // unconditional — same effective behavior as the LLM component. + userText := p.UserPrompt + var images []string + if state != nil { + var texts []string + texts, images = collectSysFiles(state) + if len(texts) > 0 { + joined := strings.Join(texts, "\n\n") + if userText != "" { + userText += "\n\n" + joined + } else { + userText = joined + } } } + current := schema.Message{Role: schema.User, Content: userText} + if len(images) > 0 { + current = userMessageWithImages(userText, images) + } + messages := []schema.Message{} + if p.MessageHistoryWindowSize > 0 && state != nil { + // Python takes the last 2*N entries from history, which already + // contains the current user input, and then removes that final + // entry before formatting the configured prompt. + priorLimit := p.MessageHistoryWindowSize*2 - 1 + messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit) + } if len(messages) > 0 && messages[len(messages)-1].Role == current.Role { messages[len(messages)-1] = current } else { diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go index f06a43a8b4..7f5ad882be 100644 --- a/internal/agent/component/llm.go +++ b/internal/agent/component/llm.go @@ -887,6 +887,15 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) [] return out } + out = append(out, userMessageWithImages(user, images)) + return out +} + +// userMessageWithImages builds a user message carrying the text plus the +// given data-image URIs as eino multi-modal content parts. Shared by the +// LLM component (buildMessagesWithImages) and the Agent component +// (buildAgentInputMessages) so both produce the exact same part shape. +func userMessageWithImages(user string, images []string) schema.Message { parts := make([]schema.MessageInputPart, 0, 1+len(images)) if user != "" { parts = append(parts, schema.MessageInputPart{ @@ -903,11 +912,10 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) [] }, }) } - out = append(out, schema.Message{ + return schema.Message{ Role: schema.User, UserInputMultiContent: parts, - }) - return out + } } // mergeLLMParam layers raw inputs over the receiver's default param set. diff --git a/internal/agent/component/vision_test.go b/internal/agent/component/vision_test.go index c0ba352084..02d43bcd4c 100644 --- a/internal/agent/component/vision_test.go +++ b/internal/agent/component/vision_test.go @@ -373,3 +373,58 @@ func TestLLM_Invoke_VisualFilesAsString(t *testing.T) { user.UserInputMultiContent[1]) } } + +// TestBuildAgentInputMessagesInjectsSysFiles guards the agent-side half of +// the upload fix: buildAgentInputMessages must fold sys.files uploads into +// the current user message (file text merged, images as multi-modal parts). +// Without it a vision agent (e.g. qwen3-vl-plus) reports it cannot see the +// attached image. +func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) { + uri := "data:image/png;base64,iVBORw0KGgo=" + state := runtime.NewCanvasState("run-agent-files", "task-agent-files") + state.Sys["files"] = []string{"parsed document text", uri} + ctx := runtime.WithState(t.Context(), state) + + messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "描述图片内容"}) + if len(messages) != 1 { + t.Fatalf("message count = %d, want 1", len(messages)) + } + msg := messages[0] + if msg.Role != schema.User { + t.Fatalf("role = %v, want user", msg.Role) + } + if len(msg.UserInputMultiContent) != 2 { + t.Fatalf("parts = %d, want 2 (text + image)", len(msg.UserInputMultiContent)) + } + textPart := msg.UserInputMultiContent[0] + if textPart.Type != schema.ChatMessagePartTypeText || + textPart.Text != "描述图片内容\n\nparsed document text" { + t.Errorf("text part = %+v, want merged prompt+file text", textPart) + } + imagePart := msg.UserInputMultiContent[1] + if imagePart.Type != schema.ChatMessagePartTypeImageURL || + imagePart.Image == nil || imagePart.Image.URL == nil || *imagePart.Image.URL != uri { + t.Errorf("image part = %+v, want image_url part carrying the upload URI", imagePart) + } +} + +// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into +// the plain string Content without creating multi-modal parts. +func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) { + state := runtime.NewCanvasState("run-agent-text", "task-agent-text") + state.Sys["files"] = []string{"first file text", "second file text"} + ctx := runtime.WithState(t.Context(), state) + + messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "summarize"}) + if len(messages) != 1 { + t.Fatalf("message count = %d, want 1", len(messages)) + } + want := "summarize\n\nfirst file text\n\nsecond file text" + if messages[0].Content != want { + t.Errorf("Content = %q, want %q", messages[0].Content, want) + } + if len(messages[0].UserInputMultiContent) != 0 { + t.Errorf("text-only files must not create multi-modal parts, got %d", + len(messages[0].UserInputMultiContent)) + } +} diff --git a/internal/entity/models/llm.go b/internal/entity/models/llm.go index 13bd54cfb9..d3cdecf3ea 100644 --- a/internal/entity/models/llm.go +++ b/internal/entity/models/llm.go @@ -91,6 +91,11 @@ func toInternalMessages(msgs []*schema.Message) []Message { role = "user" } msg := Message{Role: role, Content: mm.Content} + if len(mm.UserInputMultiContent) > 0 { + if blocks := openAIContentBlocksFromEino(mm.UserInputMultiContent); len(blocks) > 0 { + msg.Content = blocks + } + } if len(mm.ToolCalls) > 0 { msg.ToolCalls = toolCallsToInternal(mm.ToolCalls) } @@ -102,6 +107,62 @@ func toInternalMessages(msgs []*schema.Message) []Message { return out } +// openAIContentBlocksFromEino converts eino multi-modal input parts into +// OpenAI-style content blocks ("text" / "image_url"). Message.Content is +// interface{} and every driver already understands this block shape: the +// generic OpenAI-compatible request builder marshals it verbatim +// (buildChatMessages in base_model.go), while the native anthropic / +// google converters type-switch on []interface{} (anthropicContent / +// googleMessageParts). The slice MUST therefore be []interface{}, not +// []map[string]interface{}, or googleMessageParts misses it. Unsupported +// part types are skipped; a nil return tells the caller to fall back to +// the plain string Content. +func openAIContentBlocksFromEino(parts []schema.MessageInputPart) []interface{} { + blocks := make([]interface{}, 0, len(parts)) + for _, part := range parts { + switch part.Type { + case schema.ChatMessagePartTypeText: + if part.Text == "" { + continue + } + blocks = append(blocks, map[string]interface{}{"type": "text", "text": part.Text}) + case schema.ChatMessagePartTypeImageURL: + url := einoImagePartURL(part.Image) + if url == "" { + continue + } + blocks = append(blocks, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{"url": url}, + }) + } + } + if len(blocks) == 0 { + return nil + } + return blocks +} + +// einoImagePartURL resolves an image part to a single URL string: either +// the direct URL (the agent component carries data URIs this way) or a +// reassembled data URI from Base64Data + MIMEType. +func einoImagePartURL(img *schema.MessageInputImage) string { + if img == nil { + return "" + } + if img.URL != nil && *img.URL != "" { + return *img.URL + } + if img.Base64Data != nil && *img.Base64Data != "" { + mime := img.MIMEType + if mime == "" { + mime = "image/png" + } + return "data:" + mime + ";base64," + *img.Base64Data + } + return "" +} + // fromInternalResponse converts a *ChatResponse to *schema.Message. The // existing ChatResponse only carries answer text (+ optional reasoning), so // the resulting Message has Role=Assistant and Content=answer. diff --git a/internal/entity/models/llm_test.go b/internal/entity/models/llm_test.go index 7e350c1226..723831c4b4 100644 --- a/internal/entity/models/llm_test.go +++ b/internal/entity/models/llm_test.go @@ -301,3 +301,95 @@ func (d *captureToolDriver) ListTasks(ctx context.Context, _ *APIConfig) ([]List func (d *captureToolDriver) ShowTask(ctx context.Context, _ string, _ *APIConfig) (*TaskResponse, error) { return nil, nil } + +// TestToInternalMessagesConvertsMultiModalContent guards the eino→driver +// boundary: UserInputMultiContent must become OpenAI-style content blocks +// ([]interface{} of {type:text} / {type:image_url}) on Message.Content, +// otherwise image parts produced by the component layer are silently +// dropped before the request reaches any driver. +func TestToInternalMessagesConvertsMultiModalContent(t *testing.T) { + uri := "data:image/png;base64,iVBORw0KGgo=" + internal := toInternalMessages([]*schema.Message{ + { + Role: schema.User, + UserInputMultiContent: []schema.MessageInputPart{ + {Type: schema.ChatMessagePartTypeText, Text: "describe the image"}, + {Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{ + MessagePartCommon: schema.MessagePartCommon{URL: &uri}, + }}, + }, + }, + }) + if len(internal) != 1 { + t.Fatalf("len(internal) = %d, want 1", len(internal)) + } + blocks, ok := internal[0].Content.([]interface{}) + if !ok { + t.Fatalf("Content type = %T, want []interface{} content blocks", internal[0].Content) + } + if len(blocks) != 2 { + t.Fatalf("len(blocks) = %d, want 2", len(blocks)) + } + textBlock, ok := blocks[0].(map[string]interface{}) + if !ok || textBlock["type"] != "text" || textBlock["text"] != "describe the image" { + t.Fatalf("text block = %#v, want {type:text, text:describe the image}", blocks[0]) + } + imageBlock, ok := blocks[1].(map[string]interface{}) + if !ok || imageBlock["type"] != "image_url" { + t.Fatalf("image block = %#v, want type image_url", blocks[1]) + } + imageURL, ok := imageBlock["image_url"].(map[string]interface{}) + if !ok || imageURL["url"] != uri { + t.Fatalf("image_url = %#v, want url %q", imageBlock["image_url"], uri) + } +} + +// TestToInternalMessagesReassemblesBase64Image: parts that carry Base64Data +// instead of a URL are reassembled into a data URI. +func TestToInternalMessagesReassemblesBase64Image(t *testing.T) { + b64 := "aGVsbG8=" + internal := toInternalMessages([]*schema.Message{ + { + Role: schema.User, + UserInputMultiContent: []schema.MessageInputPart{ + {Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{ + MessagePartCommon: schema.MessagePartCommon{ + Base64Data: &b64, + MIMEType: "image/jpeg", + }, + }}, + }, + }, + }) + blocks, ok := internal[0].Content.([]interface{}) + if !ok || len(blocks) != 1 { + t.Fatalf("Content = %#v, want one content block", internal[0].Content) + } + imageBlock, ok := blocks[0].(map[string]interface{}) + if !ok { + t.Fatalf("block = %#v, want map", blocks[0]) + } + imageURL, ok := imageBlock["image_url"].(map[string]interface{}) + if !ok || imageURL["url"] != "data:image/jpeg;base64,aGVsbG8=" { + t.Fatalf("image_url = %#v, want reassembled data URI", imageBlock["image_url"]) + } +} + +// TestToInternalMessagesUnsupportedPartsFallBackToString: when every part is +// of an unsupported type, Content stays the plain string. +func TestToInternalMessagesUnsupportedPartsFallBackToString(t *testing.T) { + internal := toInternalMessages([]*schema.Message{ + { + Role: schema.User, + Content: "plain", + UserInputMultiContent: []schema.MessageInputPart{ + {Type: schema.ChatMessagePartTypeAudioURL}, + }, + }, + }) + if content, ok := internal[0].Content.(string); !ok || content != "plain" { + t.Fatalf("Content = %#v, want string %q", internal[0].Content, "plain") + } +}