ragflow/internal/common/multimodal.go

//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package common

import (
	"encoding/base64"
	"fmt"
	"regexp"
	"strings"
)

// ContentPart is the internal representation of a multimodal content
// fragment, decoupled from any provider's wire format. Drivers consume
// the result of RenderContentPartsForFactory to produce their per-
// provider JSON.
type ContentPart struct {
	// Type is one of: "text", "image_url", "image", "inline_data".
	Type string
	// Text is set when Type == "text".
	Text string
	// ImageURL is set when Type == "image_url" (OpenAI shape).
	ImageURL *ImageURL
	// Source is set when Type == "image" (Anthropic) or
	// Type == "inline_data" (Gemini).
	Source *ContentSource
}

// ImageURL is the OpenAI-shaped image reference.
type ImageURL struct {
	URL string `json:"url"`
}

// ContentSource is the Anthropic / Gemini source payload.
type ContentSource struct {
	Type      string `json:"type"`           // "base64" or "url"
	MediaType string `json:"media_type"`     // e.g. "image/png"
	Data      string `json:"data,omitempty"` // base64 payload
	URL       string `json:"url,omitempty"`
}

// dataURIRE detects a "data:<mediatype>;base64,<data>" string.
var dataURIRE = regexp.MustCompile(`^data:([^;,]+)(?:;base64)?,(.*)$`)

// parseDataURIOrB64 accepts a string and classifies it as a data URI,
// a plain https URL, or a raw base64 payload
func parseDataURIOrB64(s string) (ContentSource, error) {
	if s == "" {
		return ContentSource{}, fmt.Errorf("empty image source")
	}
	if m := dataURIRE.FindStringSubmatch(s); m != nil {
		mediaType := strings.TrimSpace(m[1])
		if mediaType == "" {
			mediaType = "image/png"
		}
		return ContentSource{
			Type:      "base64",
			MediaType: mediaType,
			Data:      m[2],
		}, nil
	}
	if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
		return ContentSource{Type: "url", URL: s}, nil
	}
	// Assume raw base64 (no data URI, no http scheme). The provider
	// uses the file extension or a content-type hint from the call site
	// to pick the right media type; we default to image/png.
	if _, err := base64.StdEncoding.DecodeString(s); err != nil {
		return ContentSource{}, fmt.Errorf("not a valid data URI, URL, or base64: %w", err)
	}
	return ContentSource{
		Type:      "base64",
		MediaType: "image/png",
		Data:      s,
	}, nil
}

// normalizeTextFromContent extracts a single text string from a content
// value that may be a string, []map[string]interface{}, or []interface{}.
func normalizeTextFromContent(content interface{}) string {
	switch v := content.(type) {
	case string:
		return v
	case []map[string]interface{}:
		var parts []string
		for _, p := range v {
			if t, ok := p["type"].(string); ok && (t == "text" || t == "input_text") {
				if txt, ok := p["text"].(string); ok {
					parts = append(parts, txt)
				}
			} else if txt, ok := p["text"]; ok {
				// Fallback: "text" key present even though type didn't match.
				switch tv := txt.(type) {
				case string:
					parts = append(parts, tv)
				case float64:
					parts = append(parts, fmt.Sprintf("%v", tv))
				case int:
					parts = append(parts, fmt.Sprintf("%v", tv))
				}
			}
		}
		return strings.Join(parts, "\n")
	case []interface{}:
		var parts []string
		for _, item := range v {
			switch p := item.(type) {
			case map[string]interface{}:
				if t, ok := p["type"].(string); ok && (t == "text" || t == "input_text") {
					if txt, ok := p["text"].(string); ok {
						parts = append(parts, txt)
					}
				} else if txt, ok := p["text"]; ok {
					// Fallback: "text" key present even though type didn't match.
					switch tv := txt.(type) {
					case string:
						parts = append(parts, tv)
					case float64:
						parts = append(parts, fmt.Sprintf("%v", tv))
					case int:
						parts = append(parts, fmt.Sprintf("%v", tv))
					}
				}
			case string:
				parts = append(parts, p)
			}
		}
		return strings.Join(parts, "\n")
	}
	return ""
}

// extractImageURLs pulls image_url values out of a content value. Used
// by ConvertLastUserMsgToMultimodal to assemble the ContentPart slice.
func extractImageURLs(content interface{}) []string {
	var urls []string
	process := func(p map[string]interface{}) {
		t, _ := p["type"].(string)
		if t == "image_url" {
			if u, ok := p["image_url"].(string); ok && u != "" {
				urls = append(urls, u)
			} else if obj, ok := p["image_url"].(map[string]interface{}); ok {
				if u, ok := obj["url"].(string); ok && u != "" {
					urls = append(urls, u)
				}
			}
		}
	}
	switch v := content.(type) {
	case []map[string]interface{}:
		for _, p := range v {
			process(p)
		}
	case []interface{}:
		for _, item := range v {
			if p, ok := item.(map[string]interface{}); ok {
				process(p)
			}
		}
	}
	return urls
}

// ConvertLastUserMsgToMultimodal converts a user message whose content
// is a multimodal parts array into a message whose content is a
// driver-ready content-parts value, dispatched by `factory` (provider
// name).
//
// `imageAttachments` is an additional list of image URLs from the
// `messages[-1]["files"]` array.
// When non-empty, each URL is added to the content as an image
// part regardless of the original message content.
//
// factory values supported:
//   - "gemini"     → {"text": ...} / {"inline_data": {...}}
//   - "anthropic"  → {"type": "text", ...} / {"type": "image", "source": {...}}
//   - default      → {"type": "text", ...} / {"type": "image_url", "image_url": {...}}
//
// If the message is already a string, it is returned unchanged.
// If the message has no image parts and `imageAttachments` is empty,
// the text is returned as a string for compatibility with providers
// that don't accept content arrays.
func ConvertLastUserMsgToMultimodal(msg map[string]interface{}, imageAttachments []string, factory string) (map[string]interface{}, error) {
	if msg == nil {
		return nil, fmt.Errorf("nil message")
	}
	originalContent, ok := msg["content"]
	if !ok {
		return msg, nil
	}
	// If the content is already a plain string and there are no
	// imageAttachments to add, leave it alone.
	if _, isString := originalContent.(string); isString && len(imageAttachments) == 0 {
		return msg, nil
	}

	// Combine images from the content array and from imageAttachments
	// (the `files` array on the last user message).
	// Order: content-array images first, then files-array images.
	textPart := normalizeTextFromContent(originalContent)
	imageURLs := extractImageURLs(originalContent)
	allImageURLs := append(imageURLs, imageAttachments...)
	if len(allImageURLs) == 0 {
		// No images — collapse to a string for compatibility.
		out := make(map[string]interface{}, len(msg))
		for k, v := range msg {
			out[k] = v
		}
		out["content"] = textPart
		return out, nil
	}

	// Build ContentPart slice.
	parts := make([]ContentPart, 0, 1+len(allImageURLs))
	if textPart != "" {
		parts = append(parts, ContentPart{Type: "text", Text: textPart})
	}
	for _, u := range allImageURLs {
		src, err := parseDataURIOrB64(u)
		if err != nil {
			return nil, fmt.Errorf("image_url %q: %w", u, err)
		}
		// OpenAI / default: pass the raw URL through (provider accepts
		// both data: and http(s):). Anthropic / Gemini need a Source.
		if factory == "anthropic" || factory == "gemini" {
			parts = append(parts, ContentPart{
				Type:   pickImageType(factory),
				Source: &src,
			})
		} else {
			parts = append(parts, ContentPart{
				Type:     "image_url",
				ImageURL: &ImageURL{URL: u},
			})
		}
	}

	// Render to the driver's wire format.
	rendered, err := RenderContentPartsForFactory(parts, factory)
	if err != nil {
		return nil, err
	}
	out := make(map[string]interface{}, len(msg))
	for k, v := range msg {
		out[k] = v
	}
	out["content"] = rendered
	return out, nil
}

func pickImageType(factory string) string {
	if factory == "gemini" {
		return "inline_data"
	}
	return "image"
}

// RenderContentPartsForFactory converts internal ContentPart values
// into the per-provider JSON wire format:
//
//   - gemini:    [{"text": ...}, {"inline_data": {"mime_type": ..., "data": ...}}]
//   - anthropic: [{"type": "text", "text": ...}, {"type": "image", "source": {...}}]
//   - default:   [{"type": "text", "text": ...}, {"type": "image_url", "image_url": {"url": ...}}]
//
// The return value is suitable for direct assignment to a Message's
// `Content` field (`interface{}`).
func RenderContentPartsForFactory(parts []ContentPart, factory string) (interface{}, error) {
	factory = strings.ToLower(factory)
	switch factory {
	case "gemini":
		out := make([]map[string]interface{}, 0, len(parts))
		for _, p := range parts {
			switch p.Type {
			case "text":
				out = append(out, map[string]interface{}{"text": p.Text})
			case "image", "inline_data":
				if p.Source == nil {
					return nil, fmt.Errorf("gemini image part missing source")
				}
				if p.Source.Type == "url" {
					out = append(out, map[string]interface{}{
						"file_data": map[string]interface{}{
							"file_uri":  p.Source.URL,
							"mime_type": p.Source.MediaType,
						},
					})
				} else {
					out = append(out, map[string]interface{}{
						"inline_data": map[string]interface{}{
							"mime_type": p.Source.MediaType,
							"data":      p.Source.Data,
						},
					})
				}
			}
		}
		return out, nil
	case "anthropic":
		out := make([]map[string]interface{}, 0, len(parts))
		for _, p := range parts {
			switch p.Type {
			case "text":
				out = append(out, map[string]interface{}{
					"type": "text",
					"text": p.Text,
				})
			case "image":
				if p.Source == nil {
					return nil, fmt.Errorf("anthropic image part missing source")
				}
				out = append(out, map[string]interface{}{
					"type":   "image",
					"source": p.Source,
				})
			}
		}
		return out, nil
	default:
		// OpenAI-compatible.
		out := make([]map[string]interface{}, 0, len(parts))
		for _, p := range parts {
			switch p.Type {
			case "text":
				out = append(out, map[string]interface{}{
					"type": "text",
					"text": p.Text,
				})
			case "image_url":
				if p.ImageURL == nil {
					return nil, fmt.Errorf("openai image_url part missing URL")
				}
				out = append(out, map[string]interface{}{
					"type":      "image_url",
					"image_url": p.ImageURL,
				})
			}
		}
		return out, nil
	}
}