zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-27 19:02:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support stream for multimodal chat (#14537)

Browse Source 
### What problem does this PR solve?

Support stream for multimodal chat

### Type of change

- [x] Refactoring
This commit is contained in:
qinling0210
2026-04-30 19:33:57 +08:00
committed by GitHub
parent 5fd4579a2f
commit 12af73f2ca
15 changed files with 369 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
internal/entity/models/aliyun.go
View File

@@ -60,7 +60,7 @@ func (z *AliyunModel) Name() string {
return "siliconflow" return "siliconflow"
} }
func (z *AliyunModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *AliyunModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if len(messages) == 0 { if len(messages) == 0 {
return nil, fmt.Errorf("messages is empty") return nil, fmt.Errorf("messages is empty")
} }
@@ -195,8 +195,12 @@ func (z *AliyunModel) ChatWithMessages(modelName string, apiConfig *APIConfig, m
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *AliyunModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *AliyunModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -204,13 +208,20 @@ func (z *AliyunModel) ChatStreamlyWithSender(modelName, message *string, apiConf
url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat) url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat)
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": false,
"temperature": 1, "temperature": 1,
} }

136
internal/entity/models/deepseek.go
View File

@@ -60,7 +60,7 @@ func (z *DeepSeekModel) Name() string {
return "deepseek" return "deepseek"
} }
func (z *DeepSeekModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *DeepSeekModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if len(messages) == 0 { if len(messages) == 0 {
return nil, fmt.Errorf("messages is empty") return nil, fmt.Errorf("messages is empty")
} }
@@ -227,8 +227,12 @@ func (z *DeepSeekModel) ChatWithMessages(modelName string, apiConfig *APIConfig,
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *DeepSeekModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *DeepSeekModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -236,77 +240,83 @@ func (z *DeepSeekModel) ChatStreamlyWithSender(modelName, message *string, apiCo
url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region]) url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region])
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": false,
"temperature": 1, "temperature": 1,
} }
if chatModelConfig.Stream != nil { if chatModelConfig != nil {
reqBody["stream"] = *chatModelConfig.Stream if chatModelConfig.Stream != nil {
} reqBody["stream"] = *chatModelConfig.Stream
}
if chatModelConfig.MaxTokens != nil { if chatModelConfig.MaxTokens != nil {
reqBody["max_tokens"] = *chatModelConfig.MaxTokens reqBody["max_tokens"] = *chatModelConfig.MaxTokens
} }
if chatModelConfig.Temperature != nil { if chatModelConfig.Temperature != nil {
reqBody["temperature"] = *chatModelConfig.Temperature reqBody["temperature"] = *chatModelConfig.Temperature
} }
if chatModelConfig.DoSample != nil { if chatModelConfig.DoSample != nil {
reqBody["do_sample"] = *chatModelConfig.DoSample reqBody["do_sample"] = *chatModelConfig.DoSample
} }
if chatModelConfig.TopP != nil { if chatModelConfig.TopP != nil {
reqBody["top_p"] = *chatModelConfig.TopP reqBody["top_p"] = *chatModelConfig.TopP
} }
if chatModelConfig.Stop != nil { if chatModelConfig.Stop != nil {
reqBody["stop"] = *chatModelConfig.Stop reqBody["stop"] = *chatModelConfig.Stop
} }
if chatModelConfig.Thinking != nil { if chatModelConfig.Thinking != nil {
if *chatModelConfig.Thinking { if *chatModelConfig.Thinking {
var thinkingFlag string var thinkingFlag string
switch *chatModelConfig.Effort { switch *chatModelConfig.Effort {
case "none": case "none":
thinkingFlag = "disabled" thinkingFlag = "disabled"
chatModelConfig.Thinking = nil break
break case "low":
case "low": thinkingFlag = "disabled"
thinkingFlag = "disabled" break
chatModelConfig.Thinking = nil case "medium":
break thinkingFlag = "disabled"
case "medium": break
thinkingFlag = "disabled" case "high":
chatModelConfig.Thinking = nil thinkingFlag = "enabled"
break reqBody["reasoning_effort"] = "high"
case "high": break
thinkingFlag = "enabled" case "default":
reqBody["reasoning_effort"] = "high" thinkingFlag = "enabled"
break reqBody["reasoning_effort"] = "high"
case "default": break
thinkingFlag = "enabled" case "max":
reqBody["reasoning_effort"] = "high" thinkingFlag = "enabled"
break reqBody["reasoning_effort"] = "max"
case "max": break
thinkingFlag = "enabled" default:
reqBody["reasoning_effort"] = "max" return fmt.Errorf("invalid effort level")
break }
default: reqBody["thinking"] = map[string]interface{}{
return fmt.Errorf("invalid effort level") "type": thinkingFlag,
} }
reqBody["thinking"] = map[string]interface{}{ } else {
"type": thinkingFlag, reqBody["thinking"] = map[string]interface{}{
} "type": "disabled",
} else { }
reqBody["thinking"] = map[string]interface{}{
"type": "disabled",
} }
} }
} }

6
internal/entity/models/dummy.go
View File

@@ -43,12 +43,12 @@ func (z *DummyModel) Name() string {
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *DummyModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *DummyModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
return nil, fmt.Errorf("not implemented") return nil, fmt.Errorf("not implemented")
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *DummyModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *DummyModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
return fmt.Errorf("not implemented") return fmt.Errorf("not implemented")
} }

27
internal/entity/models/gitee.go
View File

@@ -61,7 +61,7 @@ func (z *GiteeModel) Name() string {
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *GiteeModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *GiteeModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is nil or empty") return nil, fmt.Errorf("api key is nil or empty")
} }
@@ -211,8 +211,12 @@ func (z *GiteeModel) ChatWithMessages(modelName string, apiConfig *APIConfig, me
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *GiteeModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *GiteeModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -220,13 +224,20 @@ func (z *GiteeModel) ChatStreamlyWithSender(modelName, message *string, apiConfi
url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region]) url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region])
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": false,
"temperature": 1, "temperature": 1,
} }

61
internal/entity/models/google.go
View File

@@ -46,7 +46,7 @@ func (z *GoogleModel) Name() string {
return "google" return "google"
} }
func (z *GoogleModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *GoogleModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is nil or empty") return nil, fmt.Errorf("api key is nil or empty")
} }
@@ -119,8 +119,12 @@ func (z *GoogleModel) ChatWithMessages(modelName string, apiConfig *APIConfig, m
return &ChatResponse{Answer: &answer}, nil return &ChatResponse{Answer: &answer}, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *GoogleModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *GoogleModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
ctx := context.Background() ctx := context.Background()
client, err := genai.NewClient(ctx, &genai.ClientConfig{ client, err := genai.NewClient(ctx, &genai.ClientConfig{
APIKey: *apiConfig.ApiKey, APIKey: *apiConfig.ApiKey,
@@ -129,12 +133,53 @@ func (z *GoogleModel) ChatStreamlyWithSender(modelName, message *string, apiConf
if err != nil { if err != nil {
return err return err
} }
contents := []*genai.Content{
genai.NewContentFromText(*message, genai.RoleUser), // Convert messages to Google SDK format
var contents []*genai.Content
for _, msg := range messages {
var role genai.Role
switch msg.Role {
case "user":
role = genai.RoleUser
case "model", "assistant":
role = genai.RoleModel
default:
role = genai.RoleUser
}
// Handle content based on type
switch c := msg.Content.(type) {
case string:
contents = append(contents, genai.NewContentFromText(c, role))
case []interface{}:
// Multimodal content - group parts within a single content
var parts []*genai.Part
for _, item := range c {
if itemMap, ok := item.(map[string]interface{}); ok {
contentType, _ := itemMap["type"].(string)
switch contentType {
case "text":
if text, ok := itemMap["text"].(string); ok {
parts = append(parts, genai.NewPartFromText(text))
}
case "image_url":
if imgMap, ok := itemMap["image_url"].(map[string]interface{}); ok {
if url, ok := imgMap["url"].(string); ok {
parts = append(parts, genai.NewPartFromURI(url, "image/jpeg"))
}
}
}
}
}
if len(parts) > 0 {
contents = append(contents, genai.NewContentFromParts(parts, role))
}
}
} }
for response, err := range client.Models.GenerateContentStream( for response, err := range client.Models.GenerateContentStream(
ctx, ctx,
*modelName, modelName,
contents, contents,
nil, nil,
) { ) {
@@ -145,7 +190,7 @@ func (z *GoogleModel) ChatStreamlyWithSender(modelName, message *string, apiConf
content := response.Text() content := response.Text()
var responseContent string var responseContent string
if chatModelConfig.Thinking != nil && *chatModelConfig.Thinking { if chatModelConfig != nil && chatModelConfig.Thinking != nil && *chatModelConfig.Thinking {
responseContent = response.Candidates[0].Content.Parts[0].Text responseContent = response.Candidates[0].Content.Parts[0].Text
} }
@@ -157,7 +202,7 @@ func (z *GoogleModel) ChatStreamlyWithSender(modelName, message *string, apiConf
} }
if content != "" { if content != "" {
logger.Info(fmt.Sprintf("Answer: %s", responseContent)) logger.Info(fmt.Sprintf("Answer: %s", content))
if err = sender(&content, nil); err != nil { if err = sender(&content, nil); err != nil {
return err return err
} }

25
internal/entity/models/minimax.go
View File

@@ -179,7 +179,7 @@ func (z *MinimaxModel) Chat(modelName, message *string, apiConfig *APIConfig, mo
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *MinimaxModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *MinimaxModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is nil or empty") return nil, fmt.Errorf("api key is nil or empty")
} }
@@ -310,8 +310,12 @@ func (z *MinimaxModel) ChatWithMessages(modelName string, apiConfig *APIConfig,
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *MinimaxModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *MinimaxModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
@@ -320,12 +324,19 @@ func (z *MinimaxModel) ChatStreamlyWithSender(modelName, message *string, apiCon
url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat) url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat)
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]interface{}{ "messages": apiMessages,
{"role": "user", "content": *message},
},
"stream": true, "stream": true,
"temperature": 1, "temperature": 1,
} }

27
internal/entity/models/moonshot.go
View File

@@ -60,7 +60,7 @@ func (z *MoonshotModel) Name() string {
return "moonshot" return "moonshot"
} }
func (k *MoonshotModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (k *MoonshotModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if len(messages) == 0 { if len(messages) == 0 {
return nil, fmt.Errorf("messages is empty") return nil, fmt.Errorf("messages is empty")
} }
@@ -199,8 +199,12 @@ func (k *MoonshotModel) ChatWithMessages(modelName string, apiConfig *APIConfig,
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (k *MoonshotModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (k *MoonshotModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -208,13 +212,20 @@ func (k *MoonshotModel) ChatStreamlyWithSender(modelName, message *string, apiCo
url := fmt.Sprintf("%s/chat/completions", k.BaseURL[region]) url := fmt.Sprintf("%s/chat/completions", k.BaseURL[region])
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": true,
} }
if chatModelConfig.Stream != nil { if chatModelConfig.Stream != nil {

81
internal/entity/models/siliconflow.go
View File

@@ -80,7 +80,7 @@ type SiliconflowRerankResponse struct {
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *SiliconflowModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *SiliconflowModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is nil or empty") return nil, fmt.Errorf("api key is nil or empty")
} }
@@ -214,8 +214,12 @@ func (z *SiliconflowModel) ChatWithMessages(modelName string, apiConfig *APIConf
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *SiliconflowModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *SiliconflowModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -223,48 +227,57 @@ func (z *SiliconflowModel) ChatStreamlyWithSender(modelName, message *string, ap
url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region]) url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region])
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": false,
"temperature": 1, "temperature": 1,
} }
if chatModelConfig.Stream != nil { if chatModelConfig != nil {
reqBody["stream"] = *chatModelConfig.Stream if chatModelConfig.Stream != nil {
} reqBody["stream"] = *chatModelConfig.Stream
}
if chatModelConfig.MaxTokens != nil { if chatModelConfig.MaxTokens != nil {
reqBody["max_tokens"] = *chatModelConfig.MaxTokens reqBody["max_tokens"] = *chatModelConfig.MaxTokens
} }
if chatModelConfig.Temperature != nil { if chatModelConfig.Temperature != nil {
reqBody["temperature"] = *chatModelConfig.Temperature reqBody["temperature"] = *chatModelConfig.Temperature
} }
if chatModelConfig.DoSample != nil { if chatModelConfig.DoSample != nil {
reqBody["do_sample"] = *chatModelConfig.DoSample reqBody["do_sample"] = *chatModelConfig.DoSample
} }
if chatModelConfig.TopP != nil { if chatModelConfig.TopP != nil {
reqBody["top_p"] = *chatModelConfig.TopP reqBody["top_p"] = *chatModelConfig.TopP
} }
if chatModelConfig.Stop != nil { if chatModelConfig.Stop != nil {
reqBody["stop"] = *chatModelConfig.Stop reqBody["stop"] = *chatModelConfig.Stop
} }
if chatModelConfig.Thinking != nil { if chatModelConfig.Thinking != nil {
if *chatModelConfig.Thinking { if *chatModelConfig.Thinking {
reqBody["thinking"] = map[string]interface{}{ reqBody["thinking"] = map[string]interface{}{
"type": "enabled", "type": "enabled",
} }
} else { } else {
reqBody["thinking"] = map[string]interface{}{ reqBody["thinking"] = map[string]interface{}{
"type": "disabled", "type": "disabled",
}
} }
} }
} }

7
internal/entity/models/types.go
View File

@@ -18,9 +18,10 @@ type ModelDriver interface {
Name() string Name() string
// ChatWithMessages sends multiple messages with role and content // ChatWithMessages sends multiple messages with role and content
ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error)
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error // messages accepts []Message which supports multimodal content (e.g., [{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}])
ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error
// Encode encodes a list of texts into embeddings // Encode encodes a list of texts into embeddings
Encode(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([][]float64, error) Encode(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([][]float64, error)
// Rerank calculates similarity scores between query and texts // Rerank calculates similarity scores between query and texts

30
internal/entity/models/vllm.go
View File

@@ -199,7 +199,7 @@ func (z *VllmModel) Chat(modelName, message *string, apiConfig *APIConfig, chatM
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *VllmModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *VllmModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if len(messages) == 0 { if len(messages) == 0 {
return nil, fmt.Errorf("messages is empty") return nil, fmt.Errorf("messages is empty")
} }
@@ -332,27 +332,37 @@ func (z *VllmModel) ChatWithMessages(modelName string, apiConfig *APIConfig, mes
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *VllmModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *VllmModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
} }
url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat) url := fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.Chat)
// TODO figure out what's the point of these codes modelType := strings.Split(modelName, "-")[0]
modelType := strings.Split(*modelName, "-")[0]
if modelType == "qwen" || modelType == "glm" { if modelType == "qwen" || modelType == "glm" {
url = fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.AsyncChat) url = fmt.Sprintf("%s/%s", z.BaseURL[region], z.URLSuffix.AsyncChat)
} }
// Convert messages to API format (supporting multimodal content)
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": true,
} }
if modelConfig.Stream != nil { if modelConfig.Stream != nil {

25
internal/entity/models/volcengine.go
View File

@@ -211,7 +211,7 @@ func (z *VolcEngine) Chat(modelName, message *string, apiConfig *APIConfig, mode
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *VolcEngine) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *VolcEngine) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if len(messages) == 0 { if len(messages) == 0 {
return nil, fmt.Errorf("messages is empty") return nil, fmt.Errorf("messages is empty")
} }
@@ -370,8 +370,12 @@ func (z *VolcEngine) ChatWithMessages(modelName string, apiConfig *APIConfig, me
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *VolcEngine) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *VolcEngine) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
@@ -380,12 +384,19 @@ func (z *VolcEngine) ChatStreamlyWithSender(modelName, message *string, apiConfi
url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region]) url := fmt.Sprintf("%s/chat/completions", z.BaseURL[region])
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]interface{}{ "messages": apiMessages,
{"role": "user", "content": *message},
},
"stream": true, "stream": true,
"temperature": 1, "temperature": 1,
} }

81
internal/entity/models/zhipu-ai.go
View File

@@ -61,7 +61,7 @@ func (z *ZhipuAIModel) Name() string {
} }
// ChatWithMessages sends multiple messages with roles and returns response // ChatWithMessages sends multiple messages with roles and returns response
func (z *ZhipuAIModel) ChatWithMessages(modelName string, apiConfig *APIConfig, messages []Message, chatModelConfig *ChatConfig) (*ChatResponse, error) { func (z *ZhipuAIModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" { if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is nil or empty") return nil, fmt.Errorf("api key is nil or empty")
} }
@@ -201,8 +201,12 @@ func (z *ZhipuAIModel) ChatWithMessages(modelName string, apiConfig *APIConfig,
return chatResponse, nil return chatResponse, nil
} }
// ChatStreamlyWithSender sends a message and streams response via sender function (best performance, no channel) // ChatStreamlyWithSender sends messages and streams response via sender function (best performance, no channel)
func (z *ZhipuAIModel) ChatStreamlyWithSender(modelName, message *string, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error { func (z *ZhipuAIModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, sender func(*string, *string) error) error {
if len(messages) == 0 {
return fmt.Errorf("messages is empty")
}
var region = "default" var region = "default"
if apiConfig.Region != nil { if apiConfig.Region != nil {
region = *apiConfig.Region region = *apiConfig.Region
@@ -210,48 +214,57 @@ func (z *ZhipuAIModel) ChatStreamlyWithSender(modelName, message *string, apiCon
url := fmt.Sprintf("%s/%s", strings.TrimSuffix(z.BaseURL[region], "/"), z.URLSuffix.Chat) url := fmt.Sprintf("%s/%s", strings.TrimSuffix(z.BaseURL[region], "/"), z.URLSuffix.Chat)
// Convert messages to API format
apiMessages := make([]map[string]interface{}, len(messages))
for i, msg := range messages {
apiMessages[i] = map[string]interface{}{
"role": msg.Role,
"content": msg.Content,
}
}
// Build request body with streaming enabled // Build request body with streaming enabled
reqBody := map[string]interface{}{ reqBody := map[string]interface{}{
"model": modelName, "model": modelName,
"messages": []map[string]string{ "messages": apiMessages,
{"role": "user", "content": *message}, "stream": true,
},
"stream": false,
"temperature": 1, "temperature": 1,
} }
if chatModelConfig.Stream != nil { if chatModelConfig != nil {
reqBody["stream"] = *chatModelConfig.Stream if chatModelConfig.Stream != nil {
} reqBody["stream"] = *chatModelConfig.Stream
}
if chatModelConfig.MaxTokens != nil { if chatModelConfig.MaxTokens != nil {
reqBody["max_tokens"] = *chatModelConfig.MaxTokens reqBody["max_tokens"] = *chatModelConfig.MaxTokens
} }
if chatModelConfig.Temperature != nil { if chatModelConfig.Temperature != nil {
reqBody["temperature"] = *chatModelConfig.Temperature reqBody["temperature"] = *chatModelConfig.Temperature
} }
if chatModelConfig.DoSample != nil { if chatModelConfig.DoSample != nil {
reqBody["do_sample"] = *chatModelConfig.DoSample reqBody["do_sample"] = *chatModelConfig.DoSample
} }
if chatModelConfig.TopP != nil { if chatModelConfig.TopP != nil {
reqBody["top_p"] = *chatModelConfig.TopP reqBody["top_p"] = *chatModelConfig.TopP
} }
if chatModelConfig.Stop != nil { if chatModelConfig.Stop != nil {
reqBody["stop"] = *chatModelConfig.Stop reqBody["stop"] = *chatModelConfig.Stop
} }
if chatModelConfig.Thinking != nil { if chatModelConfig.Thinking != nil {
if *chatModelConfig.Thinking { if *chatModelConfig.Thinking {
reqBody["thinking"] = map[string]interface{}{ reqBody["thinking"] = map[string]interface{}{
"type": "enabled", "type": "enabled",
} }
} else { } else {
reqBody["thinking"] = map[string]interface{}{ reqBody["thinking"] = map[string]interface{}{
"type": "disabled", "type": "disabled",
}
} }
} }
} }

28
internal/handler/providers.go
View File

@@ -828,24 +828,6 @@ func (h *ProviderHandler) ChatToModel(c *gin.Context) {
// Check if it's a stream request // Check if it's a stream request
if req.Stream { if req.Stream {
// Streaming with multimodal messages not yet supported
hasMultimodal := false
for _, msg := range req.Messages {
if content, ok := msg["content"]; ok {
if _, isArray := content.([]interface{}); isArray {
hasMultimodal = true
break
}
}
}
if hasMultimodal {
c.JSON(http.StatusBadRequest, gin.H{
"code": 400,
"message": "Streaming with multimodal messages not yet supported",
})
return
}
// Set SSE headers // Set SSE headers
c.Header("Content-Type", "text/event-stream") c.Header("Content-Type", "text/event-stream")
c.Header("Cache-Control", "no-cache") c.Header("Cache-Control", "no-cache")
@@ -876,8 +858,16 @@ func (h *ProviderHandler) ChatToModel(c *gin.Context) {
return nil return nil
} }
// Convert []map[string]interface{} to []models.Message
messages := make([]models.Message, len(req.Messages))
for i, msg := range req.Messages {
role, _ := msg["role"].(string)
content := msg["content"]
messages[i] = models.Message{Role: role, Content: content}
}
// Stream response using sender function (best performance, no channel) // Stream response using sender function (best performance, no channel)
errorCode, err := h.modelProviderService.ChatToModelStreamWithSender(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, req.Messages[0]["content"].(string), &apiConfig, &chatConfig, sender) errorCode, err := h.modelProviderService.ChatToModelStreamWithSender(*req.ProviderName, *req.InstanceName, *req.ModelName, userID, messages, &apiConfig, &chatConfig, sender)
if errorCode != common.CodeSuccess { if errorCode != common.CodeSuccess {
c.SSEvent("error", err.Error()) c.SSEvent("error", err.Error())

23
internal/service/chat_session.go
View File

@@ -562,7 +562,7 @@ func (s *ChatSessionService) asyncChatSolo(dialog *entity.Chat, session *entity.
chatConfig := s.buildChatConfig(dialog, config) chatConfig := s.buildChatConfig(dialog, config)
// Perform chat // Perform chat
response, err := chatModel.ModelDriver.ChatWithMessages(*chatModel.ModelName, chatModel.APIConfig, msgs, chatConfig) response, err := chatModel.ModelDriver.ChatWithMessages(*chatModel.ModelName, msgs, chatModel.APIConfig, chatConfig)
if err != nil { if err != nil {
logger.Error("asyncChatSolo chat failed", err) logger.Error("asyncChatSolo chat failed", err)
return nil, err return nil, err
@@ -604,26 +604,31 @@ func (s *ChatSessionService) asyncChatSoloStream(dialog *entity.Chat, session *e
return return
} }
// Convert messages to single string for ChatStreamlyWithSender // Convert messages to []modelModule.Message for ChatStreamlyWithSender
var msgBuilder strings.Builder var chatMessages []modelModule.Message
if systemPrompt != "" { if systemPrompt != "" {
msgBuilder.WriteString("System: " + systemPrompt + "\n") chatMessages = append(chatMessages, modelModule.Message{
Role: "system",
Content: systemPrompt,
})
} }
for _, msg := range processedMessages { for _, msg := range processedMessages {
role, _ := msg["role"].(string) role, _ := msg["role"].(string)
content, _ := msg["content"].(string) content := msg["content"]
if role != "" && content != "" && role != "system" { if role != "" && content != nil && role != "system" {
msgBuilder.WriteString(role + ": " + content + "\n") chatMessages = append(chatMessages, modelModule.Message{
Role: role,
Content: content,
})
} }
} }
messageStr := msgBuilder.String()
// Get ChatConfig directly from dialog and config // Get ChatConfig directly from dialog and config
chatConfig := s.buildChatConfig(dialog, config) chatConfig := s.buildChatConfig(dialog, config)
// Perform streaming chat using ChatStreamlyWithSender // Perform streaming chat using ChatStreamlyWithSender
fullAnswer := "" fullAnswer := ""
err = chatModel.ModelDriver.ChatStreamlyWithSender(chatModel.ModelName, &messageStr, chatModel.APIConfig, chatConfig, func(answer *string, reason *string) error { err = chatModel.ModelDriver.ChatStreamlyWithSender(*chatModel.ModelName, chatMessages, chatModel.APIConfig, chatConfig, func(answer *string, reason *string) error {
if reason != nil && *reason != "" { if reason != nil && *reason != "" {
fullAnswer += *reason fullAnswer += *reason
ans := s.structureAnswer(session, fullAnswer, messageID, session.ID, reference) ans := s.structureAnswer(session, fullAnswer, messageID, session.ID, reference)

12
internal/service/model_service.go
View File

@@ -740,7 +740,7 @@ func (m *ModelProviderService) ChatToModelWithMessages(providerName, instanceNam
apiConfig.ApiKey = &instance.APIKey apiConfig.ApiKey = &instance.APIKey
var response *modelModule.ChatResponse var response *modelModule.ChatResponse
response, err = providerInfo.ModelDriver.ChatWithMessages(modelName, apiConfig, messages, modelConfig) response, err = providerInfo.ModelDriver.ChatWithMessages(modelName, messages, apiConfig, modelConfig)
if err != nil { if err != nil {
return nil, common.CodeServerError, err return nil, common.CodeServerError, err
} }
@@ -776,7 +776,7 @@ func (m *ModelProviderService) ChatToModelWithMessages(providerName, instanceNam
newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL) newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL)
var response *modelModule.ChatResponse var response *modelModule.ChatResponse
response, err = newProviderInfo.ChatWithMessages(modelName, apiConfig, messages, modelConfig) response, err = newProviderInfo.ChatWithMessages(modelName, messages, apiConfig, modelConfig)
if err != nil { if err != nil {
return nil, common.CodeServerError, err return nil, common.CodeServerError, err
} }
@@ -803,7 +803,7 @@ func (m *ModelProviderService) ChatWithMessagesToModelByApiKey(providerName, mod
} }
var response *modelModule.ChatResponse var response *modelModule.ChatResponse
response, err = providerInfo.ModelDriver.ChatWithMessages(modelName, &modelModule.APIConfig{ApiKey: &apiKey}, messages, nil) response, err = providerInfo.ModelDriver.ChatWithMessages(modelName, messages, &modelModule.APIConfig{ApiKey: &apiKey}, nil)
if err != nil { if err != nil {
return nil, common.CodeServerError, err return nil, common.CodeServerError, err
} }
@@ -815,7 +815,7 @@ func (m *ModelProviderService) ChatWithMessagesToModelByApiKey(providerName, mod
} }
// ChatToModelStreamWithSender streams chat response directly via sender function (best performance, no channel) // ChatToModelStreamWithSender streams chat response directly via sender function (best performance, no channel)
func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanceName, modelName, userID, message string, apiConfig *modelModule.APIConfig, modelConfig *modelModule.ChatConfig, sender func(*string, *string) error) (common.ErrorCode, error) { func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanceName, modelName, userID string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, modelConfig *modelModule.ChatConfig, sender func(*string, *string) error) (common.ErrorCode, error) {
// Get tenant ID from user // Get tenant ID from user
tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner") tenants, err := m.userTenantDAO.GetByUserIDAndRole(userID, "owner")
if err != nil { if err != nil {
@@ -861,7 +861,7 @@ func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanc
apiConfig.Region = &region apiConfig.Region = &region
apiConfig.ApiKey = &instance.APIKey apiConfig.ApiKey = &instance.APIKey
err = providerInfo.ModelDriver.ChatStreamlyWithSender(&modelName, &message, apiConfig, modelConfig, sender) err = providerInfo.ModelDriver.ChatStreamlyWithSender(modelName, messages, apiConfig, modelConfig, sender)
if err != nil { if err != nil {
return common.CodeServerError, err return common.CodeServerError, err
} }
@@ -893,7 +893,7 @@ func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanc
} }
newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL) newProviderInfo := providerInfo.ModelDriver.NewInstance(newURL)
err = newProviderInfo.ChatStreamlyWithSender(&modelName, &message, apiConfig, modelConfig, sender) err = newProviderInfo.ChatStreamlyWithSender(modelName, messages, apiConfig, modelConfig, sender)
if err != nil { if err != nil {
return common.CodeServerError, err return common.CodeServerError, err
} }