mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-01 08:15:44 +08:00
Go: implement provider: Nvidia (#14623)
### What problem does this PR solve? 1. **Implement `Nvidia` Provider:** Fully support NVIDIA NIM APIs with robust parameter handling (including the `thinking` parameter) and safe URL merging in `NewInstance`. 2. **Fix Misleading CLI Errors:** Corrected a bug in `common_command.go` where failed chat requests inaccurately reported `failed to list instance models`. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Haruko386
461
conf/models/nvidia.json
Normal file
461
conf/models/nvidia.json
Normal file
@@ -0,0 +1,461 @@
|
||||
{
|
||||
"name": "Nvidia",
|
||||
"url": {
|
||||
"default": "https://integrate.api.nvidia.com/v1"
|
||||
},
|
||||
"url_suffix": {
|
||||
"chat": "chat/completions",
|
||||
"models": "models"
|
||||
},
|
||||
"class": "nvidia",
|
||||
"models": [
|
||||
{
|
||||
"name": "abacusai/dracarys-llama-3.1-70b-instruct",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "bytedance/seed-oss-36b-instruct",
|
||||
"max_tokens": 32768,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/deepseek-v4-flash",
|
||||
"max_tokens": 1048576,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/deepseek-v4-pro",
|
||||
"max_tokens": 1048576,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/deepseek-v3.2",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/deepseek-v3.1",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "google/codegemma-7b",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google/gemma-2-2b-it",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google/gemma-4-31b-it",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google/gemma-7b",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ibm/granite-3.3-8b-instruct",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "meta/llama-3.1-405b-instruct",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "meta/llama-3.2-90b-vision-instruct",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat",
|
||||
"vision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "meta/llama-4-maverick-17b-128e-instruct",
|
||||
"max_tokens": 1048576,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "microsoft/phi-4-mini-flash-reasoning",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "minimaxai/minimax-m2.1",
|
||||
"max_tokens": 204800,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "minimaxai/minimax-m2.5",
|
||||
"max_tokens": 204800,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "minimaxai/minimax-m2.7",
|
||||
"max_tokens": 204800,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/devstral-2-123b-instruct-2512",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/magistral-small-2506",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/mistral-7b-instruct-v0.3",
|
||||
"max_tokens": 32768,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/mistral-large-3-675b-instruct-2512",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/mistral-medium-3-5-128b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat",
|
||||
"vision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/mistral-nemotron",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mistralai/mixtral-8x22b-instruct",
|
||||
"max_tokens": 65536,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "moonshotai/kimi-k2.5",
|
||||
"max_tokens": 262144,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "moonshotai/kimi-k2.6",
|
||||
"max_tokens": 262144,
|
||||
"model_types": [
|
||||
"chat",
|
||||
"vision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "moonshotai/kimi-k2-instruct",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "moonshotai/kimi-k2-instruct-0905",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "moonshotai/kimi-k2-thinking",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nvidia/gliner-pii",
|
||||
"max_tokens": 4096,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.1-nemoguard-8b-content-safety",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.1-nemoguard-8b-topic-control",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.1-nemotron-nano-8b-v1",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.1-nemotron-ultra-253b-v1",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.2-nemoretriever-1b-vlm-embed-v1",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"embedding"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.3-nemotron-super-49b-v1",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemoguard-jailbreak-detect",
|
||||
"max_tokens": 4096,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemotron-3-nano-30b-a3b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat",
|
||||
"vision"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemotron-3-super-120b-a12b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemotron-content-safety-reasoning-4b",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nemotron-mini-4b-instruct",
|
||||
"max_tokens": 4096,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/nvidia-nemotron-nano-9b-v2",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/riva-translate-4b-instruct-v1_1",
|
||||
"max_tokens": 4096,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "nvidia/usdcode",
|
||||
"max_tokens": 8192,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "openai/gpt-oss-120b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "qwen/qwen2.5-coder-7b-instruct",
|
||||
"max_tokens": 32768,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "qwen/qwen3-5-122b-a10b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "qwen/qwen3-235b-a22b",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "qwen/qwen3-coder-480b-a35b-instruct",
|
||||
"max_tokens": 262144,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "z-ai/glm-5",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "z-ai/glm-5.1",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "z-ai/glm-4.7",
|
||||
"max_tokens": 131072,
|
||||
"model_types": [
|
||||
"chat"
|
||||
],
|
||||
"thinking": {
|
||||
"default_value": true,
|
||||
"clear_thinking": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -21,7 +21,10 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
netUrl "net/url"
|
||||
"os"
|
||||
ce "ragflow/internal/cli/filesystem"
|
||||
@@ -1755,7 +1758,16 @@ func (c *RAGFlowClient) ChatToModel(cmd *Command) (ResponseIf, error) {
|
||||
|
||||
resp, err := c.HTTPClient.Request("POST", url, true, "web", nil, payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list instance models: %w", err)
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return nil, fmt.Errorf("connection closed (EOF): upstream overloaded or proxy timeout: %w", err)
|
||||
}
|
||||
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return nil, fmt.Errorf("request timeout: model took too long to respond: %w", err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
|
||||
@@ -57,8 +57,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
|
||||
return NewXAIModel(baseURL, urlSuffix), nil
|
||||
case "lmstudio":
|
||||
return NewLmStudioModel(baseURL, urlSuffix), nil
|
||||
case "openai":
|
||||
return NewOpenAIModel(baseURL, urlSuffix), nil
|
||||
case "nvidia":
|
||||
return NewNvidiaModel(baseURL, urlSuffix), nil
|
||||
default:
|
||||
return NewDummyModel(baseURL, urlSuffix), nil
|
||||
}
|
||||
|
||||
345
internal/entity/models/nvidia.go
Normal file
345
internal/entity/models/nvidia.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NvidiaModel implements ModelDriver for Nvidia
|
||||
type NvidiaModel struct {
|
||||
BaseURL map[string]string
|
||||
URLSuffix URLSuffix
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewNvidiaModel creates a new Nvidia model instance
|
||||
func NewNvidiaModel(baseURL map[string]string, urlSuffix URLSuffix) *NvidiaModel {
|
||||
return &NvidiaModel{
|
||||
BaseURL: baseURL,
|
||||
URLSuffix: urlSuffix,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (n NvidiaModel) NewInstance(baseURL map[string]string) ModelDriver {
|
||||
return &NvidiaModel{
|
||||
BaseURL: baseURL,
|
||||
URLSuffix: n.URLSuffix,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (n NvidiaModel) Name() string {
|
||||
return "nvidia"
|
||||
}
|
||||
|
||||
func (n *NvidiaModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
|
||||
if len(messages) == 0 {
|
||||
return nil, fmt.Errorf("messages is empty")
|
||||
}
|
||||
|
||||
var region = "default"
|
||||
if apiConfig != nil && apiConfig.Region != nil {
|
||||
region = *apiConfig.Region
|
||||
}
|
||||
|
||||
baseURL := n.BaseURL[region]
|
||||
if baseURL == "" {
|
||||
baseURL = n.BaseURL["default"]
|
||||
}
|
||||
url := fmt.Sprintf("%s/%s", baseURL, n.URLSuffix.Chat)
|
||||
|
||||
apiMessages := make([]map[string]interface{}, len(messages))
|
||||
for i, msg := range messages {
|
||||
apiMessages[i] = map[string]interface{}{
|
||||
"role": msg.Role,
|
||||
"content": msg.Content,
|
||||
}
|
||||
}
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
"stream": false,
|
||||
}
|
||||
|
||||
if chatModelConfig != nil {
|
||||
if chatModelConfig.Stream != nil {
|
||||
reqBody["stream"] = *chatModelConfig.Stream
|
||||
}
|
||||
if chatModelConfig.MaxTokens != nil {
|
||||
reqBody["max_tokens"] = *chatModelConfig.MaxTokens
|
||||
}
|
||||
if chatModelConfig.Temperature != nil {
|
||||
reqBody["temperature"] = *chatModelConfig.Temperature
|
||||
}
|
||||
if chatModelConfig.TopP != nil {
|
||||
reqBody["top_p"] = *chatModelConfig.TopP
|
||||
}
|
||||
if chatModelConfig.Stop != nil {
|
||||
reqBody["stop"] = *chatModelConfig.Stop
|
||||
}
|
||||
if chatModelConfig.Thinking != nil {
|
||||
if *chatModelConfig.Thinking {
|
||||
reqBody["thinking"] = map[string]interface{}{"type": "enabled"}
|
||||
} else {
|
||||
reqBody["thinking"] = map[string]interface{}{"type": "disabled"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if apiConfig != nil && apiConfig.ApiKey != nil {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
|
||||
}
|
||||
|
||||
resp, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err = json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
choices, ok := result["choices"].([]interface{})
|
||||
if !ok || len(choices) == 0 {
|
||||
return nil, fmt.Errorf("no choices in response")
|
||||
}
|
||||
|
||||
firstChoice, ok := choices[0].(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid choice format")
|
||||
}
|
||||
|
||||
messageMap, ok := firstChoice["message"].(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid message format")
|
||||
}
|
||||
|
||||
content, ok := messageMap["content"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid content format")
|
||||
}
|
||||
|
||||
var modelClass *string
|
||||
if chatModelConfig != nil {
|
||||
modelClass = chatModelConfig.ModelClass
|
||||
}
|
||||
thinking, answer := GetThinkingAndAnswer(modelClass, &content)
|
||||
|
||||
chatResponse := &ChatResponse{
|
||||
Answer: answer,
|
||||
ReasonContent: thinking,
|
||||
}
|
||||
|
||||
return chatResponse, nil
|
||||
}
|
||||
|
||||
func (n *NvidiaModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
|
||||
if len(messages) == 0 {
|
||||
return fmt.Errorf("messages is empty")
|
||||
}
|
||||
|
||||
var region = "default"
|
||||
if apiConfig != nil && apiConfig.Region != nil {
|
||||
region = *apiConfig.Region
|
||||
}
|
||||
|
||||
baseURL := n.BaseURL[region]
|
||||
if baseURL == "" {
|
||||
baseURL = n.BaseURL["default"]
|
||||
}
|
||||
url := fmt.Sprintf("%s/%s", baseURL, n.URLSuffix.Chat)
|
||||
|
||||
apiMessages := make([]map[string]interface{}, len(messages))
|
||||
for i, msg := range messages {
|
||||
apiMessages[i] = map[string]interface{}{
|
||||
"role": msg.Role,
|
||||
"content": msg.Content,
|
||||
}
|
||||
}
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
"stream": true,
|
||||
}
|
||||
|
||||
if modelConfig != nil {
|
||||
if modelConfig.Stream != nil {
|
||||
reqBody["stream"] = *modelConfig.Stream
|
||||
}
|
||||
if modelConfig.MaxTokens != nil {
|
||||
reqBody["max_tokens"] = *modelConfig.MaxTokens
|
||||
}
|
||||
if modelConfig.Temperature != nil {
|
||||
reqBody["temperature"] = *modelConfig.Temperature
|
||||
}
|
||||
if modelConfig.DoSample != nil {
|
||||
reqBody["do_sample"] = *modelConfig.DoSample
|
||||
}
|
||||
if modelConfig.TopP != nil {
|
||||
reqBody["top_p"] = *modelConfig.TopP
|
||||
}
|
||||
if modelConfig.Stop != nil {
|
||||
reqBody["stop"] = *modelConfig.Stop
|
||||
}
|
||||
if modelConfig.Thinking != nil {
|
||||
if *modelConfig.Thinking {
|
||||
reqBody["thinking"] = map[string]interface{}{"type": "enabled"}
|
||||
} else {
|
||||
reqBody["thinking"] = map[string]interface{}{"type": "disabled"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if apiConfig != nil && apiConfig.ApiKey != nil {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
|
||||
}
|
||||
|
||||
resp, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
if !strings.HasPrefix(line, "data:") {
|
||||
continue
|
||||
}
|
||||
|
||||
data := strings.TrimSpace(line[5:])
|
||||
if data == "[DONE]" {
|
||||
break
|
||||
}
|
||||
|
||||
var event map[string]interface{}
|
||||
if err = json.Unmarshal([]byte(data), &event); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
choices, ok := event["choices"].([]interface{})
|
||||
if !ok || len(choices) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
firstChoice, ok := choices[0].(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
delta, ok := firstChoice["delta"].(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
reasoningContent, ok := delta["reasoning_content"].(string)
|
||||
if ok && reasoningContent != "" {
|
||||
if err := sender(nil, &reasoningContent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
content, ok := delta["content"].(string)
|
||||
if ok && content != "" {
|
||||
if err := sender(&content, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
finishReason, ok := firstChoice["finish_reason"].(string)
|
||||
if ok && finishReason != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
endOfStream := "[DONE]"
|
||||
if err = sender(&endOfStream, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func (n NvidiaModel) Encode(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([][]float64, error) {
|
||||
return nil, fmt.Errorf("no such method")
|
||||
}
|
||||
|
||||
func (n NvidiaModel) Rerank(modelName *string, query string, texts []string, apiConfig *APIConfig) ([]float64, error) {
|
||||
return nil, fmt.Errorf("no such method")
|
||||
}
|
||||
|
||||
func (n NvidiaModel) ListModels(apiConfig *APIConfig) ([]string, error) {
|
||||
return nil, fmt.Errorf("no such method")
|
||||
}
|
||||
|
||||
func (n NvidiaModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
|
||||
return nil, fmt.Errorf("no such method")
|
||||
}
|
||||
|
||||
func (n NvidiaModel) CheckConnection(apiConfig *APIConfig) error {
|
||||
return fmt.Errorf("no such method")
|
||||
}
|
||||
Reference in New Issue
Block a user