diff --git a/conf/models/nvidia.json b/conf/models/nvidia.json new file mode 100644 index 0000000000..8ba81f1fd3 --- /dev/null +++ b/conf/models/nvidia.json @@ -0,0 +1,461 @@ +{ + "name": "Nvidia", + "url": { + "default": "https://integrate.api.nvidia.com/v1" + }, + "url_suffix": { + "chat": "chat/completions", + "models": "models" + }, + "class": "nvidia", + "models": [ + { + "name": "abacusai/dracarys-llama-3.1-70b-instruct", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "bytedance/seed-oss-36b-instruct", + "max_tokens": 32768, + "model_types": [ + "chat" + ] + }, + { + "name": "deepseek-ai/deepseek-v4-flash", + "max_tokens": 1048576, + "model_types": [ + "chat" + ] + }, + { + "name": "deepseek-ai/deepseek-v4-pro", + "max_tokens": 1048576, + "model_types": [ + "chat" + ] + }, + { + "name": "deepseek-ai/deepseek-v3.2", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "deepseek-ai/deepseek-v3.1", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "google/codegemma-7b", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "google/gemma-2-2b-it", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "google/gemma-4-31b-it", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "google/gemma-7b", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "ibm/granite-3.3-8b-instruct", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "meta/llama-3.1-405b-instruct", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "meta/llama-3.2-90b-vision-instruct", + "max_tokens": 131072, + "model_types": [ + "chat", + "vision" + ] + }, + { + "name": "meta/llama-4-maverick-17b-128e-instruct", + "max_tokens": 1048576, + "model_types": [ + "chat" + ] + }, + { + "name": "microsoft/phi-4-mini-flash-reasoning", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "minimaxai/minimax-m2.1", + "max_tokens": 204800, + "model_types": [ + "chat" + ] + }, + { + "name": "minimaxai/minimax-m2.5", + "max_tokens": 204800, + "model_types": [ + "chat" + ] + }, + { + "name": "minimaxai/minimax-m2.7", + "max_tokens": 204800, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/devstral-2-123b-instruct-2512", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/magistral-small-2506", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/mistral-7b-instruct-v0.3", + "max_tokens": 32768, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/mistral-large-3-675b-instruct-2512", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/mistral-medium-3-5-128b", + "max_tokens": 131072, + "model_types": [ + "chat", + "vision" + ] + }, + { + "name": "mistralai/mistral-nemotron", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "mistralai/mixtral-8x22b-instruct", + "max_tokens": 65536, + "model_types": [ + "chat" + ] + }, + { + "name": "moonshotai/kimi-k2.5", + "max_tokens": 262144, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "moonshotai/kimi-k2.6", + "max_tokens": 262144, + "model_types": [ + "chat", + "vision" + ] + }, + { + "name": "moonshotai/kimi-k2-instruct", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "moonshotai/kimi-k2-instruct-0905", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "moonshotai/kimi-k2-thinking", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "nvidia/gliner-pii", + "max_tokens": 4096, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.1-nemoguard-8b-content-safety", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.1-nemoguard-8b-topic-control", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.1-nemotron-nano-8b-v1", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.1-nemotron-ultra-253b-v1", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "nvidia/llama-3.2-nemoretriever-1b-vlm-embed-v1", + "max_tokens": 8192, + "model_types": [ + "embedding" + ] + }, + { + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "nvidia/nemoguard-jailbreak-detect", + "max_tokens": 4096, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/nemotron-3-nano-30b-a3b", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "max_tokens": 131072, + "model_types": [ + "chat", + "vision" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "nvidia/nemotron-3-super-120b-a12b", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/nemotron-content-safety-reasoning-4b", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/nemotron-mini-4b-instruct", + "max_tokens": 4096, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/nvidia-nemotron-nano-9b-v2", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/riva-translate-4b-instruct-v1_1", + "max_tokens": 4096, + "model_types": [ + "chat" + ] + }, + { + "name": "nvidia/usdcode", + "max_tokens": 8192, + "model_types": [ + "chat" + ] + }, + { + "name": "openai/gpt-oss-120b", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "qwen/qwen2.5-coder-7b-instruct", + "max_tokens": 32768, + "model_types": [ + "chat" + ] + }, + { + "name": "qwen/qwen3-5-122b-a10b", + "max_tokens": 131072, + "model_types": [ + "chat" + ] + }, + { + "name": "qwen/qwen3-235b-a22b", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "qwen/qwen3-coder-480b-a35b-instruct", + "max_tokens": 262144, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "z-ai/glm-5", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "z-ai/glm-5.1", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + }, + { + "name": "z-ai/glm-4.7", + "max_tokens": 131072, + "model_types": [ + "chat" + ], + "thinking": { + "default_value": true, + "clear_thinking": true + } + } + ] +} \ No newline at end of file diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go index 5d87b2f643..fac17dbf4a 100644 --- a/internal/cli/user_command.go +++ b/internal/cli/user_command.go @@ -21,7 +21,10 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" + "io" + "net" netUrl "net/url" "os" ce "ragflow/internal/cli/filesystem" @@ -1755,7 +1758,16 @@ func (c *RAGFlowClient) ChatToModel(cmd *Command) (ResponseIf, error) { resp, err := c.HTTPClient.Request("POST", url, true, "web", nil, payload) if err != nil { - return nil, fmt.Errorf("failed to list instance models: %w", err) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return nil, fmt.Errorf("connection closed (EOF): upstream overloaded or proxy timeout: %w", err) + } + + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return nil, fmt.Errorf("request timeout: model took too long to respond: %w", err) + } + + return nil, fmt.Errorf("request failed: %w", err) } if resp.StatusCode != 200 { diff --git a/internal/entity/models/factory.go b/internal/entity/models/factory.go index 9efd33e472..dfb5854a28 100644 --- a/internal/entity/models/factory.go +++ b/internal/entity/models/factory.go @@ -57,8 +57,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string return NewXAIModel(baseURL, urlSuffix), nil case "lmstudio": return NewLmStudioModel(baseURL, urlSuffix), nil - case "openai": - return NewOpenAIModel(baseURL, urlSuffix), nil + case "nvidia": + return NewNvidiaModel(baseURL, urlSuffix), nil default: return NewDummyModel(baseURL, urlSuffix), nil } diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.go new file mode 100644 index 0000000000..2700ab86b6 --- /dev/null +++ b/internal/entity/models/nvidia.go @@ -0,0 +1,345 @@ +package models + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// NvidiaModel implements ModelDriver for Nvidia +type NvidiaModel struct { + BaseURL map[string]string + URLSuffix URLSuffix + httpClient *http.Client +} + +// NewNvidiaModel creates a new Nvidia model instance +func NewNvidiaModel(baseURL map[string]string, urlSuffix URLSuffix) *NvidiaModel { + return &NvidiaModel{ + BaseURL: baseURL, + URLSuffix: urlSuffix, + httpClient: &http.Client{ + Timeout: 120 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + DisableCompression: false, + }, + }, + } +} + +func (n NvidiaModel) NewInstance(baseURL map[string]string) ModelDriver { + return &NvidiaModel{ + BaseURL: baseURL, + URLSuffix: n.URLSuffix, + httpClient: &http.Client{ + Timeout: 120 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + DisableCompression: false, + }, + }, + } +} + +func (n NvidiaModel) Name() string { + return "nvidia" +} + +func (n *NvidiaModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) { + if len(messages) == 0 { + return nil, fmt.Errorf("messages is empty") + } + + var region = "default" + if apiConfig != nil && apiConfig.Region != nil { + region = *apiConfig.Region + } + + baseURL := n.BaseURL[region] + if baseURL == "" { + baseURL = n.BaseURL["default"] + } + url := fmt.Sprintf("%s/%s", baseURL, n.URLSuffix.Chat) + + apiMessages := make([]map[string]interface{}, len(messages)) + for i, msg := range messages { + apiMessages[i] = map[string]interface{}{ + "role": msg.Role, + "content": msg.Content, + } + } + + reqBody := map[string]interface{}{ + "model": modelName, + "messages": apiMessages, + "stream": false, + } + + if chatModelConfig != nil { + if chatModelConfig.Stream != nil { + reqBody["stream"] = *chatModelConfig.Stream + } + if chatModelConfig.MaxTokens != nil { + reqBody["max_tokens"] = *chatModelConfig.MaxTokens + } + if chatModelConfig.Temperature != nil { + reqBody["temperature"] = *chatModelConfig.Temperature + } + if chatModelConfig.TopP != nil { + reqBody["top_p"] = *chatModelConfig.TopP + } + if chatModelConfig.Stop != nil { + reqBody["stop"] = *chatModelConfig.Stop + } + if chatModelConfig.Thinking != nil { + if *chatModelConfig.Thinking { + reqBody["thinking"] = map[string]interface{}{"type": "enabled"} + } else { + reqBody["thinking"] = map[string]interface{}{"type": "disabled"} + } + } + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if apiConfig != nil && apiConfig.ApiKey != nil { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + } + + resp, err := n.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + var result map[string]interface{} + if err = json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + choices, ok := result["choices"].([]interface{}) + if !ok || len(choices) == 0 { + return nil, fmt.Errorf("no choices in response") + } + + firstChoice, ok := choices[0].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid choice format") + } + + messageMap, ok := firstChoice["message"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid message format") + } + + content, ok := messageMap["content"].(string) + if !ok { + return nil, fmt.Errorf("invalid content format") + } + + var modelClass *string + if chatModelConfig != nil { + modelClass = chatModelConfig.ModelClass + } + thinking, answer := GetThinkingAndAnswer(modelClass, &content) + + chatResponse := &ChatResponse{ + Answer: answer, + ReasonContent: thinking, + } + + return chatResponse, nil +} + +func (n *NvidiaModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error { + if len(messages) == 0 { + return fmt.Errorf("messages is empty") + } + + var region = "default" + if apiConfig != nil && apiConfig.Region != nil { + region = *apiConfig.Region + } + + baseURL := n.BaseURL[region] + if baseURL == "" { + baseURL = n.BaseURL["default"] + } + url := fmt.Sprintf("%s/%s", baseURL, n.URLSuffix.Chat) + + apiMessages := make([]map[string]interface{}, len(messages)) + for i, msg := range messages { + apiMessages[i] = map[string]interface{}{ + "role": msg.Role, + "content": msg.Content, + } + } + + reqBody := map[string]interface{}{ + "model": modelName, + "messages": apiMessages, + "stream": true, + } + + if modelConfig != nil { + if modelConfig.Stream != nil { + reqBody["stream"] = *modelConfig.Stream + } + if modelConfig.MaxTokens != nil { + reqBody["max_tokens"] = *modelConfig.MaxTokens + } + if modelConfig.Temperature != nil { + reqBody["temperature"] = *modelConfig.Temperature + } + if modelConfig.DoSample != nil { + reqBody["do_sample"] = *modelConfig.DoSample + } + if modelConfig.TopP != nil { + reqBody["top_p"] = *modelConfig.TopP + } + if modelConfig.Stop != nil { + reqBody["stop"] = *modelConfig.Stop + } + if modelConfig.Thinking != nil { + if *modelConfig.Thinking { + reqBody["thinking"] = map[string]interface{}{"type": "enabled"} + } else { + reqBody["thinking"] = map[string]interface{}{"type": "disabled"} + } + } + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if apiConfig != nil && apiConfig.ApiKey != nil { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey)) + } + + resp, err := n.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + + if !strings.HasPrefix(line, "data:") { + continue + } + + data := strings.TrimSpace(line[5:]) + if data == "[DONE]" { + break + } + + var event map[string]interface{} + if err = json.Unmarshal([]byte(data), &event); err != nil { + continue + } + + choices, ok := event["choices"].([]interface{}) + if !ok || len(choices) == 0 { + continue + } + + firstChoice, ok := choices[0].(map[string]interface{}) + if !ok { + continue + } + + delta, ok := firstChoice["delta"].(map[string]interface{}) + if !ok { + continue + } + + reasoningContent, ok := delta["reasoning_content"].(string) + if ok && reasoningContent != "" { + if err := sender(nil, &reasoningContent); err != nil { + return err + } + } + + content, ok := delta["content"].(string) + if ok && content != "" { + if err := sender(&content, nil); err != nil { + return err + } + } + + finishReason, ok := firstChoice["finish_reason"].(string) + if ok && finishReason != "" { + break + } + } + + endOfStream := "[DONE]" + if err = sender(&endOfStream, nil); err != nil { + return err + } + + return scanner.Err() +} + +func (n NvidiaModel) Encode(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([][]float64, error) { + return nil, fmt.Errorf("no such method") +} + +func (n NvidiaModel) Rerank(modelName *string, query string, texts []string, apiConfig *APIConfig) ([]float64, error) { + return nil, fmt.Errorf("no such method") +} + +func (n NvidiaModel) ListModels(apiConfig *APIConfig) ([]string, error) { + return nil, fmt.Errorf("no such method") +} + +func (n NvidiaModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) { + return nil, fmt.Errorf("no such method") +} + +func (n NvidiaModel) CheckConnection(apiConfig *APIConfig) error { + return fmt.Errorf("no such method") +}