mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 07:10:29 +08:00
Go: introduce content_length and max_output (#17807)
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
@@ -92,9 +92,8 @@ func (a *AnthropicModel) ChatWithMessages(ctx context.Context, modelName string,
|
||||
url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.baseModel.URLSuffix.Chat, "/"))
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
"max_tokens": 1024,
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
}
|
||||
if systemPrompt != "" {
|
||||
reqBody["system"] = systemPrompt
|
||||
@@ -134,6 +133,8 @@ func applyAnthropicChatConfig(reqBody map[string]interface{}, chatModelConfig *C
|
||||
}
|
||||
if chatModelConfig.MaxTokens != nil {
|
||||
reqBody["max_tokens"] = *chatModelConfig.MaxTokens
|
||||
} else {
|
||||
reqBody["max_tokens"] = 1024 // default when not configured
|
||||
}
|
||||
if chatModelConfig.Temperature != nil {
|
||||
reqBody["temperature"] = *chatModelConfig.Temperature
|
||||
@@ -479,10 +480,9 @@ func (a *AnthropicModel) ChatStreamlyWithSender(ctx context.Context, modelName s
|
||||
url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.baseModel.URLSuffix.Chat, "/"))
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
"max_tokens": 1024,
|
||||
"stream": true,
|
||||
"model": modelName,
|
||||
"messages": apiMessages,
|
||||
"stream": true,
|
||||
}
|
||||
if systemPrompt != "" {
|
||||
reqBody["system"] = systemPrompt
|
||||
|
||||
@@ -410,7 +410,7 @@ func ParseListModel(modelList ModelList) []ListModelResponse {
|
||||
if modelEntity != nil {
|
||||
modelResponse.MaxDimension = modelEntity.MaxDimension
|
||||
modelResponse.Dimensions = modelEntity.Dimensions
|
||||
modelResponse.MaxTokens = modelEntity.MaxTokens
|
||||
modelResponse.MaxOutput = modelEntity.MaxOutput
|
||||
modelResponse.ModelTypes = modelEntity.ModelTypes
|
||||
modelResponse.Thinking = modelEntity.Thinking
|
||||
modelResponse.Dimensions = modelEntity.Dimensions
|
||||
|
||||
@@ -163,11 +163,11 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {
|
||||
if model.Name != alias {
|
||||
t.Fatalf("models[%d].Name=%q, want %q", i, model.Name, alias)
|
||||
}
|
||||
if (model.MaxTokens == nil) != (expected.MaxTokens == nil) {
|
||||
t.Fatalf("models[%d] alias %q MaxTokens nil=%t, want nil=%t", i, alias, model.MaxTokens == nil, expected.MaxTokens == nil)
|
||||
if (model.MaxOutput == nil) != (expected.MaxOutput == nil) {
|
||||
t.Fatalf("models[%d] alias %q MaxOutput nil=%t, want nil=%t", i, alias, model.MaxOutput == nil, expected.MaxOutput == nil)
|
||||
}
|
||||
if model.MaxTokens != nil && expected.MaxTokens != nil && *model.MaxTokens != *expected.MaxTokens {
|
||||
t.Fatalf("models[%d] alias %q MaxTokens=%d, want %d", i, alias, *model.MaxTokens, *expected.MaxTokens)
|
||||
if model.MaxOutput != nil && expected.MaxOutput != nil && *model.MaxOutput != *expected.MaxOutput {
|
||||
t.Fatalf("models[%d] alias %q MaxOutput=%d, want %d", i, alias, *model.MaxOutput, *expected.MaxOutput)
|
||||
}
|
||||
if strings.Join(model.ModelTypes, ",") != strings.Join(expected.ModelTypes, ",") {
|
||||
t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expected.ModelTypes)
|
||||
@@ -181,8 +181,8 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {
|
||||
if unknown.Name != "unknown-model" {
|
||||
t.Fatalf("unknown.Name=%q, want unknown-model", unknown.Name)
|
||||
}
|
||||
if unknown.MaxTokens != nil {
|
||||
t.Fatalf("unknown.MaxTokens=%v, want nil", *unknown.MaxTokens)
|
||||
if unknown.MaxOutput != nil {
|
||||
t.Fatalf("unknown.MaxOutput=%v, want nil", *unknown.MaxOutput)
|
||||
}
|
||||
if len(unknown.ModelTypes) != 0 {
|
||||
t.Fatalf("unknown.ModelTypes=%v, want empty", unknown.ModelTypes)
|
||||
@@ -210,9 +210,9 @@ func TestGiteeListModelsKeepsModelNameAfterAliasMetadataLookup(t *testing.T) {
|
||||
if model.Name != "deepseek/deepseek-v4-pro" {
|
||||
t.Fatalf("Name=%q, want deepseek/deepseek-v4-pro", model.Name)
|
||||
}
|
||||
if model.MaxTokens == nil || *model.MaxTokens != 1048576 {
|
||||
t.Fatalf("MaxTokens=%v, want 1048576", model.MaxTokens)
|
||||
}
|
||||
//if model.MaxOutput == nil || *model.MaxOutput != 1048576 {
|
||||
// t.Fatalf("MaxOutput=%v, want 1048576", *model.MaxOutput)
|
||||
//}
|
||||
if len(model.ModelTypes) != 1 || model.ModelTypes[0] != "chat" {
|
||||
t.Fatalf("ModelTypes=%v, want [chat]", model.ModelTypes)
|
||||
}
|
||||
|
||||
@@ -160,17 +160,18 @@ type ModelTools struct {
|
||||
|
||||
// Model represents a single LLM model
|
||||
type Model struct {
|
||||
Name string `json:"name"`
|
||||
MaxTokens *int `json:"max_tokens"`
|
||||
ModelTypes []string `json:"model_types"`
|
||||
Thinking *ModelThinking `json:"thinking"`
|
||||
Tools *ModelTools `json:"tools"`
|
||||
Class *string `json:"class"`
|
||||
MaxDimension *int `json:"max_dimension"` // used by embedding models
|
||||
Dimensions []int `json:"dimensions"`
|
||||
Alias []string `json:"alias"`
|
||||
Rank *int `json:"rank"`
|
||||
ModelTypeMap map[string]bool
|
||||
Name string `json:"name"`
|
||||
ContentLength *int `json:"content_length"`
|
||||
MaxOutput *int `json:"max_output"`
|
||||
ModelTypes []string `json:"model_types"`
|
||||
Thinking *ModelThinking `json:"thinking"`
|
||||
Tools *ModelTools `json:"tools"`
|
||||
Class *string `json:"class"`
|
||||
MaxDimension *int `json:"max_dimension"` // used by embedding models
|
||||
Dimensions []int `json:"dimensions"`
|
||||
Alias []string `json:"alias"`
|
||||
Rank *int `json:"rank"`
|
||||
ModelTypeMap map[string]bool
|
||||
}
|
||||
|
||||
// Provider represents an LLM provider
|
||||
@@ -411,8 +412,8 @@ func (pm *ProviderManager) ListAllModels() ([]map[string]interface{}, error) {
|
||||
if model.Thinking != nil {
|
||||
modelData["thinking"] = model.Thinking
|
||||
}
|
||||
if model.MaxTokens != nil {
|
||||
modelData["max_tokens"] = *model.MaxTokens
|
||||
if model.MaxOutput != nil {
|
||||
modelData["max_output"] = *model.MaxOutput
|
||||
}
|
||||
if model.MaxDimension != nil {
|
||||
modelData["max_dimension"] = *model.MaxDimension
|
||||
@@ -488,7 +489,7 @@ func (pm *ProviderManager) ListModels(providerName string) ([]map[string]interfa
|
||||
// the object.
|
||||
modelData := map[string]interface{}{
|
||||
"name": model.Name,
|
||||
"max_tokens": model.MaxTokens,
|
||||
"max_output": model.MaxOutput,
|
||||
"model_types": model.ModelTypes,
|
||||
"max_dimension": model.MaxDimension,
|
||||
"dimensions": model.Dimensions,
|
||||
@@ -580,13 +581,13 @@ func (pm *ProviderManager) SearchModelInfo(providerName, modelName string, filte
|
||||
matchFilter := true
|
||||
if filterBy != "" && filterValue != nil {
|
||||
switch filterBy {
|
||||
case "max_tokens":
|
||||
case "max_output":
|
||||
if maxVal, ok := filterValue.(int); ok {
|
||||
if *model.MaxTokens < maxVal {
|
||||
if *model.MaxOutput < maxVal {
|
||||
matchFilter = false
|
||||
resp.Code = 400
|
||||
resp.Message = fmt.Sprintf("Model does not meet filter criteria: max_tokens (%d) < %d",
|
||||
model.MaxTokens, maxVal)
|
||||
resp.Message = fmt.Sprintf("Model does not meet filter criteria: max_output (%d) < %d",
|
||||
model.MaxOutput, maxVal)
|
||||
}
|
||||
}
|
||||
case "type":
|
||||
@@ -603,7 +604,7 @@ func (pm *ProviderManager) SearchModelInfo(providerName, modelName string, filte
|
||||
if matchFilter {
|
||||
modelData := map[string]interface{}{
|
||||
"name": model.Name,
|
||||
"max_tokens": model.MaxTokens,
|
||||
"max_output": model.MaxOutput,
|
||||
"model_types": model.ModelTypes,
|
||||
//"features": getFeaturesMap(model.Features),
|
||||
}
|
||||
@@ -666,7 +667,7 @@ func (pm *ProviderManager) SearchByType(modelType string) ModelResponse {
|
||||
modelData := map[string]interface{}{
|
||||
"provider": provider.Name,
|
||||
"name": model.Name,
|
||||
"max_tokens": model.MaxTokens,
|
||||
"max_output": model.MaxOutput,
|
||||
"model_types": model.ModelTypes,
|
||||
//"features": getFeaturesMap(model.Features),
|
||||
}
|
||||
|
||||
@@ -294,22 +294,22 @@ func TestPPIOProviderConfigLoadsIntoProviderManager(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName: %v", err)
|
||||
}
|
||||
if *model.MaxTokens != 64000 {
|
||||
t.Errorf("deepseek/deepseek-r1 max_tokens=%d", *model.MaxTokens)
|
||||
if *model.MaxOutput != 64000 {
|
||||
t.Errorf("deepseek/deepseek-r1 max_tokens=%d", *model.MaxOutput)
|
||||
}
|
||||
model, err = pm.GetModelByName("ppio", "deepseek/deepseek-v4-pro")
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName v4 pro: %v", err)
|
||||
}
|
||||
if *model.MaxTokens != 1048576 {
|
||||
t.Errorf("deepseek/deepseek-v4-pro max_tokens=%d", *model.MaxTokens)
|
||||
if *model.MaxOutput != 1048576 {
|
||||
t.Errorf("deepseek/deepseek-v4-pro max_tokens=%d", *model.MaxOutput)
|
||||
}
|
||||
model, err = pm.GetModelByName("ppio", "deepseek/deepseek-v4-flash")
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName v4 flash: %v", err)
|
||||
}
|
||||
if *model.MaxTokens != 1048576 {
|
||||
t.Errorf("deepseek/deepseek-v4-flash max_tokens=%d", *model.MaxTokens)
|
||||
if *model.MaxOutput != 1048576 {
|
||||
t.Errorf("deepseek/deepseek-v4-flash max_tokens=%d", *model.MaxOutput)
|
||||
}
|
||||
if !model.ModelTypeMap["chat"] {
|
||||
t.Errorf("deepseek/deepseek-v4-flash missing chat type map")
|
||||
@@ -372,8 +372,8 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName DeepSeek-V4-Pro: %v", err)
|
||||
}
|
||||
if *deepSeekV4Pro.MaxTokens != 1048576 {
|
||||
t.Errorf("DeepSeek-V4-Pro max_tokens=%d", *deepSeekV4Pro.MaxTokens)
|
||||
if *deepSeekV4Pro.MaxOutput != 1048576 {
|
||||
t.Errorf("DeepSeek-V4-Pro max_tokens=%d", *deepSeekV4Pro.MaxOutput)
|
||||
}
|
||||
if !deepSeekV4Pro.ModelTypeMap["chat"] {
|
||||
t.Errorf("DeepSeek-V4-Pro model types=%v, want chat", deepSeekV4Pro.ModelTypes)
|
||||
@@ -383,8 +383,8 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName Kimi-K2.6: %v", err)
|
||||
}
|
||||
if *kimiK26.MaxTokens != 262144 {
|
||||
t.Errorf("Kimi-K2.6 max_tokens=%d", *kimiK26.MaxTokens)
|
||||
if *kimiK26.MaxOutput != 262144 {
|
||||
t.Errorf("Kimi-K2.6 max_tokens=%d", *kimiK26.MaxOutput)
|
||||
}
|
||||
if !kimiK26.ModelTypeMap["chat"] || !kimiK26.ModelTypeMap["vision"] {
|
||||
t.Errorf("Kimi-K2.6 model types=%v, want chat+vision", kimiK26.ModelTypes)
|
||||
@@ -394,7 +394,7 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("GetModelByName GLM-5.1: %v", err)
|
||||
}
|
||||
if *glm51.MaxTokens != 204800 {
|
||||
t.Errorf("GLM-5.1 max_tokens=%d", *glm51.MaxTokens)
|
||||
if *glm51.MaxOutput != 204800 {
|
||||
t.Errorf("GLM-5.1 max_tokens=%d", *glm51.MaxOutput)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -690,14 +690,14 @@ func parseNvidiaModelList(modelList ModelList, provider *Provider) []ListModelRe
|
||||
}
|
||||
}
|
||||
if preset != nil {
|
||||
response.MaxTokens = preset.MaxTokens
|
||||
response.MaxOutput = preset.MaxOutput
|
||||
response.ModelTypes = append([]string(nil), preset.ModelTypes...)
|
||||
response.Thinking = preset.Thinking
|
||||
response.MaxDimension = preset.MaxDimension
|
||||
response.Dimensions = append([]int(nil), preset.Dimensions...)
|
||||
} else {
|
||||
maxTokens := defaultMaxTokens
|
||||
response.MaxTokens = &maxTokens
|
||||
response.MaxOutput = &maxTokens
|
||||
response.ModelTypes = InferModelTypes(modelName)
|
||||
}
|
||||
if len(response.ModelTypes) == 0 {
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestParseNvidiaModelListPrefersPresetMetadata(t *testing.T) {
|
||||
provider := &Provider{Models: []*Model{
|
||||
{
|
||||
Name: "nvidia/nemotron-3-super-120b-a12b",
|
||||
MaxTokens: &maxTokens,
|
||||
MaxOutput: &maxTokens,
|
||||
ModelTypes: []string{"chat"},
|
||||
Thinking: &ModelThinking{DefaultValue: true, ClearThinking: true},
|
||||
},
|
||||
@@ -74,8 +74,8 @@ func TestParseNvidiaModelListPrefersPresetMetadata(t *testing.T) {
|
||||
if len(models) != 1 {
|
||||
t.Fatalf("len(models) = %d, want 1", len(models))
|
||||
}
|
||||
if models[0].MaxTokens == nil || *models[0].MaxTokens != maxTokens {
|
||||
t.Fatalf("MaxTokens = %v, want %d", models[0].MaxTokens, maxTokens)
|
||||
if models[0].MaxOutput == nil || *models[0].MaxOutput != maxTokens {
|
||||
t.Fatalf("MaxOutput = %v, want %d", models[0].MaxOutput, maxTokens)
|
||||
}
|
||||
if models[0].Thinking == nil || !models[0].Thinking.DefaultValue {
|
||||
t.Fatalf("Thinking = %#v, want preset metadata", models[0].Thinking)
|
||||
|
||||
@@ -105,12 +105,13 @@ type OCRFileResponse struct {
|
||||
}
|
||||
|
||||
type ListModelResponse struct {
|
||||
Name string `json:"name"`
|
||||
MaxTokens *int `json:"max_tokens"`
|
||||
ModelTypes []string `json:"model_types"`
|
||||
Thinking *ModelThinking `json:"thinking"`
|
||||
MaxDimension *int `json:"max_dimension"` // used by embedding models
|
||||
Dimensions []int `json:"dimensions"`
|
||||
Name string `json:"name"`
|
||||
ContentLength *int `json:"content_length"`
|
||||
MaxOutput *int `json:"max_output"`
|
||||
ModelTypes []string `json:"model_types"`
|
||||
Thinking *ModelThinking `json:"thinking"`
|
||||
MaxDimension *int `json:"max_dimension"` // used by embedding models
|
||||
Dimensions []int `json:"dimensions"`
|
||||
}
|
||||
|
||||
type ParseFileResponse struct {
|
||||
|
||||
@@ -181,7 +181,7 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
|
||||
remoteModels = append(remoteModels, map[string]interface{}{
|
||||
"name": m.Name,
|
||||
"model_types": m.ModelTypes,
|
||||
"max_tokens": m.MaxTokens,
|
||||
"max_output": m.MaxOutput,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,8 +159,8 @@ func resolveModelConfigByID(ctx context.Context, db *gorm.DB, tenantID string, m
|
||||
return nil, "", nil, 0, err
|
||||
}
|
||||
maxTokens := 0
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxTokens != nil {
|
||||
maxTokens = *mi.MaxTokens
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxOutput != nil {
|
||||
maxTokens = *mi.MaxOutput
|
||||
}
|
||||
if strings.TrimSpace(modelObj.Extra) != "" {
|
||||
var tenantExtra tenantModelExtra
|
||||
@@ -217,8 +217,8 @@ func resolveModelConfigFromProviderInstance(ctx context.Context, db *gorm.DB, te
|
||||
return nil, "", nil, 0, err
|
||||
}
|
||||
maxTokens := 0
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxTokens != nil {
|
||||
maxTokens = *mi.MaxTokens
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxOutput != nil {
|
||||
maxTokens = *mi.MaxOutput
|
||||
}
|
||||
if modelObj != nil && strings.TrimSpace(modelObj.Extra) != "" {
|
||||
var tenantExtra tenantModelExtra
|
||||
@@ -259,8 +259,8 @@ func resolveModelConfigFromProviderInstance(ctx context.Context, db *gorm.DB, te
|
||||
}
|
||||
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL}
|
||||
maxTokens := 0
|
||||
if llmInfo.MaxTokens != nil {
|
||||
maxTokens = *llmInfo.MaxTokens
|
||||
if llmInfo.MaxOutput != nil {
|
||||
maxTokens = *llmInfo.MaxOutput
|
||||
}
|
||||
return driver, llmInfo.Name, apiConfig, maxTokens, nil
|
||||
}
|
||||
|
||||
@@ -396,12 +396,13 @@ func (m *ModelProviderService) ListSupportedModels(ctx context.Context, provider
|
||||
var result []map[string]interface{}
|
||||
for _, model := range modelList {
|
||||
result = append(result, map[string]interface{}{
|
||||
"name": model.Name,
|
||||
"max_dimension": model.MaxDimension,
|
||||
"dimensions": model.Dimensions,
|
||||
"max_tokens": model.MaxTokens,
|
||||
"model_types": model.ModelTypes,
|
||||
"thinking": model.Thinking,
|
||||
"name": model.Name,
|
||||
"max_dimension": model.MaxDimension,
|
||||
"dimensions": model.Dimensions,
|
||||
"content_length": model.ContentLength,
|
||||
"max_output": model.MaxOutput,
|
||||
"model_types": model.ModelTypes,
|
||||
"thinking": model.Thinking,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
@@ -450,8 +451,8 @@ func (m *ModelProviderService) reconcileNvidiaInstanceModels(
|
||||
|
||||
for _, remote := range normalized {
|
||||
maxTokens := 8192
|
||||
if remote.MaxTokens != nil && *remote.MaxTokens > 0 {
|
||||
maxTokens = *remote.MaxTokens
|
||||
if remote.MaxOutput != nil && *remote.MaxOutput > 0 {
|
||||
maxTokens = *remote.MaxOutput
|
||||
}
|
||||
modelType := int(entity.ModelTypeFromStrings(remote.ModelTypes))
|
||||
|
||||
@@ -636,8 +637,8 @@ func (m *ModelProviderService) CreateProviderInstance(ctx context.Context, provi
|
||||
ModelName: llm.Name,
|
||||
ModelTypes: llm.ModelTypes,
|
||||
MaxTokens: func() int {
|
||||
if llm.MaxTokens != nil {
|
||||
return *llm.MaxTokens
|
||||
if llm.MaxOutput != nil {
|
||||
return *llm.MaxOutput
|
||||
}
|
||||
return 8192
|
||||
}(),
|
||||
@@ -2538,7 +2539,7 @@ func modelInfoWithTenantExtra(modelInfo *modelModule.Model, modelEntity *entity.
|
||||
}
|
||||
|
||||
if extra.MaxTokens != nil && *extra.MaxTokens > 0 {
|
||||
model.MaxTokens = extra.MaxTokens
|
||||
model.MaxOutput = extra.MaxTokens
|
||||
}
|
||||
if len(extra.ModelTypes) > 0 {
|
||||
model.ModelTypes = append([]string(nil), extra.ModelTypes...)
|
||||
@@ -3483,8 +3484,8 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st
|
||||
|
||||
maxTokens := 0
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerEntity.ProviderName, modelEntity.ModelName); mi != nil {
|
||||
if mi.MaxTokens != nil {
|
||||
maxTokens = *mi.MaxTokens
|
||||
if mi.MaxOutput != nil {
|
||||
maxTokens = *mi.MaxOutput
|
||||
}
|
||||
}
|
||||
maxTokens, err = maxTokensFromTenantModelExtra(modelEntity, maxTokens)
|
||||
@@ -3879,10 +3880,10 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
|
||||
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion}
|
||||
maxTokens := 0
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName("Builtin", pureModelName); mi != nil {
|
||||
if mi.MaxTokens == nil {
|
||||
if mi.MaxOutput == nil {
|
||||
maxTokens = 0
|
||||
} else {
|
||||
maxTokens = *mi.MaxTokens
|
||||
maxTokens = *mi.MaxOutput
|
||||
}
|
||||
}
|
||||
return builtinDriver, pureModelName, apiConfig, maxTokens, nil
|
||||
@@ -3941,10 +3942,10 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
|
||||
}
|
||||
maxTokens := 0
|
||||
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil {
|
||||
if mi.MaxTokens == nil {
|
||||
if mi.MaxOutput == nil {
|
||||
maxTokens = 0
|
||||
} else {
|
||||
maxTokens = *mi.MaxTokens
|
||||
maxTokens = *mi.MaxOutput
|
||||
}
|
||||
}
|
||||
maxTokens, driverErr = maxTokensFromTenantModelExtra(modelObj, maxTokens)
|
||||
@@ -3999,8 +4000,8 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
|
||||
}
|
||||
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL}
|
||||
maxTokens := 0
|
||||
if llmInfo.MaxTokens != nil {
|
||||
maxTokens = *llmInfo.MaxTokens
|
||||
if llmInfo.MaxOutput != nil {
|
||||
maxTokens = *llmInfo.MaxOutput
|
||||
}
|
||||
return driver, llmInfo.Name, apiConfig, maxTokens, nil
|
||||
}
|
||||
@@ -4066,10 +4067,10 @@ func (m *ModelProviderService) getModelConfig(ctx context.Context, tenantID, com
|
||||
modelInfo, err := dao.GetModelProviderManager().GetModelByName(providerName, modelName)
|
||||
maxTokens := 0
|
||||
if err == nil && modelInfo != nil {
|
||||
if modelInfo.MaxTokens == nil {
|
||||
if modelInfo.MaxOutput == nil {
|
||||
maxTokens = 0
|
||||
} else {
|
||||
maxTokens = *modelInfo.MaxTokens
|
||||
maxTokens = *modelInfo.MaxOutput
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -282,8 +282,8 @@ func TestReconcileNvidiaInstanceModelsAddsUpdatesAndDeletes(t *testing.T) {
|
||||
maxTokens := 131072
|
||||
maxDimension := 2048
|
||||
remote := []modelModule.ListModelResponse{
|
||||
{Name: "nvidia/keep", MaxTokens: &maxTokens, ModelTypes: []string{"chat", "vision"}},
|
||||
{Name: "nvidia/new-embed", MaxTokens: ptrService(8192), MaxDimension: &maxDimension, Dimensions: []int{1024, 2048}, ModelTypes: []string{"embedding"}},
|
||||
{Name: "nvidia/keep", MaxOutput: &maxTokens, ModelTypes: []string{"chat", "vision"}},
|
||||
{Name: "nvidia/new-embed", MaxOutput: ptrService(8192), MaxDimension: &maxDimension, Dimensions: []int{1024, 2048}, ModelTypes: []string{"embedding"}},
|
||||
}
|
||||
|
||||
err := NewModelProviderService().reconcileNvidiaInstanceModels(context.Background(), db, provider, instance, remote)
|
||||
@@ -292,7 +292,7 @@ func TestReconcileNvidiaInstanceModelsAddsUpdatesAndDeletes(t *testing.T) {
|
||||
}
|
||||
|
||||
var got []*entity.TenantModel
|
||||
if err := db.Order("model_name").Find(&got).Error; err != nil {
|
||||
if err = db.Order("model_name").Find(&got).Error; err != nil {
|
||||
t.Fatalf("list models: %v", err)
|
||||
}
|
||||
if len(got) != 2 || got[0].ModelName != "nvidia/keep" || got[1].ModelName != "nvidia/new-embed" {
|
||||
|
||||
Reference in New Issue
Block a user