Go: introduce content_length and max_output (#17807)

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-08-04 19:08:31 +08:00
committed by GitHub
parent f707cca074
commit c1f960cd47
19 changed files with 254 additions and 148 deletions

View File

@@ -92,9 +92,8 @@ func (a *AnthropicModel) ChatWithMessages(ctx context.Context, modelName string,
url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.baseModel.URLSuffix.Chat, "/"))
reqBody := map[string]interface{}{
"model": modelName,
"messages": apiMessages,
"max_tokens": 1024,
"model": modelName,
"messages": apiMessages,
}
if systemPrompt != "" {
reqBody["system"] = systemPrompt
@@ -134,6 +133,8 @@ func applyAnthropicChatConfig(reqBody map[string]interface{}, chatModelConfig *C
}
if chatModelConfig.MaxTokens != nil {
reqBody["max_tokens"] = *chatModelConfig.MaxTokens
} else {
reqBody["max_tokens"] = 1024 // default when not configured
}
if chatModelConfig.Temperature != nil {
reqBody["temperature"] = *chatModelConfig.Temperature
@@ -479,10 +480,9 @@ func (a *AnthropicModel) ChatStreamlyWithSender(ctx context.Context, modelName s
url := fmt.Sprintf("%s/%s", baseURL, strings.TrimLeft(a.baseModel.URLSuffix.Chat, "/"))
reqBody := map[string]interface{}{
"model": modelName,
"messages": apiMessages,
"max_tokens": 1024,
"stream": true,
"model": modelName,
"messages": apiMessages,
"stream": true,
}
if systemPrompt != "" {
reqBody["system"] = systemPrompt

View File

@@ -410,7 +410,7 @@ func ParseListModel(modelList ModelList) []ListModelResponse {
if modelEntity != nil {
modelResponse.MaxDimension = modelEntity.MaxDimension
modelResponse.Dimensions = modelEntity.Dimensions
modelResponse.MaxTokens = modelEntity.MaxTokens
modelResponse.MaxOutput = modelEntity.MaxOutput
modelResponse.ModelTypes = modelEntity.ModelTypes
modelResponse.Thinking = modelEntity.Thinking
modelResponse.Dimensions = modelEntity.Dimensions

View File

@@ -163,11 +163,11 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {
if model.Name != alias {
t.Fatalf("models[%d].Name=%q, want %q", i, model.Name, alias)
}
if (model.MaxTokens == nil) != (expected.MaxTokens == nil) {
t.Fatalf("models[%d] alias %q MaxTokens nil=%t, want nil=%t", i, alias, model.MaxTokens == nil, expected.MaxTokens == nil)
if (model.MaxOutput == nil) != (expected.MaxOutput == nil) {
t.Fatalf("models[%d] alias %q MaxOutput nil=%t, want nil=%t", i, alias, model.MaxOutput == nil, expected.MaxOutput == nil)
}
if model.MaxTokens != nil && expected.MaxTokens != nil && *model.MaxTokens != *expected.MaxTokens {
t.Fatalf("models[%d] alias %q MaxTokens=%d, want %d", i, alias, *model.MaxTokens, *expected.MaxTokens)
if model.MaxOutput != nil && expected.MaxOutput != nil && *model.MaxOutput != *expected.MaxOutput {
t.Fatalf("models[%d] alias %q MaxOutput=%d, want %d", i, alias, *model.MaxOutput, *expected.MaxOutput)
}
if strings.Join(model.ModelTypes, ",") != strings.Join(expected.ModelTypes, ",") {
t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expected.ModelTypes)
@@ -181,8 +181,8 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {
if unknown.Name != "unknown-model" {
t.Fatalf("unknown.Name=%q, want unknown-model", unknown.Name)
}
if unknown.MaxTokens != nil {
t.Fatalf("unknown.MaxTokens=%v, want nil", *unknown.MaxTokens)
if unknown.MaxOutput != nil {
t.Fatalf("unknown.MaxOutput=%v, want nil", *unknown.MaxOutput)
}
if len(unknown.ModelTypes) != 0 {
t.Fatalf("unknown.ModelTypes=%v, want empty", unknown.ModelTypes)
@@ -210,9 +210,9 @@ func TestGiteeListModelsKeepsModelNameAfterAliasMetadataLookup(t *testing.T) {
if model.Name != "deepseek/deepseek-v4-pro" {
t.Fatalf("Name=%q, want deepseek/deepseek-v4-pro", model.Name)
}
if model.MaxTokens == nil || *model.MaxTokens != 1048576 {
t.Fatalf("MaxTokens=%v, want 1048576", model.MaxTokens)
}
//if model.MaxOutput == nil || *model.MaxOutput != 1048576 {
// t.Fatalf("MaxOutput=%v, want 1048576", *model.MaxOutput)
//}
if len(model.ModelTypes) != 1 || model.ModelTypes[0] != "chat" {
t.Fatalf("ModelTypes=%v, want [chat]", model.ModelTypes)
}

View File

@@ -160,17 +160,18 @@ type ModelTools struct {
// Model represents a single LLM model
type Model struct {
Name string `json:"name"`
MaxTokens *int `json:"max_tokens"`
ModelTypes []string `json:"model_types"`
Thinking *ModelThinking `json:"thinking"`
Tools *ModelTools `json:"tools"`
Class *string `json:"class"`
MaxDimension *int `json:"max_dimension"` // used by embedding models
Dimensions []int `json:"dimensions"`
Alias []string `json:"alias"`
Rank *int `json:"rank"`
ModelTypeMap map[string]bool
Name string `json:"name"`
ContentLength *int `json:"content_length"`
MaxOutput *int `json:"max_output"`
ModelTypes []string `json:"model_types"`
Thinking *ModelThinking `json:"thinking"`
Tools *ModelTools `json:"tools"`
Class *string `json:"class"`
MaxDimension *int `json:"max_dimension"` // used by embedding models
Dimensions []int `json:"dimensions"`
Alias []string `json:"alias"`
Rank *int `json:"rank"`
ModelTypeMap map[string]bool
}
// Provider represents an LLM provider
@@ -411,8 +412,8 @@ func (pm *ProviderManager) ListAllModels() ([]map[string]interface{}, error) {
if model.Thinking != nil {
modelData["thinking"] = model.Thinking
}
if model.MaxTokens != nil {
modelData["max_tokens"] = *model.MaxTokens
if model.MaxOutput != nil {
modelData["max_output"] = *model.MaxOutput
}
if model.MaxDimension != nil {
modelData["max_dimension"] = *model.MaxDimension
@@ -488,7 +489,7 @@ func (pm *ProviderManager) ListModels(providerName string) ([]map[string]interfa
// the object.
modelData := map[string]interface{}{
"name": model.Name,
"max_tokens": model.MaxTokens,
"max_output": model.MaxOutput,
"model_types": model.ModelTypes,
"max_dimension": model.MaxDimension,
"dimensions": model.Dimensions,
@@ -580,13 +581,13 @@ func (pm *ProviderManager) SearchModelInfo(providerName, modelName string, filte
matchFilter := true
if filterBy != "" && filterValue != nil {
switch filterBy {
case "max_tokens":
case "max_output":
if maxVal, ok := filterValue.(int); ok {
if *model.MaxTokens < maxVal {
if *model.MaxOutput < maxVal {
matchFilter = false
resp.Code = 400
resp.Message = fmt.Sprintf("Model does not meet filter criteria: max_tokens (%d) < %d",
model.MaxTokens, maxVal)
resp.Message = fmt.Sprintf("Model does not meet filter criteria: max_output (%d) < %d",
model.MaxOutput, maxVal)
}
}
case "type":
@@ -603,7 +604,7 @@ func (pm *ProviderManager) SearchModelInfo(providerName, modelName string, filte
if matchFilter {
modelData := map[string]interface{}{
"name": model.Name,
"max_tokens": model.MaxTokens,
"max_output": model.MaxOutput,
"model_types": model.ModelTypes,
//"features": getFeaturesMap(model.Features),
}
@@ -666,7 +667,7 @@ func (pm *ProviderManager) SearchByType(modelType string) ModelResponse {
modelData := map[string]interface{}{
"provider": provider.Name,
"name": model.Name,
"max_tokens": model.MaxTokens,
"max_output": model.MaxOutput,
"model_types": model.ModelTypes,
//"features": getFeaturesMap(model.Features),
}

View File

@@ -294,22 +294,22 @@ func TestPPIOProviderConfigLoadsIntoProviderManager(t *testing.T) {
if err != nil {
t.Fatalf("GetModelByName: %v", err)
}
if *model.MaxTokens != 64000 {
t.Errorf("deepseek/deepseek-r1 max_tokens=%d", *model.MaxTokens)
if *model.MaxOutput != 64000 {
t.Errorf("deepseek/deepseek-r1 max_tokens=%d", *model.MaxOutput)
}
model, err = pm.GetModelByName("ppio", "deepseek/deepseek-v4-pro")
if err != nil {
t.Fatalf("GetModelByName v4 pro: %v", err)
}
if *model.MaxTokens != 1048576 {
t.Errorf("deepseek/deepseek-v4-pro max_tokens=%d", *model.MaxTokens)
if *model.MaxOutput != 1048576 {
t.Errorf("deepseek/deepseek-v4-pro max_tokens=%d", *model.MaxOutput)
}
model, err = pm.GetModelByName("ppio", "deepseek/deepseek-v4-flash")
if err != nil {
t.Fatalf("GetModelByName v4 flash: %v", err)
}
if *model.MaxTokens != 1048576 {
t.Errorf("deepseek/deepseek-v4-flash max_tokens=%d", *model.MaxTokens)
if *model.MaxOutput != 1048576 {
t.Errorf("deepseek/deepseek-v4-flash max_tokens=%d", *model.MaxOutput)
}
if !model.ModelTypeMap["chat"] {
t.Errorf("deepseek/deepseek-v4-flash missing chat type map")
@@ -372,8 +372,8 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
if err != nil {
t.Fatalf("GetModelByName DeepSeek-V4-Pro: %v", err)
}
if *deepSeekV4Pro.MaxTokens != 1048576 {
t.Errorf("DeepSeek-V4-Pro max_tokens=%d", *deepSeekV4Pro.MaxTokens)
if *deepSeekV4Pro.MaxOutput != 1048576 {
t.Errorf("DeepSeek-V4-Pro max_tokens=%d", *deepSeekV4Pro.MaxOutput)
}
if !deepSeekV4Pro.ModelTypeMap["chat"] {
t.Errorf("DeepSeek-V4-Pro model types=%v, want chat", deepSeekV4Pro.ModelTypes)
@@ -383,8 +383,8 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
if err != nil {
t.Fatalf("GetModelByName Kimi-K2.6: %v", err)
}
if *kimiK26.MaxTokens != 262144 {
t.Errorf("Kimi-K2.6 max_tokens=%d", *kimiK26.MaxTokens)
if *kimiK26.MaxOutput != 262144 {
t.Errorf("Kimi-K2.6 max_tokens=%d", *kimiK26.MaxOutput)
}
if !kimiK26.ModelTypeMap["chat"] || !kimiK26.ModelTypeMap["vision"] {
t.Errorf("Kimi-K2.6 model types=%v, want chat+vision", kimiK26.ModelTypes)
@@ -394,7 +394,7 @@ func TestSiliconFlowProviderConfigLoadsLatestProModels(t *testing.T) {
if err != nil {
t.Fatalf("GetModelByName GLM-5.1: %v", err)
}
if *glm51.MaxTokens != 204800 {
t.Errorf("GLM-5.1 max_tokens=%d", *glm51.MaxTokens)
if *glm51.MaxOutput != 204800 {
t.Errorf("GLM-5.1 max_tokens=%d", *glm51.MaxOutput)
}
}

View File

@@ -690,14 +690,14 @@ func parseNvidiaModelList(modelList ModelList, provider *Provider) []ListModelRe
}
}
if preset != nil {
response.MaxTokens = preset.MaxTokens
response.MaxOutput = preset.MaxOutput
response.ModelTypes = append([]string(nil), preset.ModelTypes...)
response.Thinking = preset.Thinking
response.MaxDimension = preset.MaxDimension
response.Dimensions = append([]int(nil), preset.Dimensions...)
} else {
maxTokens := defaultMaxTokens
response.MaxTokens = &maxTokens
response.MaxOutput = &maxTokens
response.ModelTypes = InferModelTypes(modelName)
}
if len(response.ModelTypes) == 0 {

View File

@@ -62,7 +62,7 @@ func TestParseNvidiaModelListPrefersPresetMetadata(t *testing.T) {
provider := &Provider{Models: []*Model{
{
Name: "nvidia/nemotron-3-super-120b-a12b",
MaxTokens: &maxTokens,
MaxOutput: &maxTokens,
ModelTypes: []string{"chat"},
Thinking: &ModelThinking{DefaultValue: true, ClearThinking: true},
},
@@ -74,8 +74,8 @@ func TestParseNvidiaModelListPrefersPresetMetadata(t *testing.T) {
if len(models) != 1 {
t.Fatalf("len(models) = %d, want 1", len(models))
}
if models[0].MaxTokens == nil || *models[0].MaxTokens != maxTokens {
t.Fatalf("MaxTokens = %v, want %d", models[0].MaxTokens, maxTokens)
if models[0].MaxOutput == nil || *models[0].MaxOutput != maxTokens {
t.Fatalf("MaxOutput = %v, want %d", models[0].MaxOutput, maxTokens)
}
if models[0].Thinking == nil || !models[0].Thinking.DefaultValue {
t.Fatalf("Thinking = %#v, want preset metadata", models[0].Thinking)

View File

@@ -105,12 +105,13 @@ type OCRFileResponse struct {
}
type ListModelResponse struct {
Name string `json:"name"`
MaxTokens *int `json:"max_tokens"`
ModelTypes []string `json:"model_types"`
Thinking *ModelThinking `json:"thinking"`
MaxDimension *int `json:"max_dimension"` // used by embedding models
Dimensions []int `json:"dimensions"`
Name string `json:"name"`
ContentLength *int `json:"content_length"`
MaxOutput *int `json:"max_output"`
ModelTypes []string `json:"model_types"`
Thinking *ModelThinking `json:"thinking"`
MaxDimension *int `json:"max_dimension"` // used by embedding models
Dimensions []int `json:"dimensions"`
}
type ParseFileResponse struct {

View File

@@ -181,7 +181,7 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
remoteModels = append(remoteModels, map[string]interface{}{
"name": m.Name,
"model_types": m.ModelTypes,
"max_tokens": m.MaxTokens,
"max_output": m.MaxOutput,
})
}
}

View File

@@ -159,8 +159,8 @@ func resolveModelConfigByID(ctx context.Context, db *gorm.DB, tenantID string, m
return nil, "", nil, 0, err
}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxTokens != nil {
maxTokens = *mi.MaxTokens
if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxOutput != nil {
maxTokens = *mi.MaxOutput
}
if strings.TrimSpace(modelObj.Extra) != "" {
var tenantExtra tenantModelExtra
@@ -217,8 +217,8 @@ func resolveModelConfigFromProviderInstance(ctx context.Context, db *gorm.DB, te
return nil, "", nil, 0, err
}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxTokens != nil {
maxTokens = *mi.MaxTokens
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxOutput != nil {
maxTokens = *mi.MaxOutput
}
if modelObj != nil && strings.TrimSpace(modelObj.Extra) != "" {
var tenantExtra tenantModelExtra
@@ -259,8 +259,8 @@ func resolveModelConfigFromProviderInstance(ctx context.Context, db *gorm.DB, te
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
maxTokens := 0
if llmInfo.MaxTokens != nil {
maxTokens = *llmInfo.MaxTokens
if llmInfo.MaxOutput != nil {
maxTokens = *llmInfo.MaxOutput
}
return driver, llmInfo.Name, apiConfig, maxTokens, nil
}

View File

@@ -396,12 +396,13 @@ func (m *ModelProviderService) ListSupportedModels(ctx context.Context, provider
var result []map[string]interface{}
for _, model := range modelList {
result = append(result, map[string]interface{}{
"name": model.Name,
"max_dimension": model.MaxDimension,
"dimensions": model.Dimensions,
"max_tokens": model.MaxTokens,
"model_types": model.ModelTypes,
"thinking": model.Thinking,
"name": model.Name,
"max_dimension": model.MaxDimension,
"dimensions": model.Dimensions,
"content_length": model.ContentLength,
"max_output": model.MaxOutput,
"model_types": model.ModelTypes,
"thinking": model.Thinking,
})
}
return result, nil
@@ -450,8 +451,8 @@ func (m *ModelProviderService) reconcileNvidiaInstanceModels(
for _, remote := range normalized {
maxTokens := 8192
if remote.MaxTokens != nil && *remote.MaxTokens > 0 {
maxTokens = *remote.MaxTokens
if remote.MaxOutput != nil && *remote.MaxOutput > 0 {
maxTokens = *remote.MaxOutput
}
modelType := int(entity.ModelTypeFromStrings(remote.ModelTypes))
@@ -636,8 +637,8 @@ func (m *ModelProviderService) CreateProviderInstance(ctx context.Context, provi
ModelName: llm.Name,
ModelTypes: llm.ModelTypes,
MaxTokens: func() int {
if llm.MaxTokens != nil {
return *llm.MaxTokens
if llm.MaxOutput != nil {
return *llm.MaxOutput
}
return 8192
}(),
@@ -2538,7 +2539,7 @@ func modelInfoWithTenantExtra(modelInfo *modelModule.Model, modelEntity *entity.
}
if extra.MaxTokens != nil && *extra.MaxTokens > 0 {
model.MaxTokens = extra.MaxTokens
model.MaxOutput = extra.MaxTokens
}
if len(extra.ModelTypes) > 0 {
model.ModelTypes = append([]string(nil), extra.ModelTypes...)
@@ -3483,8 +3484,8 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerEntity.ProviderName, modelEntity.ModelName); mi != nil {
if mi.MaxTokens != nil {
maxTokens = *mi.MaxTokens
if mi.MaxOutput != nil {
maxTokens = *mi.MaxOutput
}
}
maxTokens, err = maxTokensFromTenantModelExtra(modelEntity, maxTokens)
@@ -3879,10 +3880,10 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName("Builtin", pureModelName); mi != nil {
if mi.MaxTokens == nil {
if mi.MaxOutput == nil {
maxTokens = 0
} else {
maxTokens = *mi.MaxTokens
maxTokens = *mi.MaxOutput
}
}
return builtinDriver, pureModelName, apiConfig, maxTokens, nil
@@ -3941,10 +3942,10 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil {
if mi.MaxTokens == nil {
if mi.MaxOutput == nil {
maxTokens = 0
} else {
maxTokens = *mi.MaxTokens
maxTokens = *mi.MaxOutput
}
}
maxTokens, driverErr = maxTokensFromTenantModelExtra(modelObj, maxTokens)
@@ -3999,8 +4000,8 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
maxTokens := 0
if llmInfo.MaxTokens != nil {
maxTokens = *llmInfo.MaxTokens
if llmInfo.MaxOutput != nil {
maxTokens = *llmInfo.MaxOutput
}
return driver, llmInfo.Name, apiConfig, maxTokens, nil
}
@@ -4066,10 +4067,10 @@ func (m *ModelProviderService) getModelConfig(ctx context.Context, tenantID, com
modelInfo, err := dao.GetModelProviderManager().GetModelByName(providerName, modelName)
maxTokens := 0
if err == nil && modelInfo != nil {
if modelInfo.MaxTokens == nil {
if modelInfo.MaxOutput == nil {
maxTokens = 0
} else {
maxTokens = *modelInfo.MaxTokens
maxTokens = *modelInfo.MaxOutput
}
}

View File

@@ -282,8 +282,8 @@ func TestReconcileNvidiaInstanceModelsAddsUpdatesAndDeletes(t *testing.T) {
maxTokens := 131072
maxDimension := 2048
remote := []modelModule.ListModelResponse{
{Name: "nvidia/keep", MaxTokens: &maxTokens, ModelTypes: []string{"chat", "vision"}},
{Name: "nvidia/new-embed", MaxTokens: ptrService(8192), MaxDimension: &maxDimension, Dimensions: []int{1024, 2048}, ModelTypes: []string{"embedding"}},
{Name: "nvidia/keep", MaxOutput: &maxTokens, ModelTypes: []string{"chat", "vision"}},
{Name: "nvidia/new-embed", MaxOutput: ptrService(8192), MaxDimension: &maxDimension, Dimensions: []int{1024, 2048}, ModelTypes: []string{"embedding"}},
}
err := NewModelProviderService().reconcileNvidiaInstanceModels(context.Background(), db, provider, instance, remote)
@@ -292,7 +292,7 @@ func TestReconcileNvidiaInstanceModelsAddsUpdatesAndDeletes(t *testing.T) {
}
var got []*entity.TenantModel
if err := db.Order("model_name").Find(&got).Error; err != nil {
if err = db.Order("model_name").Find(&got).Error; err != nil {
t.Fatalf("list models: %v", err)
}
if len(got) != 2 || got[0].ModelName != "nvidia/keep" || got[1].ModelName != "nvidia/new-embed" {