zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-07 03:48:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: fix model type check when use the model (#14843)

Browse Source 
### What problem does this PR solve?
```
RAGFlow(user)> chat with 'glm-ocr@test@zhipu-ai' message 'what is this'
CLI error: expect  model glm-ocr@zhipu-ai is a chat or multimodal model
```
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-05-12 19:44:01 +08:00
committed by GitHub
parent 76d5240fb5
commit ad4717f40a
3 changed files with 113 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/entity/models/jina.go
View File

@@ -250,3 +250,26 @@ func (j *JinaModel) CheckConnection(apiConfig *APIConfig) error {
_, err := j.ListModels(apiConfig)
return err
}
// TranscribeAudio transcribe audio
func (z *JinaModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
func (z *JinaModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s, no such method", z.Name())
}
// AudioSpeech convert audio to text
func (z *JinaModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
func (z *JinaModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s, no such method", z.Name())
}
// OCRFile OCR file
func (z *JinaModel) OCRFile(modelName *string, fileContent *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}

23
internal/entity/models/mistral.go
View File

@@ -563,3 +563,26 @@ func (m *MistralModel) CheckConnection(apiConfig *APIConfig) error {
func (m *MistralModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
return nil, fmt.Errorf("no such method")
}
// TranscribeAudio transcribe audio
func (z *MistralModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
func (z *MistralModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s, no such method", z.Name())
}
// AudioSpeech convert audio to text
func (z *MistralModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, asrConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}
func (z *MistralModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s, no such method", z.Name())
}
// OCRFile OCR file
func (z *MistralModel) OCRFile(modelName *string, fileContent *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRResponse, error) {
return nil, fmt.Errorf("%s, no such method", z.Name())
}

74
internal/service/model_service.go
View File

@@ -738,6 +738,10 @@ func (m *ModelProviderService) ChatToModelWithMessages(providerName, instanceNam
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s not found", providerName, modelName))
}
if !model.ModelTypeMap["chat"] && !model.ModelTypeMap["vision"] {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("expect model %s@%s is a chat or multimodal model", modelName, providerName))
}
modelConfig.ModelClass = model.Class
var extra map[string]string
@@ -763,6 +767,9 @@ func (m *ModelProviderService) ChatToModelWithMessages(providerName, instanceNam
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "chat" && modelInfo.ModelType != "vision" {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("expect model %s@%s is a chat or multimodal model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -833,11 +840,16 @@ func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanc
return common.CodeNotFound, err
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return common.CodeNotFound, err
}
if !model.ModelTypeMap["chat"] && !model.ModelTypeMap["vision"] {
return common.CodeNotFound, errors.New(fmt.Sprintf("expect model %s@%s is a chat or multimodal model", modelName, providerName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
if err != nil {
@@ -857,6 +869,9 @@ func (m *ModelProviderService) ChatToModelStreamWithSender(providerName, instanc
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "chat" && modelInfo.ModelType != "vision" {
return common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is a chat or multimodal model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -962,6 +977,9 @@ func (m *ModelProviderService) EmbedText(providerName, instanceName, modelName,
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "embedding" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is an embedding model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1044,7 +1062,7 @@ func (m *ModelProviderService) RerankDocument(providerName, instanceName, modelN
}
if !model.ModelTypeMap["rerank"] {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not an embedding model", providerName, modelName))
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a rerank model", providerName, modelName))
}
var extra map[string]string
@@ -1067,6 +1085,9 @@ func (m *ModelProviderService) RerankDocument(providerName, instanceName, modelN
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "rerank" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is a rerank model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1139,11 +1160,16 @@ func (m *ModelProviderService) TranscribeAudio(providerName, instanceName, model
return nil, common.CodeNotFound, errors.New("provider not found")
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s not found", providerName, modelName))
}
if !model.ModelTypeMap["asr"] {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not an ASR model", providerName, modelName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
if err != nil {
@@ -1167,6 +1193,9 @@ func (m *ModelProviderService) TranscribeAudio(providerName, instanceName, model
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "asr" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is an ASR model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1235,10 +1264,14 @@ func (m *ModelProviderService) TranscribeAudioStream(providerName, instanceName,
return common.CodeNotFound, err
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return common.CodeNotFound, err
}
if !model.ModelTypeMap["asr"] {
return common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not an ASR model", providerName, modelName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
@@ -1259,6 +1292,9 @@ func (m *ModelProviderService) TranscribeAudioStream(providerName, instanceName,
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "asr" {
return common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is an ASR model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1329,11 +1365,16 @@ func (m *ModelProviderService) AudioSpeech(providerName, instanceName, modelName
return nil, common.CodeNotFound, errors.New("provider not found")
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s not found", providerName, modelName))
}
if !model.ModelTypeMap["tts"] {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a TTS model", providerName, modelName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
if err != nil {
@@ -1357,6 +1398,9 @@ func (m *ModelProviderService) AudioSpeech(providerName, instanceName, modelName
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "tts" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is a TTS model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1424,11 +1468,16 @@ func (m *ModelProviderService) AudioSpeechStream(providerName, instanceName, mod
return common.CodeNotFound, err
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return common.CodeNotFound, err
}
if !model.ModelTypeMap["tts"] {
return common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a TTS model", providerName, modelName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
if err != nil {
@@ -1448,6 +1497,9 @@ func (m *ModelProviderService) AudioSpeechStream(providerName, instanceName, mod
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "tts" {
return common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is a TTS model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
@@ -1517,11 +1569,16 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
return nil, common.CodeNotFound, errors.New("provider not found")
}
_, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
var model *entity.Model = nil
model, err = dao.GetModelProviderManager().GetModelByName(providerName, modelName)
if err != nil {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s not found", providerName, modelName))
}
if !model.ModelTypeMap["ocr"] {
return nil, common.CodeNotFound, errors.New(fmt.Sprintf("provider %s model %s is not a TTS model", providerName, modelName))
}
var extra map[string]string
err = json.Unmarshal([]byte(instance.Extra), &extra)
if err != nil {
@@ -1545,6 +1602,9 @@ func (m *ModelProviderService) OCRFile(providerName, instanceName, modelName, us
}
if modelInfo.Status == "active" {
if modelInfo.ModelType != "tts" {
return nil, common.CodeServerError, errors.New(fmt.Sprintf("expect model %s@%s is an OCR model", modelName, providerName))
}
// For local deployed models
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {