zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-06-29 23:41:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused codes (#15579)

Browse Source 
### What problem does this PR solve?

Remove unused code.

### Type of change

- [x] Refactoring

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-06-03 17:35:36 +08:00
committed by GitHub
parent b363146997
commit 2061edd308
9 changed files with 7 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/ingestion_server.go
View File

@@ -110,13 +110,6 @@ func main() {
common.Fatal("Failed to initialize database", zap.Error(err))
}
// Initialize LLM factory data models from configuration file
if err := dao.InitLLMFactory(); err != nil {
common.Error("Failed to initialize LLM factory", err)
} else {
common.Info("LLM factory initialized successfully")
}
// Initialize doc engine
if err := engine.Init(&config.DocEngine); err != nil {
common.Fatal("Failed to initialize doc engine", zap.Error(err))

13
cmd/server_main.go
View File

@@ -91,12 +91,6 @@ func main() {
common.Fatal("Server port is not configured. Please specify via --port flag or config file.")
}
// Load model providers configuration
if err := server.LoadModelProviders(""); err != nil {
common.Fatal("Failed to load model providers", zap.Error(err))
}
common.Info("Model providers loaded", zap.Int("count", len(server.GetModelProviders())))
// Reinitialize logger with configured level if different
if config.Log.Level != "" && config.Log.Level != "info" {
if err := common.Init(config.Log.Level); err != nil {
@@ -118,13 +112,6 @@ func main() {
common.Fatal("Failed to initialize database", zap.Error(err))
}
// Initialize LLM factory data models from configuration file
if err := dao.InitLLMFactory(); err != nil {
common.Error("Failed to initialize LLM factory", err)
} else {
common.Info("LLM factory initialized successfully")
}
// Initialize doc engine
if err := engine.Init(&config.DocEngine); err != nil {
common.Fatal("Failed to initialize doc engine", zap.Error(err))

93
internal/dao/database.go
View File

@@ -17,18 +17,14 @@
package dao
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"ragflow/internal/common"
"ragflow/internal/entity"
"strings"
"time"
"ragflow/internal/server"
"ragflow/internal/utility"
"go.uber.org/zap"
gormLogger "gorm.io/gorm/logger"
@@ -212,92 +208,3 @@ func autoMigrateSafely(db *gorm.DB, model interface{}) error {
return err
}
// InitLLMFactory initializes LLM factories and models from JSON file.
// It reads the llm_factories.json configuration file and populates the database
// with LLM factory and model information. If a factory or model already exists,
// it will be updated with the new configuration.
//
// Returns:
// - error: An error if the initialization fails, nil otherwise.
func InitLLMFactory() error {
configPath := filepath.Join(utility.GetProjectBaseDirectory(), "conf", "llm_factories.json")
data, err := os.ReadFile(configPath)
if err != nil {
return fmt.Errorf("failed to read llm_factories.json: %w", err)
}
var fileData LLMFactoriesFile
if err := json.Unmarshal(data, &fileData); err != nil {
return fmt.Errorf("failed to parse llm_factories.json: %w", err)
}
db := DB
for _, factory := range fileData.FactoryLLMInfos {
status := factory.Status
if status == "" {
status = "1"
}
llmFactory := &entity.LLMFactories{
Name: factory.Name,
Logo: utility.StringPtr(factory.Logo),
Tags: factory.Tags,
Rank: utility.ParseInt64(factory.Rank),
Status: &status,
}
var existingFactory entity.LLMFactories
result := db.Where("name = ?", factory.Name).First(&existingFactory)
if result.Error != nil {
if err := db.Create(llmFactory).Error; err != nil {
log.Printf("Failed to create LLM factory %s: %v", factory.Name, err)
continue
}
} else {
if err := db.Model(&entity.LLMFactories{}).Where("name = ?", factory.Name).Updates(map[string]interface{}{
"logo": llmFactory.Logo,
"tags": llmFactory.Tags,
"rank": llmFactory.Rank,
"status": llmFactory.Status,
}).Error; err != nil {
log.Printf("Failed to update LLM factory %s: %v", factory.Name, err)
}
}
for _, llm := range factory.LLM {
llmStatus := "1"
llmModel := &entity.LLM{
LLMName: llm.LLMName,
ModelType: llm.ModelType,
FID: factory.Name,
MaxTokens: llm.MaxTokens,
Tags: llm.Tags,
IsTools: llm.IsTools,
Status: &llmStatus,
}
var existingLLM entity.LLM
result := db.Where("llm_name = ? AND fid = ?", llm.LLMName, factory.Name).First(&existingLLM)
if result.Error != nil {
if err := db.Create(llmModel).Error; err != nil {
log.Printf("Failed to create LLM %s/%s: %v", factory.Name, llm.LLMName, err)
}
} else {
if err := db.Model(&entity.LLM{}).Where("llm_name = ? AND fid = ?", llm.LLMName, factory.Name).Updates(map[string]interface{}{
"model_type": llmModel.ModelType,
"max_tokens": llmModel.MaxTokens,
"tags": llmModel.Tags,
"is_tools": llmModel.IsTools,
"status": llmModel.Status,
}).Error; err != nil {
log.Printf("Failed to update LLM %s/%s: %v", factory.Name, llm.LLMName, err)
}
}
}
}
return nil
}

123
internal/dao/model_provider.go
View File

@@ -1,123 +0,0 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dao
import (
"ragflow/internal/server"
"sync"
)
// ModelProviderDAO provides access to model provider configuration data
type ModelProviderDAO struct{}
var (
modelProviderDAOInstance *ModelProviderDAO
modelProviderDAOOnce sync.Once
)
// NewModelProviderDAO creates a new ModelProviderDAO instance (singleton)
func NewModelProviderDAO() *ModelProviderDAO {
modelProviderDAOOnce.Do(func() {
modelProviderDAOInstance = &ModelProviderDAO{}
})
return modelProviderDAOInstance
}
// GetAllProviders returns all model providers
func (dao *ModelProviderDAO) GetAllProviders() []server.ModelProvider {
return server.GetModelProviders()
}
// GetProviderByName returns the model provider with the given name
func (dao *ModelProviderDAO) GetProviderByName(name string) *server.ModelProvider {
return server.GetModelProviderByName(name)
}
// GetLLMByProviderAndName returns the LLM with the given provider name and model name
func (dao *ModelProviderDAO) GetLLMByProviderAndName(providerName, modelName string) *server.LLM {
return server.GetLLMByProviderAndName(providerName, modelName)
}
// GetLLMsByType returns all LLMs across all providers that match the given model type
func (dao *ModelProviderDAO) GetLLMsByType(modelType string) []server.LLM {
var result []server.LLM
for _, provider := range server.GetModelProviders() {
for _, llm := range provider.LLMs {
if llm.ModelType == modelType {
result = append(result, llm)
}
}
}
return result
}
// GetProvidersByTag returns providers that have the given tag in their tags string
func (dao *ModelProviderDAO) GetProvidersByTag(tag string) []server.ModelProvider {
var result []server.ModelProvider
for _, provider := range server.GetModelProviders() {
if containsTag(provider.Tags, tag) {
result = append(result, provider)
}
}
return result
}
// GetLLMsByProviderAndType returns LLMs for a specific provider that match the given model type
func (dao *ModelProviderDAO) GetLLMsByProviderAndType(providerName, modelType string) []server.LLM {
provider := server.GetModelProviderByName(providerName)
if provider == nil {
return nil
}
var result []server.LLM
for _, llm := range provider.LLMs {
if llm.ModelType == modelType {
result = append(result, llm)
}
}
return result
}
// helper function to check if a comma-separated tag string contains a specific tag
func containsTag(tags, tag string) bool {
// Simple implementation: check substring with boundaries
// Assuming tags are uppercase and comma-separated without spaces
// This may need refinement based on actual tag format
for _, t := range splitTags(tags) {
if t == tag {
return true
}
}
return false
}
func splitTags(tags string) []string {
// Split by comma and trim spaces
var result []string
start := 0
for i, ch := range tags {
if ch == ',' {
if start < i {
result = append(result, tags[start:i])
}
start = i + 1
}
}
if start < len(tags) {
result = append(result, tags[start:])
}
return result
}

68
internal/handler/llm.go
View File

@@ -22,7 +22,6 @@ import (
"github.com/gin-gonic/gin"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/service"
)
@@ -142,73 +141,6 @@ func (h *LLMHandler) SetAPIKey(c *gin.Context) {
})
}
// Factories get model provider factories
// @Summary Get Model Provider Factories
// @Description Get list of model provider factories
// @Tags llm
// @Accept json
// @Produce json
// @Security ApiKeyAuth
// @Success 200 {array} FactoryResponse
// @Router /v1/llm/factories [get]
func (h *LLMHandler) Factories(c *gin.Context) {
_, errorCode, errorMessage := GetUser(c)
if errorCode != common.CodeSuccess {
jsonError(c, errorCode, errorMessage)
return
}
// Get model providers
dao := dao.NewModelProviderDAO()
providers := dao.GetAllProviders()
// Filter out unwanted providers
filtered := make([]FactoryResponse, 0)
excluded := map[string]bool{
"Youdao": true,
"FastEmbed": true,
"BAAI": true,
"Builtin": true,
}
for _, provider := range providers {
if excluded[provider.Name] {
continue
}
// Collect unique model types from LLMs
modelTypes := make(map[string]bool)
for _, llm := range provider.LLMs {
modelTypes[llm.ModelType] = true
}
// Convert to slice
modelTypeSlice := make([]string, 0, len(modelTypes))
for mt := range modelTypes {
modelTypeSlice = append(modelTypeSlice, mt)
}
// If no model types found, use defaults
if len(modelTypeSlice) == 0 {
modelTypeSlice = []string{"chat", "embedding", "rerank", "image2text", "speech2text", "tts", "ocr"}
}
filtered = append(filtered, FactoryResponse{
Name: provider.Name,
Logo: provider.Logo,
Tags: provider.Tags,
Status: provider.Status,
Rank: provider.Rank,
ModelTypes: modelTypeSlice,
})
}
c.JSON(http.StatusOK, gin.H{
"code": common.CodeSuccess,
"data": filtered,
})
}
// ListApp lists LLMs grouped by factory
// @Summary List LLMs
// @Description Get list of LLMs grouped by factory with availability info

9
internal/router/router.go
View File

@@ -442,15 +442,6 @@ func (r *Router) Setup(engine *gin.Engine) {
chunk.POST("/update", r.chunkHandler.UpdateChunk) // Internal API only for GO
}
// LLM routes
llm := authorized.Group("/v1/llm")
{
llm.GET("/my_llms", r.llmHandler.GetMyLLMs)
llm.GET("/factories", r.llmHandler.Factories)
llm.GET("/list", r.llmHandler.ListApp)
llm.POST("/set_api_key", r.llmHandler.SetAPIKey)
}
// Chat routes
chat := authorized.Group("/v1/dialog")
{

116
internal/server/model_provider.go
View File

@@ -1,116 +0,0 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package server
import (
"encoding/json"
"fmt"
"os"
"sync"
)
// ModelProvider represents a model provider configuration
type ModelProvider struct {
Name string `json:"name"`
Logo string `json:"logo"`
Tags string `json:"tags"`
Status string `json:"status"`
Rank string `json:"rank"`
LLMs []LLM `json:"llm"`
DefaultURL string `json:"url,omitempty"`
}
// LLM represents a language model within a provider
type LLM struct {
LLMName string `json:"llm_name"`
Tags string `json:"tags"`
MaxTokens int `json:"max_tokens"`
ModelType string `json:"model_type"`
IsTools bool `json:"is_tools"`
}
var (
modelProviders []ModelProvider
modelProviderMap map[string]int // name -> index in modelProviders slice
modelProvidersOnce sync.Once
modelProvidersErr error
)
// LoadModelProviders loads model providers from JSON file.
// If path is empty, it defaults to "conf/model_providers.json" relative to current working directory.
func LoadModelProviders(path string) error {
modelProvidersOnce.Do(func() {
if path == "" {
path = "conf/llm_factories.json"
//path = "conf/model_providers.json"
}
data, err := os.ReadFile(path)
if err != nil {
modelProvidersErr = fmt.Errorf("failed to read model providers file %s: %w", path, err)
return
}
var root struct {
Providers []ModelProvider `json:"factory_llm_infos"`
}
if err := json.Unmarshal(data, &root); err != nil {
modelProvidersErr = fmt.Errorf("failed to unmarshal model providers JSON: %w", err)
return
}
modelProviders = root.Providers
// Build name to index map for fast lookup
modelProviderMap = make(map[string]int, len(modelProviders))
for i, provider := range modelProviders {
modelProviderMap[provider.Name] = i
}
})
return modelProvidersErr
}
// GetModelProviders returns the loaded model providers.
// Call LoadModelProviders first, otherwise returns empty slice.
func GetModelProviders() []ModelProvider {
return modelProviders
}
// GetModelProviderByName returns the model provider with the given name.
func GetModelProviderByName(name string) *ModelProvider {
if modelProviderMap == nil {
return nil
}
if idx, ok := modelProviderMap[name]; ok {
return &modelProviders[idx]
}
return nil
}
// GetLLMByProviderAndName returns the LLM with the given provider name and model name.
func GetLLMByProviderAndName(providerName, modelName string) *LLM {
provider := GetModelProviderByName(providerName)
if provider == nil {
return nil
}
for i := range provider.LLMs {
if provider.LLMs[i].LLMName == modelName {
return &provider.LLMs[i]
}
}
return nil
}

11
internal/service/dataset.go
View File

@@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"ragflow/internal/entity"
"ragflow/internal/utility"
"strings"
"time"
@@ -413,7 +414,7 @@ func (s *DatasetService) CreateDataset(req *CreateDatasetRequest, tenantID strin
embdID = embeddingModel
}
kbID, err := generateUUID1Hex()
kbID, err := utility.GenerateUUID1()
if err != nil {
return nil, common.CodeServerError, errors.New("Internal server error")
}
@@ -1008,14 +1009,6 @@ func (s *DatasetService) verifyEmbeddingAvailability(embdID string, tenantID str
return false, fmt.Sprintf("Unauthorized model: <%s>", embdID)
}
func generateUUID1Hex() (string, error) {
generatedUUID, err := uuid.NewUUID()
if err != nil {
return "", err
}
return strings.ReplaceAll(generatedUUID.String(), "-", ""), nil
}
func applyAutoMetadataConfig(parserConfig map[string]interface{}, config *AutoMetadataConfig) map[string]interface{} {
if parserConfig == nil {
parserConfig = make(map[string]interface{})

9
internal/service/model_service.go
View File

@@ -24,6 +24,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/utility"
"strings"
"gorm.io/gorm"
@@ -116,7 +117,7 @@ func (m *ModelProviderService) AddModelProvider(providerName, userID string) (co
tenantID := tenants[0].TenantID
providerID, err := generateUUID1Hex()
providerID, err := utility.GenerateUUID1()
if err != nil {
return common.CodeServerError, errors.New("fail to get UUID")
}
@@ -258,7 +259,7 @@ func (m *ModelProviderService) CreateProviderInstance(providerName, instanceName
return common.CodeServerError, err
}
instanceID, err := generateUUID1Hex()
instanceID, err := utility.GenerateUUID1()
if err != nil {
return common.CodeServerError, errors.New("fail to get UUID")
}
@@ -840,7 +841,7 @@ func (m *ModelProviderService) UpdateModelStatus(providerName, instanceName, mod
model, err := m.modelDAO.GetModelByProviderIDAndInstanceIDAndModelName(provider.ID, instance.ID, modelName)
if err != nil {
var modelID string
modelID, err = generateUUID1Hex()
modelID, err = utility.GenerateUUID1()
if err != nil {
return common.CodeServerError, errors.New("fail to get UUID")
}
@@ -2035,7 +2036,7 @@ func (m *ModelProviderService) AddModel(request *AddModelRequest, userID string)
return common.CodeServerError, err
}
modelID, err := generateUUID1Hex()
modelID, err := utility.GenerateUUID1()
if err != nil {
return common.CodeServerError, errors.New("fail to get UUID")
}