diff --git a/Dockerfile b/Dockerfile index ee19086b3a..071efdfc33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -212,7 +212,13 @@ COPY pyproject.toml uv.lock ./ COPY mcp mcp COPY common common COPY memory memory -COPY bin bin + +RUN if [ -d bin ]; then \ + cp -r bin ./; \ + echo "✓ bin copied"; \ + else \ + echo "✗ bin ignored"; \ + fi COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template COPY docker/entrypoint.sh ./ diff --git a/cmd/admin_server.go b/cmd/admin_server.go index d553a44d9e..8a7587487b 100644 --- a/cmd/admin_server.go +++ b/cmd/admin_server.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "os" "github.com/gin-gonic/gin" @@ -65,7 +66,6 @@ func (s *AdminServer) Init() error { // Run start admin server func (s *AdminServer) Run() error { - logger.Info("Starting admin server", zap.String("port", s.port)) return s.engine.Run(":" + s.port) } @@ -107,21 +107,20 @@ func main() { os.Exit(1) } + // Print all configuration settings + server.PrintAll() + // Print RAGFlow Admin logo logger.Info("" + "\n ____ ___ ______________ ___ __ _ \n" + " / __ \\/ | / ____/ ____/ /___ _ __ / | ____/ /___ ___ (_)___ \n" + " / /_/ / /| |/ / __/ /_ / / __ \\ | /| / / / /| |/ __ / __ `__ \\/ / __ \\ \n" + - " / _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ / / ___ / /_/ / / / / / / / / / / /\n" + + " / _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ / / ___ / /_/ / / / / / / / / / /\n" + " /_/ |_/_/ |_\\____/_/ /_/\\____/|__/|__/ /_/ |_\\__,_/_/ /_/ /_/_/_/ /_/ \n") // Print RAGFlow version - logger.Info("RAGFlow version", zap.String("version", utility.GetRAGFlowVersion())) - - // Print all configuration settings - server.PrintAll() - - logger.Info("Starting RAGFlow Admin Server", zap.String("port", "9381")) + logger.Info(fmt.Sprintf("Version: %s", utility.GetRAGFlowVersion())) + logger.Info(fmt.Sprintf("Starting RAGFlow admin server on port: 9381")) if err := adminServer.Run(); err != nil { logger.Error("Admin server error", err) os.Exit(1) diff --git a/cmd/server_main.go b/cmd/server_main.go index e079371e33..011869145b 100644 --- a/cmd/server_main.go +++ b/cmd/server_main.go @@ -7,6 +7,8 @@ import ( "os" "os/signal" "ragflow/internal/server" + "ragflow/internal/utility" + "strings" "syscall" "time" @@ -154,6 +156,14 @@ func main() { // Start server in a goroutine go func() { + logger.Info( + "\n ____ ___ ______ ______ __\n" + + " / __ \\ / | / ____// ____// /____ _ __\n" + + " / /_/ // /| | / / __ / /_ / // __ \\| | /| / /\n" + + " / _, _// ___ |/ /_/ // __/ / // /_/ /| |/ |/ /\n" + + " /_/ |_|/_/ |_|\\____//_/ /_/ \\____/ |__/|__/\n", + ) + logger.Info(fmt.Sprintf("Version: %s", utility.GetRAGFlowVersion())) logger.Info(fmt.Sprintf("Server starting on port: %d", cfg.Server.Port)) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Fatal("Failed to start server", zap.Error(err)) @@ -165,7 +175,7 @@ func main() { signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR2) sig := <-quit - logger.Info("Received signal", zap.String("signal", sig.String())) + logger.Info(fmt.Sprintf("Receives %s signal to shutdown server", strings.ToUpper(sig.String()))) logger.Info("Shutting down server...") // Create context with timeout for graceful shutdown diff --git a/internal/admin/service.go b/internal/admin/service.go index 1c4b430f58..80b2792dfe 100644 --- a/internal/admin/service.go +++ b/internal/admin/service.go @@ -94,7 +94,7 @@ type UserInfo struct { Email string Nickname string IsActive string - CreateTime int64 + CreateTime *int64 UpdateTime *int64 } diff --git a/internal/dao/database.go b/internal/dao/database.go index 163f172df9..391759431e 100644 --- a/internal/dao/database.go +++ b/internal/dao/database.go @@ -77,15 +77,51 @@ func InitDB() error { sqlDB.SetMaxOpenConns(100) sqlDB.SetConnMaxLifetime(time.Hour) - // Auto migrate - if err := DB.AutoMigrate( + // Auto migrate all models + models := []interface{}{ &model.User{}, &model.Tenant{}, &model.UserTenant{}, &model.File{}, &model.File2Document{}, - ); err != nil { - return fmt.Errorf("failed to migrate database: %w", err) + &model.TenantLLM{}, + &model.Chat{}, + &model.ChatSession{}, + &model.Task{}, + &model.APIToken{}, + &model.API4Conversation{}, + &model.Knowledgebase{}, + &model.InvitationCode{}, + &model.Document{}, + &model.UserCanvas{}, + &model.CanvasTemplate{}, + &model.UserCanvasVersion{}, + &model.LLMFactories{}, + &model.LLM{}, + &model.TenantLangfuse{}, + &model.SystemSettings{}, + &model.Connector{}, + &model.Connector2Kb{}, + &model.SyncLogs{}, + &model.MCPServer{}, + &model.Memory{}, + &model.Search{}, + &model.PipelineOperationLog{}, + &model.EvaluationDataset{}, + &model.EvaluationCase{}, + &model.EvaluationRun{}, + &model.EvaluationResult{}, + } + + for _, m := range models { + if err := DB.AutoMigrate(m); err != nil { + return fmt.Errorf("failed to migrate model %T: %w", m, err) + } + } + + // Run manual migrations for complex schema changes + if err := RunMigrations(DB); err != nil { + return fmt.Errorf("failed to run manual migrations: %w", err) } logger.Info("Database connected and migrated successfully") diff --git a/internal/dao/migration.go b/internal/dao/migration.go new file mode 100644 index 0000000000..4c2627d65d --- /dev/null +++ b/internal/dao/migration.go @@ -0,0 +1,307 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package dao + +import ( + "fmt" + "ragflow/internal/logger" + + "go.uber.org/zap" + "gorm.io/gorm" +) + +// RunMigrations runs all manual database migrations +// These are migrations that cannot be handled by AutoMigrate alone +func RunMigrations(db *gorm.DB) error { + // Check if tenant_llm table has composite primary key and migrate to ID primary key + if err := migrateTenantLLMPrimaryKey(db); err != nil { + return fmt.Errorf("failed to migrate tenant_llm primary key: %w", err) + } + + // Rename columns (correct typos) + if err := renameColumnIfExists(db, "task", "process_duation", "process_duration"); err != nil { + return fmt.Errorf("failed to rename task.process_duation: %w", err) + } + if err := renameColumnIfExists(db, "document", "process_duation", "process_duration"); err != nil { + return fmt.Errorf("failed to rename document.process_duation: %w", err) + } + + // Add unique index on user.email + if err := migrateAddUniqueEmail(db); err != nil { + return fmt.Errorf("failed to add unique index on user.email: %w", err) + } + + // Modify column types that AutoMigrate may not handle correctly + if err := modifyColumnTypes(db); err != nil { + return fmt.Errorf("failed to modify column types: %w", err) + } + + logger.Info("All manual migrations completed successfully") + return nil +} + +// migrateTenantLLMPrimaryKey migrates tenant_llm from composite primary key to ID primary key +// This corresponds to Python's update_tenant_llm_to_id_primary_key function +func migrateTenantLLMPrimaryKey(db *gorm.DB) error { + // Check if tenant_llm table exists + if !db.Migrator().HasTable("tenant_llm") { + return nil + } + + // Check if 'id' column already exists using raw SQL + var idColumnExists int64 + err := db.Raw(` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'id' + `).Scan(&idColumnExists).Error + if err != nil { + return err + } + + if idColumnExists > 0 { + // Check if id is already a primary key with auto_increment + var count int64 + err := db.Raw(` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = 'tenant_llm' + AND COLUMN_NAME = 'id' + AND EXTRA LIKE '%auto_increment%' + `).Scan(&count).Error + if err != nil { + return err + } + if count > 0 { + // Already migrated + return nil + } + } + + logger.Info("Migrating tenant_llm to use ID primary key...") + + // Start transaction + return db.Transaction(func(tx *gorm.DB) error { + // Check for temp_id column and drop it if exists + var tempIdExists int64 + tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'temp_id'`).Scan(&tempIdExists) + if tempIdExists > 0 { + if err := tx.Exec("ALTER TABLE tenant_llm DROP COLUMN temp_id").Error; err != nil { + logger.Warn("Failed to drop temp_id column", zap.Error(err)) + } + } + + // Check if there's already an 'id' column + if idColumnExists > 0 { + // Modify existing id column to be auto_increment primary key + if err := tx.Exec(` + ALTER TABLE tenant_llm + MODIFY COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY + `).Error; err != nil { + return fmt.Errorf("failed to modify id column: %w", err) + } + } else { + // Add id column as auto_increment primary key + if err := tx.Exec(` + ALTER TABLE tenant_llm + ADD COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST + `).Error; err != nil { + return fmt.Errorf("failed to add id column: %w", err) + } + } + + // Add unique index on (tenant_id, llm_factory, llm_name) + var idxExists int64 + tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_NAME = 'tenant_llm' AND INDEX_NAME = 'idx_tenant_llm_unique'`).Scan(&idxExists) + if idxExists == 0 { + if err := tx.Exec(` + ALTER TABLE tenant_llm + ADD UNIQUE INDEX idx_tenant_llm_unique (tenant_id, llm_factory, llm_name) + `).Error; err != nil { + logger.Warn("Failed to add unique index idx_tenant_llm_unique", zap.Error(err)) + } + } + + logger.Info("tenant_llm primary key migration completed") + return nil + }) +} + +// migrateAddUniqueEmail adds unique index on user.email +func migrateAddUniqueEmail(db *gorm.DB) error { + if !db.Migrator().HasTable("user") { + return nil + } + + // Check if unique index already exists using raw SQL + var count int64 + db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_NAME = 'user' AND INDEX_NAME = 'idx_user_email_unique'`).Scan(&count) + if count > 0 { + return nil + } + + // Check if there's a duplicate email issue first + var duplicateCount int64 + err := db.Raw(` + SELECT COUNT(*) FROM ( + SELECT email FROM user GROUP BY email HAVING COUNT(*) > 1 + ) AS duplicates + `).Scan(&duplicateCount).Error + if err != nil { + return err + } + + if duplicateCount > 0 { + logger.Warn("Found duplicate emails in user table, cannot add unique index", zap.Int64("count", duplicateCount)) + return nil + } + + logger.Info("Adding unique index on user.email...") + if err := db.Exec(`ALTER TABLE user ADD UNIQUE INDEX idx_user_email_unique (email)`).Error; err != nil { + return fmt.Errorf("failed to add unique index on email: %w", err) + } + + return nil +} + +// modifyColumnTypes modifies column types that need explicit ALTER statements +func modifyColumnTypes(db *gorm.DB) error { + // Helper function to check if column exists + columnExists := func(table, column string) bool { + var count int64 + db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, table, column).Scan(&count) + return count > 0 + } + + // dialog.top_k: ensure it's INTEGER with default 1024 + if db.Migrator().HasTable("dialog") && columnExists("dialog", "top_k") { + if err := db.Exec(`ALTER TABLE dialog MODIFY COLUMN top_k BIGINT NOT NULL DEFAULT 1024`).Error; err != nil { + logger.Warn("Failed to modify dialog.top_k", zap.Error(err)) + } + } + + // tenant_llm.api_key: ensure it's TEXT type + if db.Migrator().HasTable("tenant_llm") && columnExists("tenant_llm", "api_key") { + if err := db.Exec(`ALTER TABLE tenant_llm MODIFY COLUMN api_key LONGTEXT`).Error; err != nil { + logger.Warn("Failed to modify tenant_llm.api_key", zap.Error(err)) + } + } + + // api_token.dialog_id: ensure it's varchar(32) + if db.Migrator().HasTable("api_token") && columnExists("api_token", "dialog_id") { + if err := db.Exec(`ALTER TABLE api_token MODIFY COLUMN dialog_id VARCHAR(32)`).Error; err != nil { + logger.Warn("Failed to modify api_token.dialog_id", zap.Error(err)) + } + } + + // canvas_template.title and description: ensure they're LONGTEXT type (same as Python JSONField) + // Note: Python's JSONField uses null=True with application-level default, not database DEFAULT + if db.Migrator().HasTable("canvas_template") { + if columnExists("canvas_template", "title") { + if err := db.Exec(`ALTER TABLE canvas_template MODIFY COLUMN title LONGTEXT NULL`).Error; err != nil { + logger.Warn("Failed to modify canvas_template.title", zap.Error(err)) + } + } + if columnExists("canvas_template", "description") { + if err := db.Exec(`ALTER TABLE canvas_template MODIFY COLUMN description LONGTEXT NULL`).Error; err != nil { + logger.Warn("Failed to modify canvas_template.description", zap.Error(err)) + } + } + } + + // system_settings.value: ensure it's LONGTEXT + if db.Migrator().HasTable("system_settings") && columnExists("system_settings", "value") { + if err := db.Exec(`ALTER TABLE system_settings MODIFY COLUMN value LONGTEXT NOT NULL`).Error; err != nil { + logger.Warn("Failed to modify system_settings.value", zap.Error(err)) + } + } + + // knowledgebase.raptor_task_finish_at: ensure it's DateTime + if db.Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "raptor_task_finish_at") { + if err := db.Exec(`ALTER TABLE knowledgebase MODIFY COLUMN raptor_task_finish_at DATETIME`).Error; err != nil { + logger.Warn("Failed to modify knowledgebase.raptor_task_finish_at", zap.Error(err)) + } + } + + // knowledgebase.mindmap_task_finish_at: ensure it's DateTime + if db.Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "mindmap_task_finish_at") { + if err := db.Exec(`ALTER TABLE knowledgebase MODIFY COLUMN mindmap_task_finish_at DATETIME`).Error; err != nil { + logger.Warn("Failed to modify knowledgebase.mindmap_task_finish_at", zap.Error(err)) + } + } + + return nil +} + +// renameColumnIfExists renames a column if it exists and the new column doesn't exist +func renameColumnIfExists(db *gorm.DB, tableName, oldName, newName string) error { + if !db.Migrator().HasTable(tableName) { + return nil + } + + // Helper to check if column exists + columnExists := func(column string) bool { + var count int64 + db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, column).Scan(&count) + return count > 0 + } + + // Check if old column exists + if !columnExists(oldName) { + return nil + } + + // Check if new column already exists + if columnExists(newName) { + // Both exist, drop the old one + logger.Warn("Both old and new columns exist, dropping old one", + zap.String("table", tableName), + zap.String("oldColumn", oldName), + zap.String("newColumn", newName)) + return db.Migrator().DropColumn(tableName, oldName) + } + + logger.Info("Renaming column", + zap.String("table", tableName), + zap.String("oldColumn", oldName), + zap.String("newColumn", newName)) + return db.Migrator().RenameColumn(tableName, oldName, newName) +} + +// addColumnIfNotExists adds a column if it doesn't exist +func addColumnIfNotExists(db *gorm.DB, tableName, columnName, columnDef string) error { + if !db.Migrator().HasTable(tableName) { + return nil + } + + // Check if column exists using raw SQL + var count int64 + db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, columnName).Scan(&count) + if count > 0 { + return nil + } + + logger.Info("Adding column", + zap.String("table", tableName), + zap.String("column", columnName)) + sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDef) + return db.Exec(sql).Error +} diff --git a/internal/model/api.go b/internal/model/api.go index afc3a985fb..1f22e8b8dd 100644 --- a/internal/model/api.go +++ b/internal/model/api.go @@ -33,18 +33,20 @@ func (APIToken) TableName() string { // API4Conversation API for conversation model type API4Conversation struct { - ID string `gorm:"column:id;primaryKey;size:32" json:"id"` - DialogID string `gorm:"column:dialog_id;size:32;not null;index" json:"dialog_id"` - UserID string `gorm:"column:user_id;size:255;not null;index" json:"user_id"` - Message JSONMap `gorm:"column:message;type:json" json:"message,omitempty"` - Reference JSONMap `gorm:"column:reference;type:json;default:'[]'" json:"reference"` - Tokens int64 `gorm:"column:tokens;default:0" json:"tokens"` - Source *string `gorm:"column:source;size:16;index" json:"source,omitempty"` - DSL JSONMap `gorm:"column:dsl;type:json" json:"dsl,omitempty"` - Duration float64 `gorm:"column:duration;default:0;index" json:"duration"` - Round int64 `gorm:"column:round;default:0;index" json:"round"` - ThumbUp int64 `gorm:"column:thumb_up;default:0;index" json:"thumb_up"` - Errors *string `gorm:"column:errors;type:longtext" json:"errors,omitempty"` + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + Name *string `gorm:"column:name;size:255" json:"name,omitempty"` + DialogID string `gorm:"column:dialog_id;size:32;not null;index" json:"dialog_id"` + UserID string `gorm:"column:user_id;size:255;not null;index" json:"user_id"` + ExpUserID *string `gorm:"column:exp_user_id;size:255;index" json:"exp_user_id,omitempty"` + Message JSONMap `gorm:"column:message;type:longtext" json:"message,omitempty"` + Reference JSONMap `gorm:"column:reference;type:longtext" json:"reference"` + Tokens int64 `gorm:"column:tokens;default:0" json:"tokens"` + Source *string `gorm:"column:source;size:16;index" json:"source,omitempty"` + DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"` + Duration float64 `gorm:"column:duration;default:0;index" json:"duration"` + Round int64 `gorm:"column:round;default:0;index" json:"round"` + ThumbUp int64 `gorm:"column:thumb_up;default:0;index" json:"thumb_up"` + Errors *string `gorm:"column:errors;type:longtext" json:"errors,omitempty"` BaseModel } diff --git a/internal/model/base.go b/internal/model/base.go index dfccc45a80..beb60682a4 100644 --- a/internal/model/base.go +++ b/internal/model/base.go @@ -23,8 +23,9 @@ import ( ) // BaseModel base model +// All time fields are nullable to match Python Peewee model (null=True) type BaseModel struct { - CreateTime int64 `gorm:"column:create_time;index" json:"create_time"` + CreateTime *int64 `gorm:"column:create_time;index" json:"create_time,omitempty"` CreateDate *time.Time `gorm:"column:create_date;index" json:"create_date,omitempty"` UpdateTime *int64 `gorm:"column:update_time;index" json:"update_time,omitempty"` UpdateDate *time.Time `gorm:"column:update_date;index" json:"update_date,omitempty"` diff --git a/internal/model/canvas.go b/internal/model/canvas.go index 06a0be3edd..c10c3a19f6 100644 --- a/internal/model/canvas.go +++ b/internal/model/canvas.go @@ -23,10 +23,11 @@ type UserCanvas struct { UserID string `gorm:"column:user_id;size:255;not null;index" json:"user_id"` Title *string `gorm:"column:title;size:255" json:"title,omitempty"` Permission string `gorm:"column:permission;size:16;not null;default:me;index" json:"permission"` + Release bool `gorm:"column:release;not null;default:false;index" json:"release"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` CanvasType *string `gorm:"column:canvas_type;size:32;index" json:"canvas_type,omitempty"` CanvasCategory string `gorm:"column:canvas_category;size:32;not null;default:agent_canvas;index" json:"canvas_category"` - DSL JSONMap `gorm:"column:dsl;type:json" json:"dsl,omitempty"` + DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"` BaseModel } @@ -39,11 +40,11 @@ func (UserCanvas) TableName() string { type CanvasTemplate struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"` - Title JSONMap `gorm:"column:title;type:json;default:'{}'" json:"title"` - Description JSONMap `gorm:"column:description;type:json;default:'{}'" json:"description"` + Title JSONMap `gorm:"column:title;type:longtext" json:"title"` + Description JSONMap `gorm:"column:description;type:longtext" json:"description"` CanvasType *string `gorm:"column:canvas_type;size:32;index" json:"canvas_type,omitempty"` CanvasCategory string `gorm:"column:canvas_category;size:32;not null;default:agent_canvas;index" json:"canvas_category"` - DSL JSONMap `gorm:"column:dsl;type:json" json:"dsl,omitempty"` + DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"` BaseModel } @@ -58,7 +59,7 @@ type UserCanvasVersion struct { UserCanvasID string `gorm:"column:user_canvas_id;size:255;not null;index" json:"user_canvas_id"` Title *string `gorm:"column:title;size:255" json:"title,omitempty"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` - DSL JSONMap `gorm:"column:dsl;type:json" json:"dsl,omitempty"` + DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"` BaseModel } diff --git a/internal/model/chat.go b/internal/model/chat.go index 2bb54aec40..cceb4aceca 100644 --- a/internal/model/chat.go +++ b/internal/model/chat.go @@ -27,17 +27,19 @@ type Chat struct { Icon *string `gorm:"column:icon;type:longtext" json:"icon,omitempty"` Language *string `gorm:"column:language;size:32;index" json:"language,omitempty"` LLMID string `gorm:"column:llm_id;size:128;not null" json:"llm_id"` - LLMSetting JSONMap `gorm:"column:llm_setting;type:json;not null;default:'{\"temperature\":0.1,\"top_p\":0.3,\"frequency_penalty\":0.7,\"presence_penalty\":0.4,\"max_tokens\":512}'" json:"llm_setting"` - PromptType string `gorm:"column:prompt_type;size:16;not null;default:simple;index" json:"prompt_type"` - PromptConfig JSONMap `gorm:"column:prompt_config;type:json;not null;default:'{\"system\":\"\",\"prologue\":\"Hi! I'm your assistant. What can I do for you?\",\"parameters\":[],\"empty_response\":\"Sorry! No relevant content was found in the knowledge base!\"}'" json:"prompt_config"` - MetaDataFilter *JSONMap `gorm:"column:meta_data_filter;type:json" json:"meta_data_filter,omitempty"` + TenantLLMID *int64 `gorm:"column:tenant_llm_id;index" json:"tenant_llm_id,omitempty"` + LLMSetting JSONMap `gorm:"column:llm_setting;type:longtext;not null" json:"llm_setting"` + PromptType string `gorm:"column:prompt_type;size:16;not null;default:'simple';index" json:"prompt_type"` + PromptConfig JSONMap `gorm:"column:prompt_config;type:longtext;not null" json:"prompt_config"` + MetaDataFilter *JSONMap `gorm:"column:meta_data_filter;type:longtext" json:"meta_data_filter,omitempty"` SimilarityThreshold float64 `gorm:"column:similarity_threshold;default:0.2" json:"similarity_threshold"` VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight;default:0.3" json:"vector_similarity_weight"` TopN int64 `gorm:"column:top_n;default:6" json:"top_n"` TopK int64 `gorm:"column:top_k;default:1024" json:"top_k"` DoRefer string `gorm:"column:do_refer;size:1;not null;default:1" json:"do_refer"` RerankID string `gorm:"column:rerank_id;size:128;not null;default:''" json:"rerank_id"` - KBIDs JSONSlice `gorm:"column:kb_ids;type:json;not null;default:'[]'" json:"kb_ids"` + TenantRerankID *int64 `gorm:"column:tenant_rerank_id;index" json:"tenant_rerank_id,omitempty"` + KBIDs JSONSlice `gorm:"column:kb_ids;type:longtext;not null" json:"kb_ids"` Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` BaseModel } @@ -52,8 +54,8 @@ type ChatSession struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` DialogID string `gorm:"column:dialog_id;size:32;not null;index" json:"dialog_id"` Name *string `gorm:"column:name;size:255;index" json:"name,omitempty"` - Message json.RawMessage `gorm:"column:message;type:json" json:"message,omitempty"` - Reference json.RawMessage `gorm:"column:reference;type:json;default:'[]'" json:"reference"` + Message json.RawMessage `gorm:"column:message;type:longtext" json:"message,omitempty"` + Reference json.RawMessage `gorm:"column:reference;type:longtext" json:"reference"` UserID *string `gorm:"column:user_id;size:255;index" json:"user_id,omitempty"` BaseModel } diff --git a/internal/model/connector.go b/internal/model/connector.go index 893c12fb63..33b38ec5c1 100644 --- a/internal/model/connector.go +++ b/internal/model/connector.go @@ -25,7 +25,7 @@ type Connector struct { Name string `gorm:"column:name;size:128;not null" json:"name"` Source string `gorm:"column:source;size:128;not null;index" json:"source"` InputType string `gorm:"column:input_type;size:128;not null;index" json:"input_type"` - Config JSONMap `gorm:"column:config;type:json;not null;default:'{}'" json:"config"` + Config JSONMap `gorm:"column:config;type:longtext;not null" json:"config"` RefreshFreq int64 `gorm:"column:refresh_freq;default:0" json:"refresh_freq"` PruneFreq int64 `gorm:"column:prune_freq;default:0" json:"prune_freq"` TimeoutSecs int64 `gorm:"column:timeout_secs;default:3600" json:"timeout_secs"` @@ -62,7 +62,7 @@ type SyncLogs struct { NewDocsIndexed int64 `gorm:"column:new_docs_indexed;default:0" json:"new_docs_indexed"` TotalDocsIndexed int64 `gorm:"column:total_docs_indexed;default:0" json:"total_docs_indexed"` DocsRemovedFromIndex int64 `gorm:"column:docs_removed_from_index;default:0" json:"docs_removed_from_index"` - ErrorMsg string `gorm:"column:error_msg;type:longtext;not null;default:''" json:"error_msg"` + ErrorMsg string `gorm:"column:error_msg;type:longtext;not null" json:"error_msg"` ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"` FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"` TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"` diff --git a/internal/model/document.go b/internal/model/document.go index a161e08f77..ffe13fa57f 100644 --- a/internal/model/document.go +++ b/internal/model/document.go @@ -25,7 +25,7 @@ type Document struct { KbID string `gorm:"column:kb_id;size:256;not null;index" json:"kb_id"` ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"` PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"` - ParserConfig JSONMap `gorm:"column:parser_config;type:json;not null;default:'{\"pages\":[[1,1000000]],\"table_context_size\":0,\"image_context_size\":0}'" json:"parser_config"` + ParserConfig JSONMap `gorm:"column:parser_config;type:longtext;not null" json:"parser_config"` SourceType string `gorm:"column:source_type;size:128;not null;default:local;index" json:"source_type"` Type string `gorm:"column:type;size:32;not null;index" json:"type"` CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` @@ -38,7 +38,8 @@ type Document struct { ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"` ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"` ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"` - MetaFields *JSONMap `gorm:"column:meta_fields;type:json" json:"meta_fields,omitempty"` + ContentHash *string `gorm:"column:content_hash;size:32;index" json:"content_hash,omitempty"` + MetaFields *JSONMap `gorm:"column:meta_fields;type:longtext" json:"meta_fields,omitempty"` Suffix string `gorm:"column:suffix;size:32;not null;index" json:"suffix"` Run *string `gorm:"column:run;size:1;index" json:"run,omitempty"` Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` diff --git a/internal/model/evaluation.go b/internal/model/evaluation.go index 5b9bac787a..3e2de1fa5a 100644 --- a/internal/model/evaluation.go +++ b/internal/model/evaluation.go @@ -17,15 +17,18 @@ package model // EvaluationDataset evaluation dataset model +// Note: Python defines custom create_time/update_time (not null) instead of using BaseModel's type EvaluationDataset struct { - ID string `gorm:"column:id;primaryKey;size:32" json:"id"` - TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` - Name string `gorm:"column:name;size:255;not null;index" json:"name"` + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` + Name string `gorm:"column:name;size:255;not null;index" json:"name"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` - KbIDs JSONMap `gorm:"column:kb_ids;type:json;not null" json:"kb_ids"` - CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` - Status int64 `gorm:"column:status;default:1;index" json:"status"` - BaseModel + KbIDs JSONMap `gorm:"column:kb_ids;type:longtext;not null" json:"kb_ids"` + CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` + // Custom time fields (not null) to match Python + CreateTime int64 `gorm:"column:create_time;not null;index" json:"create_time"` + UpdateTime int64 `gorm:"column:update_time;not null" json:"update_time"` + Status int64 `gorm:"column:status;default:1;index" json:"status"` } // TableName specify table name @@ -34,15 +37,17 @@ func (EvaluationDataset) TableName() string { } // EvaluationCase evaluation case model +// Note: Python defines custom create_time (not null) instead of using BaseModel's type EvaluationCase struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` DatasetID string `gorm:"column:dataset_id;size:32;not null;index" json:"dataset_id"` Question string `gorm:"column:question;type:longtext;not null" json:"question"` ReferenceAnswer *string `gorm:"column:reference_answer;type:longtext" json:"reference_answer,omitempty"` - RelevantDocIDs *JSONMap `gorm:"column:relevant_doc_ids;type:json" json:"relevant_doc_ids,omitempty"` - RelevantChunkIDs *JSONMap `gorm:"column:relevant_chunk_ids;type:json" json:"relevant_chunk_ids,omitempty"` - Metadata *JSONMap `gorm:"column:metadata;type:json" json:"metadata,omitempty"` - BaseModel + RelevantDocIDs *JSONMap `gorm:"column:relevant_doc_ids;type:longtext" json:"relevant_doc_ids,omitempty"` + RelevantChunkIDs *JSONMap `gorm:"column:relevant_chunk_ids;type:longtext" json:"relevant_chunk_ids,omitempty"` + Metadata *JSONMap `gorm:"column:metadata;type:longtext" json:"metadata,omitempty"` + // Custom time field (not null) to match Python + CreateTime int64 `gorm:"column:create_time;not null" json:"create_time"` } // TableName specify table name @@ -51,16 +56,19 @@ func (EvaluationCase) TableName() string { } // EvaluationRun evaluation run model +// Note: Python defines custom create_time/complete_time instead of using BaseModel's type EvaluationRun struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` DatasetID string `gorm:"column:dataset_id;size:32;not null;index" json:"dataset_id"` DialogID string `gorm:"column:dialog_id;size:32;not null;index" json:"dialog_id"` Name string `gorm:"column:name;size:255;not null" json:"name"` - ConfigSnapshot JSONMap `gorm:"column:config_snapshot;type:json;not null" json:"config_snapshot"` - MetricsSummary *JSONMap `gorm:"column:metrics_summary;type:json" json:"metrics_summary,omitempty"` + ConfigSnapshot JSONMap `gorm:"column:config_snapshot;type:longtext;not null" json:"config_snapshot"` + MetricsSummary *JSONMap `gorm:"column:metrics_summary;type:longtext" json:"metrics_summary,omitempty"` Status string `gorm:"column:status;size:32;not null;default:PENDING" json:"status"` CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` - BaseModel + // Custom time fields to match Python + CreateTime int64 `gorm:"column:create_time;not null;index" json:"create_time"` + CompleteTime *int64 `gorm:"column:complete_time" json:"complete_time,omitempty"` } // TableName specify table name @@ -69,16 +77,18 @@ func (EvaluationRun) TableName() string { } // EvaluationResult evaluation result model +// Note: Python defines custom create_time (not null) instead of using BaseModel's type EvaluationResult struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` RunID string `gorm:"column:run_id;size:32;not null;index" json:"run_id"` CaseID string `gorm:"column:case_id;size:32;not null;index" json:"case_id"` GeneratedAnswer string `gorm:"column:generated_answer;type:longtext;not null" json:"generated_answer"` - RetrievedChunks JSONMap `gorm:"column:retrieved_chunks;type:json;not null" json:"retrieved_chunks"` - Metrics JSONMap `gorm:"column:metrics;type:json;not null" json:"metrics"` + RetrievedChunks JSONMap `gorm:"column:retrieved_chunks;type:longtext;not null" json:"retrieved_chunks"` + Metrics JSONMap `gorm:"column:metrics;type:longtext;not null" json:"metrics"` ExecutionTime float64 `gorm:"column:execution_time;not null" json:"execution_time"` - TokenUsage *JSONMap `gorm:"column:token_usage;type:json" json:"token_usage,omitempty"` - BaseModel + TokenUsage *JSONMap `gorm:"column:token_usage;type:longtext" json:"token_usage,omitempty"` + // Custom time field to match Python + CreateTime int64 `gorm:"column:create_time;not null" json:"create_time"` } // TableName specify table name diff --git a/internal/model/kb.go b/internal/model/kb.go index 8862b1e1ac..78cc643721 100644 --- a/internal/model/kb.go +++ b/internal/model/kb.go @@ -27,6 +27,7 @@ type Knowledgebase struct { Language *string `gorm:"column:language;size:32;index" json:"language,omitempty"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` EmbdID string `gorm:"column:embd_id;size:128;not null;index" json:"embd_id"` + TenantEmbdID *int64 `gorm:"column:tenant_embd_id;index" json:"tenant_embd_id,omitempty"` Permission string `gorm:"column:permission;size:16;not null;default:me;index" json:"permission"` CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` DocNum int64 `gorm:"column:doc_num;default:0;index" json:"doc_num"` @@ -36,7 +37,7 @@ type Knowledgebase struct { VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight;default:0.3;index" json:"vector_similarity_weight"` ParserID string `gorm:"column:parser_id;size:32;not null;default:naive;index" json:"parser_id"` PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"` - ParserConfig JSONMap `gorm:"column:parser_config;type:json;not null;default:'{\"pages\":[[1,1000000]],\"table_context_size\":0,\"image_context_size\":0}'" json:"parser_config"` + ParserConfig JSONMap `gorm:"column:parser_config;type:longtext;not null" json:"parser_config"` Pagerank int64 `gorm:"column:pagerank;default:0" json:"pagerank"` GraphragTaskID *string `gorm:"column:graphrag_task_id;size:32;index" json:"graphrag_task_id,omitempty"` GraphragTaskFinishAt *time.Time `gorm:"column:graphrag_task_finish_at" json:"graphrag_task_finish_at,omitempty"` diff --git a/internal/model/llm.go b/internal/model/llm.go index 96377d1ebb..9b9054e7e6 100644 --- a/internal/model/llm.go +++ b/internal/model/llm.go @@ -51,8 +51,8 @@ func (LLM) TableName() string { // TenantLangfuse tenant langfuse model type TenantLangfuse struct { TenantID string `gorm:"column:tenant_id;primaryKey;size:32" json:"tenant_id"` - SecretKey string `gorm:"column:secret_key;size:2048;not null;index" json:"secret_key"` - PublicKey string `gorm:"column:public_key;size:2048;not null;index" json:"public_key"` + SecretKey string `gorm:"column:secret_key;size:2048;not null" json:"secret_key"` + PublicKey string `gorm:"column:public_key;size:2048;not null" json:"public_key"` Host string `gorm:"column:host;size:128;not null;index" json:"host"` BaseModel } diff --git a/internal/model/mcp.go b/internal/model/mcp.go index 044bbdab14..deaf1ac821 100644 --- a/internal/model/mcp.go +++ b/internal/model/mcp.go @@ -24,8 +24,8 @@ type MCPServer struct { URL string `gorm:"column:url;size:2048;not null" json:"url"` ServerType string `gorm:"column:server_type;size:32;not null" json:"server_type"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` - Variables JSONMap `gorm:"column:variables;type:json;default:'{}'" json:"variables,omitempty"` - Headers JSONMap `gorm:"column:headers;type:json;default:'{}'" json:"headers,omitempty"` + Variables JSONMap `gorm:"column:variables;type:longtext" json:"variables,omitempty"` + Headers JSONMap `gorm:"column:headers;type:longtext" json:"headers,omitempty"` BaseModel } diff --git a/internal/model/memory.go b/internal/model/memory.go index 28f9f58c1c..9e6480ad96 100644 --- a/internal/model/memory.go +++ b/internal/model/memory.go @@ -25,7 +25,9 @@ type Memory struct { MemoryType int64 `gorm:"column:memory_type;default:1;index" json:"memory_type"` StorageType string `gorm:"column:storage_type;size:32;not null;default:table;index" json:"storage_type"` EmbdID string `gorm:"column:embd_id;size:128;not null" json:"embd_id"` + TenantEmbdID *int64 `gorm:"column:tenant_embd_id;index" json:"tenant_embd_id,omitempty"` LLMID string `gorm:"column:llm_id;size:128;not null" json:"llm_id"` + TenantLLMID *int64 `gorm:"column:tenant_llm_id;index" json:"tenant_llm_id,omitempty"` Permissions string `gorm:"column:permissions;size:16;not null;default:me;index" json:"permissions"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` MemorySize int64 `gorm:"column:memory_size;default:5242880;not null" json:"memory_size"` diff --git a/internal/model/pipeline.go b/internal/model/pipeline.go index a47d611987..a9e9dcbdb5 100644 --- a/internal/model/pipeline.go +++ b/internal/model/pipeline.go @@ -35,7 +35,7 @@ type PipelineOperationLog struct { ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"` ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"` ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"` - DSL JSONMap `gorm:"column:dsl;type:json" json:"dsl,omitempty"` + DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"` TaskType string `gorm:"column:task_type;size:32;not null;default:''" json:"task_type"` OperationStatus string `gorm:"column:operation_status;size:32;not null" json:"operation_status"` Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"` diff --git a/internal/model/search.go b/internal/model/search.go index da95ccd693..c70bf94bb6 100644 --- a/internal/model/search.go +++ b/internal/model/search.go @@ -24,7 +24,7 @@ type Search struct { Name string `gorm:"column:name;size:128;not null;index" json:"name"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` CreatedBy string `gorm:"column:created_by;size:32;not null;index" json:"created_by"` - SearchConfig JSONMap `gorm:"column:search_config;type:json;not null;default:'{\"kb_ids\":[],\"doc_ids\":[],\"similarity_threshold\":0.2,\"vector_similarity_weight\":0.3,\"use_kg\":false,\"rerank_id\":\"\",\"top_k\":1024,\"summary\":false,\"chat_id\":\"\",\"chat_settingcross_languages\":[],\"highlight\":false,\"keyword\":false,\"web_search\":false,\"related_search\":false,\"query_mindmap\":false}'" json:"search_config"` + SearchConfig JSONMap `gorm:"column:search_config;type:longtext;not null" json:"search_config"` Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` BaseModel } diff --git a/internal/model/system.go b/internal/model/system.go index 4877556113..be94f1653a 100644 --- a/internal/model/system.go +++ b/internal/model/system.go @@ -21,7 +21,7 @@ type SystemSettings struct { Name string `gorm:"column:name;primaryKey;size:128" json:"name"` Source string `gorm:"column:source;size:32;not null" json:"source"` DataType string `gorm:"column:data_type;size:32;not null" json:"data_type"` - Value string `gorm:"column:value;size:1024;not null" json:"value"` + Value string `gorm:"column:value;type:longtext;not null" json:"value"` } // TableName specify table name diff --git a/internal/model/tenant.go b/internal/model/tenant.go index f7f76df8d2..046bd701f7 100644 --- a/internal/model/tenant.go +++ b/internal/model/tenant.go @@ -18,18 +18,24 @@ package model // Tenant tenant model type Tenant struct { - ID string `gorm:"column:id;primaryKey;size:32" json:"id"` - Name *string `gorm:"column:name;size:100;index" json:"name,omitempty"` - PublicKey *string `gorm:"column:public_key;size:255;index" json:"public_key,omitempty"` - LLMID string `gorm:"column:llm_id;size:128;not null;index" json:"llm_id"` - EmbDID string `gorm:"column:embd_id;size:128;not null;index" json:"embd_id"` - ASRID string `gorm:"column:asr_id;size:128;not null;index" json:"asr_id"` - Img2TxtID string `gorm:"column:img2txt_id;size:128;not null;index" json:"img2txt_id"` - RerankID string `gorm:"column:rerank_id;size:128;not null;index" json:"rerank_id"` - TTSID *string `gorm:"column:tts_id;size:256;index" json:"tts_id,omitempty"` - ParserIDs string `gorm:"column:parser_ids;size:256;not null" json:"parser_ids"` - Credit int64 `gorm:"column:credit;default:512;index" json:"credit"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + Name *string `gorm:"column:name;size:100;index" json:"name,omitempty"` + PublicKey *string `gorm:"column:public_key;size:255;index" json:"public_key,omitempty"` + LLMID string `gorm:"column:llm_id;size:128;not null;index" json:"llm_id"` + TenantLLMID *int64 `gorm:"column:tenant_llm_id;index" json:"tenant_llm_id,omitempty"` + EmbdID string `gorm:"column:embd_id;size:128;not null;index" json:"embd_id"` + TenantEmbdID *int64 `gorm:"column:tenant_embd_id;index" json:"tenant_embd_id,omitempty"` + ASRID string `gorm:"column:asr_id;size:128;not null;index" json:"asr_id"` + TenantASRID *int64 `gorm:"column:tenant_asr_id;index" json:"tenant_asr_id,omitempty"` + Img2TxtID string `gorm:"column:img2txt_id;size:128;not null;index" json:"img2txt_id"` + TenantImg2TxtID *int64 `gorm:"column:tenant_img2txt_id;index" json:"tenant_img2txt_id,omitempty"` + RerankID string `gorm:"column:rerank_id;size:128;not null;index" json:"rerank_id"` + TenantRerankID *int64 `gorm:"column:tenant_rerank_id;index" json:"tenant_rerank_id,omitempty"` + TTSID *string `gorm:"column:tts_id;size:256;index" json:"tts_id,omitempty"` + TenantTTSID *int64 `gorm:"column:tenant_tts_id;index" json:"tenant_tts_id,omitempty"` + ParserIDs string `gorm:"column:parser_ids;size:256;not null;index" json:"parser_ids"` + Credit int64 `gorm:"column:credit;default:512;index" json:"credit"` + Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` BaseModel } diff --git a/internal/model/tenant_llm.go b/internal/model/tenant_llm.go index dbadca6bd9..613032372d 100644 --- a/internal/model/tenant_llm.go +++ b/internal/model/tenant_llm.go @@ -17,16 +17,18 @@ package model // TenantLLM tenant LLM model +// Python uses PrimaryKeyField (auto-increment ID) with unique index on (tenant_id, llm_factory, llm_name) type TenantLLM struct { - TenantID string `gorm:"column:tenant_id;size:32;not null;primaryKey" json:"tenant_id"` - LLMFactory string `gorm:"column:llm_factory;size:128;not null;primaryKey" json:"llm_factory"` - ModelType string `gorm:"column:model_type;size:128;not null;index" json:"model_type"` - LLMName string `gorm:"column:llm_name;size:128;not null;primaryKey;default:\"\"" json:"llm_name"` - APIKey string `gorm:"column:api_key;type:longtext" json:"api_key,omitempty"` - APIBase string `gorm:"column:api_base;size:255" json:"api_base,omitempty"` - MaxTokens int64 `gorm:"column:max_tokens;default:8192;index" json:"max_tokens"` - UsedTokens int64 `gorm:"column:used_tokens;default:0;index" json:"used_tokens"` - Status string `gorm:"column:status;size:1;not null;default:1;index" json:"status"` + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + TenantID string `gorm:"column:tenant_id;size:32;not null;index:idx_tenant_llm_unique,unique" json:"tenant_id"` + LLMFactory string `gorm:"column:llm_factory;size:128;not null;index:idx_tenant_llm_unique,unique" json:"llm_factory"` + ModelType *string `gorm:"column:model_type;size:128;index" json:"model_type,omitempty"` + LLMName *string `gorm:"column:llm_name;size:128;index:idx_tenant_llm_unique,unique;default:\"\"" json:"llm_name,omitempty"` + APIKey *string `gorm:"column:api_key;type:longtext" json:"api_key,omitempty"` + APIBase *string `gorm:"column:api_base;size:255" json:"api_base,omitempty"` + MaxTokens int64 `gorm:"column:max_tokens;default:8192;index" json:"max_tokens"` + UsedTokens int64 `gorm:"column:used_tokens;default:0;index" json:"used_tokens"` + Status string `gorm:"column:status;size:1;not null;default:1;index" json:"status"` BaseModel } diff --git a/internal/service/chat.go b/internal/service/chat.go index 3192a2152d..b53706d180 100644 --- a/internal/service/chat.go +++ b/internal/service/chat.go @@ -460,7 +460,7 @@ func (s *ChatService) SetDialog(userID string, req *SetDialogRequest) (*SetDialo KBIDs: kbIDsJSON, Status: strPtr("1"), } - chat.CreateTime = createTime + chat.CreateTime = &createTime chat.CreateDate = &now chat.UpdateTime = &createTime chat.UpdateDate = &now diff --git a/internal/service/chat_session.go b/internal/service/chat_session.go index 7de702e92c..e32dea3fa3 100644 --- a/internal/service/chat_session.go +++ b/internal/service/chat_session.go @@ -139,7 +139,7 @@ func (s *ChatSessionService) SetChatSession(userID string, req *SetChatSessionRe UserID: &userID, Reference: referenceJSON, } - session.CreateTime = createTime + session.CreateTime = &createTime session.CreateDate = &now session.UpdateTime = &createTime session.UpdateDate = &now diff --git a/internal/service/document.go b/internal/service/document.go index 94267b797e..ad94ce647c 100644 --- a/internal/service/document.go +++ b/internal/service/document.go @@ -178,7 +178,10 @@ func (s *DocumentService) GetDocumentsByAuthorID(authorID, page, pageSize int) ( // toResponse convert model.Document to DocumentResponse func (s *DocumentService) toResponse(doc *model.Document) *DocumentResponse { - createdAt := time.Unix(doc.CreateTime, 0).Format("2006-01-02 15:04:05") + createdAt := "" + if doc.CreateTime != nil { + createdAt = time.Unix(*doc.CreateTime, 0).Format("2006-01-02 15:04:05") + } updatedAt := "" if doc.UpdateTime != nil { updatedAt = time.Unix(*doc.UpdateTime, 0).Format("2006-01-02 15:04:05") diff --git a/internal/service/llm.go b/internal/service/llm.go index 5478f3d18f..85b1cd99f8 100644 --- a/internal/service/llm.go +++ b/internal/service/llm.go @@ -114,18 +114,18 @@ func (s *LLMService) GetMyLLMs(tenantID string, includeDetails bool) (map[string // LLMListItem represents a single LLM item in the list response type LLMListItem struct { - LLMName string `json:"llm_name"` - ModelType string `json:"model_type"` - FID string `json:"fid"` - Available bool `json:"available"` - Status string `json:"status"` - MaxTokens int64 `json:"max_tokens,omitempty"` - CreateDate *string `json:"create_date,omitempty"` - CreateTime int64 `json:"create_time,omitempty"` - UpdateDate *string `json:"update_date,omitempty"` - UpdateTime *int64 `json:"update_time,omitempty"` - IsTools bool `json:"is_tools"` - Tags string `json:"tags,omitempty"` + LLMName string `json:"llm_name"` + ModelType string `json:"model_type"` + FID string `json:"fid"` + Available bool `json:"available"` + Status string `json:"status"` + MaxTokens int64 `json:"max_tokens,omitempty"` + CreateDate *string `json:"create_date,omitempty"` + CreateTime *int64 `json:"create_time,omitempty"` + UpdateDate *string `json:"update_date,omitempty"` + UpdateTime *int64 `json:"update_time,omitempty"` + IsTools bool `json:"is_tools"` + Tags string `json:"tags,omitempty"` } // ListLLMsResponse represents the response for list LLMs @@ -153,10 +153,14 @@ func (s *LLMService) ListLLMs(tenantID string, modelType string) (ListLLMsRespon // Build set of valid LLM names@factories status := make(map[string]bool) for _, tl := range tenantLLMs { - if tl.APIKey != "" && tl.Status == "1" { + if tl.APIKey != nil && *tl.APIKey != "" && tl.Status == "1" { facts[tl.LLMFactory] = true } - key := tl.LLMName + "@" + tl.LLMFactory + llmName := "" + if tl.LLMName != nil { + llmName = *tl.LLMName + } + key := llmName + "@" + tl.LLMFactory if tl.Status == "1" { status[key] = true } @@ -223,19 +227,27 @@ func (s *LLMService) ListLLMs(tenantID string, modelType string) (ListLLMsRespon // Add tenant LLMs that are not in the global list for _, tl := range tenantLLMs { - key := tl.LLMName + "@" + tl.LLMFactory + llmName := "" + if tl.LLMName != nil { + llmName = *tl.LLMName + } + key := llmName + "@" + tl.LLMFactory if llmSet[key] { continue } // Filter by model type if specified - if modelType != "" && !strings.Contains(tl.ModelType, modelType) { + modelTypeValue := "" + if tl.ModelType != nil { + modelTypeValue = *tl.ModelType + } + if modelType != "" && !strings.Contains(modelTypeValue, modelType) { continue } item := LLMListItem{ - LLMName: tl.LLMName, - ModelType: tl.ModelType, + LLMName: llmName, + ModelType: modelTypeValue, FID: tl.LLMFactory, Available: true, Status: tl.Status, diff --git a/internal/service/model_service.go b/internal/service/model_service.go index 423c685607..75082485c8 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -80,7 +80,7 @@ func (p *ModelProviderImpl) GetEmbeddingModel(ctx context.Context, tenantID stri } apiKey := embeddingModel.APIKey - if apiKey == "" { + if apiKey == nil || *apiKey == "" { return nil, fmt.Errorf("no API key found for tenant %s and model %s", tenantID, compositeModelName) } // Always get API base from model provider configuration @@ -91,7 +91,7 @@ func (p *ModelProviderImpl) GetEmbeddingModel(ctx context.Context, tenantID stri } apiBase := providerConfig.DefaultEmbeddingURL - return models.CreateEmbeddingModel(provider, apiKey, apiBase, modelName, p.httpClient) + return models.CreateEmbeddingModel(provider, *apiKey, apiBase, modelName, p.httpClient) } // GetChatModel returns a chat model for the given tenant diff --git a/internal/service/user.go b/internal/service/user.go index 99dd5351b5..9db2a26439 100644 --- a/internal/service/user.go +++ b/internal/service/user.go @@ -144,7 +144,7 @@ func (s *UserService) Register(req *RegisterRequest) (*model.User, common.ErrorC } now := time.Now().Unix() - user.CreateTime = now + user.CreateTime = &now user.UpdateTime = &now now_date := time.Now() user.CreateDate = &now_date @@ -156,13 +156,13 @@ func (s *UserService) Register(req *RegisterRequest) (*model.User, common.ErrorC ID: userID, Name: &tenantName, LLMID: cfg.Server.Mode, - EmbDID: cfg.Server.Mode, + EmbdID: cfg.Server.Mode, ASRID: cfg.Server.Mode, Img2TxtID: cfg.Server.Mode, RerankID: cfg.Server.Mode, ParserIDs: "naive:General,Q&A:Q&A,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag", } - tenant.CreateTime = now + tenant.CreateTime = &now tenant.UpdateTime = &now tenant.CreateDate = &now_date tenant.UpdateDate = &now_date @@ -176,7 +176,7 @@ func (s *UserService) Register(req *RegisterRequest) (*model.User, common.ErrorC InvitedBy: userID, Status: &status, } - userTenant.CreateTime = now + userTenant.CreateTime = &now userTenant.UpdateTime = &now userTenant.CreateDate = &now_date userTenant.UpdateDate = &now_date @@ -191,7 +191,7 @@ func (s *UserService) Register(req *RegisterRequest) (*model.User, common.ErrorC Type: "folder", Size: 0, } - rootFile.CreateTime = now + rootFile.CreateTime = &now rootFile.UpdateTime = &now rootFile.CreateDate = &now_date rootFile.UpdateDate = &now_date @@ -205,20 +205,38 @@ func (s *UserService) Register(req *RegisterRequest) (*model.User, common.ErrorC } if err := tenantDAO.Create(tenant); err != nil { - s.userDAO.DeleteByID(userID) + err := s.userDAO.DeleteByID(userID) + if err != nil { + return nil, 0, err + } return nil, common.CodeServerError, fmt.Errorf("failed to create tenant: %w", err) } if err := userTenantDAO.Create(userTenant); err != nil { - s.userDAO.DeleteByID(userID) - tenantDAO.Delete(userID) + err := s.userDAO.DeleteByID(userID) + if err != nil { + return nil, 0, err + } + err = tenantDAO.Delete(userID) + if err != nil { + return nil, 0, err + } return nil, common.CodeServerError, fmt.Errorf("failed to create user tenant relation: %w", err) } if err := fileDAO.Create(rootFile); err != nil { - s.userDAO.DeleteByID(userID) - tenantDAO.Delete(userID) - userTenantDAO.Delete(userTenantID) + err := s.userDAO.DeleteByID(userID) + if err != nil { + return nil, 0, err + } + err = tenantDAO.Delete(userID) + if err != nil { + return nil, 0, err + } + err = userTenantDAO.Delete(userTenantID) + if err != nil { + return nil, 0, err + } return nil, common.CodeServerError, fmt.Errorf("failed to create root folder: %w", err) } @@ -314,11 +332,16 @@ func (s *UserService) GetUserByID(id uint) (*UserResponse, common.ErrorCode, err } return &UserResponse{ - ID: user.ID, - Email: user.Email, - Nickname: user.Nickname, - Status: user.Status, - CreatedAt: time.Unix(user.CreateTime, 0).Format("2006-01-02 15:04:05"), + ID: user.ID, + Email: user.Email, + Nickname: user.Nickname, + Status: user.Status, + CreatedAt: func() string { + if user.CreateTime != nil { + return time.Unix(*user.CreateTime, 0).Format("2006-01-02 15:04:05") + } + return "" + }(), }, common.CodeSuccess, nil } @@ -333,11 +356,16 @@ func (s *UserService) ListUsers(page, pageSize int) ([]*UserResponse, int64, com responses := make([]*UserResponse, len(users)) for i, user := range users { responses[i] = &UserResponse{ - ID: user.ID, - Email: user.Email, - Nickname: user.Nickname, - Status: user.Status, - CreatedAt: time.Unix(user.CreateTime, 0).Format("2006-01-02 15:04:05"), + ID: user.ID, + Email: user.Email, + Nickname: user.Nickname, + Status: user.Status, + CreatedAt: func() string { + if user.CreateTime != nil { + return time.Unix(*user.CreateTime, 0).Format("2006-01-02 15:04:05") + } + return "" + }(), } } diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index 54f89d3486..9fe895e711 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -347,8 +347,7 @@ func (p *analyzerPool) Close() { p.baseAnalyzer = nil } - logger.Info("Analyzer pool closed", - zap.Int32("final_size", atomic.LoadInt32(&p.currentSize))) + logger.Info(fmt.Sprintf("Analyzer pool closed, final_size: %d", atomic.LoadInt32(&p.currentSize))) } // GetPoolStats returns current pool statistics