From 8bb2cd0fb7f1528e64a6959a9f033dc751996f10 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Thu, 16 Jul 2026 16:00:56 +0800 Subject: [PATCH] Go: fix db alignment (#16964) ### Summary Align go db schema with python --------- Signed-off-by: Jin Hai --- internal/dao/connector.go | 2 +- internal/entity/compilation_template.go | 36 +++++++++++++++++++ internal/entity/compilation_template_group.go | 33 +++++++++++++++++ internal/entity/connector.go | 4 +-- internal/entity/dataset.go | 8 +++-- internal/entity/document.go | 6 ++-- internal/entity/file_commit.go | 28 +++++++++------ internal/entity/llm.go | 2 +- internal/entity/tenant.go | 10 +++--- internal/entity/tenant_langfuse.go | 2 +- internal/entity/tenant_llm.go | 12 +++---- internal/entity/user.go | 6 ++-- internal/entity/user_tenant.go | 2 +- internal/handler/document.go | 2 +- .../ingestion/component/dispatch_model.go | 4 +-- internal/service/chunk/chunk_test.go | 5 ++- internal/service/model_service.go | 4 +-- internal/service/tenant.go | 4 +-- internal/service/user.go | 4 +-- 19 files changed, 129 insertions(+), 45 deletions(-) create mode 100644 internal/entity/compilation_template.go create mode 100644 internal/entity/compilation_template_group.go diff --git a/internal/dao/connector.go b/internal/dao/connector.go index 00b4847423..66f6dd3607 100644 --- a/internal/dao/connector.go +++ b/internal/dao/connector.go @@ -346,7 +346,7 @@ func scheduleConnectorTask(tx *gorm.DB, connectorID, kbID, taskType string, rein return nil } - var pollRangeStart *string + var pollRangeStart *time.Time var totalDocsIndexed int64 if taskType == connectorTaskTypeSync { var latest entity.SyncLogs diff --git a/internal/entity/compilation_template.go b/internal/entity/compilation_template.go new file mode 100644 index 0000000000..c788526e81 --- /dev/null +++ b/internal/entity/compilation_template.go @@ -0,0 +1,36 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package entity + +// CompilationTemplate represents a compilation template +type CompilationTemplate struct { + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + TenantID *string `gorm:"column:tenant_id;size:32;index" json:"tenant_id,omitempty"` + GroupID *string `gorm:"column:group_id;size:32;index" json:"group_id,omitempty"` + Name string `gorm:"column:name;size:128;not null;index" json:"name"` + Description *string `gorm:"column:description;type:text" json:"description,omitempty"` + Kind string `gorm:"column:kind;size:64;not null;index" json:"kind"` + Config JSONMap `gorm:"column:config;type:text;not null;default:'{}'" json:"config"` + IsBuiltin bool `gorm:"column:is_builtin;not null;default:false;index" json:"is_builtin"` + Status *string `gorm:"column:status;size:1;default:1;index" json:"status,omitempty"` + BaseModel +} + +// TableName returns the table name for CompilationTemplate model +func (CompilationTemplate) TableName() string { + return "compilation_template" +} diff --git a/internal/entity/compilation_template_group.go b/internal/entity/compilation_template_group.go new file mode 100644 index 0000000000..4d60c2f76c --- /dev/null +++ b/internal/entity/compilation_template_group.go @@ -0,0 +1,33 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package entity + +// CompilationTemplateGroup represents a group for compilation templates +type CompilationTemplateGroup struct { + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` + Name string `gorm:"column:name;size:128;not null;index" json:"name"` + Description *string `gorm:"column:description;type:text" json:"description,omitempty"` + Scope string `gorm:"column:scope;size:16;not null;index" json:"scope"` + Status *string `gorm:"column:status;size:1;default:1;index" json:"status,omitempty"` + BaseModel +} + +// TableName returns the table name for CompilationTemplateGroup model +func (CompilationTemplateGroup) TableName() string { + return "compilation_template_group" +} diff --git a/internal/entity/connector.go b/internal/entity/connector.go index 0658b42f60..a8dd73c121 100644 --- a/internal/entity/connector.go +++ b/internal/entity/connector.go @@ -117,8 +117,8 @@ type SyncLogs struct { ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"` FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"` TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"` - PollRangeStart *string `gorm:"column:poll_range_start;size:255;index" json:"poll_range_start,omitempty"` - PollRangeEnd *string `gorm:"column:poll_range_end;size:255;index" json:"poll_range_end,omitempty"` + PollRangeStart *time.Time `gorm:"column:poll_range_start;index" json:"poll_range_start,omitempty"` + PollRangeEnd *time.Time `gorm:"column:poll_range_end;index" json:"poll_range_end,omitempty"` KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"` BaseModel } diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go index f89c56eff8..38bbdb16a2 100644 --- a/internal/entity/dataset.go +++ b/internal/entity/dataset.go @@ -100,7 +100,7 @@ type Knowledgebase struct { Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"` TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` Name string `gorm:"column:name;size:128;not null;index" json:"name"` - Language *string `gorm:"column:language;size:32;index" json:"language,omitempty"` + Language *string `gorm:"column:language;size:32;index;default:'English'" json:"language,omitempty"` Description *string `gorm:"column:description;type:longtext" json:"description,omitempty"` EmbdID string `gorm:"column:embd_id;size:128;not null;index" json:"embd_id"` TenantEmbdID *string `gorm:"column:tenant_embd_id;size:32;index" json:"tenant_embd_id,omitempty"` @@ -121,7 +121,11 @@ type Knowledgebase struct { RaptorTaskFinishAt *time.Time `gorm:"column:raptor_task_finish_at" json:"raptor_task_finish_at,omitempty"` MindmapTaskID *string `gorm:"column:mindmap_task_id;size:32;index" json:"mindmap_task_id,omitempty"` MindmapTaskFinishAt *time.Time `gorm:"column:mindmap_task_finish_at" json:"mindmap_task_finish_at,omitempty"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + ArtifactTaskID *string `gorm:"column:artifact_task_id;size:32;index" json:"artifact_task_id,omitempty"` + ArtifactTaskFinishAt *time.Time `gorm:"column:artifact_task_finish_at" json:"artifact_task_finish_at,omitempty"` + SkillTaskID *string `gorm:"column:skill_task_id;size:32;index" json:"skill_task_id,omitempty"` + SkillTaskFinishAt *time.Time `gorm:"column:skill_task_finish_at" json:"skill_task_finish_at,omitempty"` + Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"` BaseModel } diff --git a/internal/entity/document.go b/internal/entity/document.go index 9b25ff4663..4b3683b34e 100644 --- a/internal/entity/document.go +++ b/internal/entity/document.go @@ -38,11 +38,11 @@ type Document struct { ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"` ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"` ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"` - ContentHash *string `gorm:"column:content_hash;size:32;index" json:"content_hash,omitempty"` + ContentHash *string `gorm:"column:content_hash;size:32;index;default:''" json:"content_hash,omitempty"` MetaFields *JSONMap `gorm:"column:meta_fields;type:longtext" json:"meta_fields,omitempty"` Suffix string `gorm:"column:suffix;size:32;not null;index" json:"suffix"` - Run *string `gorm:"column:run;size:1;index" json:"run,omitempty"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + Run *string `gorm:"column:run;size:1;index;default:'0'" json:"run,omitempty"` + Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"` BaseModel } diff --git a/internal/entity/file_commit.go b/internal/entity/file_commit.go index 356952bffa..9e98b1c645 100644 --- a/internal/entity/file_commit.go +++ b/internal/entity/file_commit.go @@ -25,6 +25,8 @@ type FileCommit struct { AuthorID string `gorm:"column:author_id;size:32;not null;index" json:"author_id"` FileCount int `gorm:"column:file_count;default:0" json:"file_count"` TreeState *string `gorm:"column:tree_state;type:longtext" json:"tree_state,omitempty"` + Title *string `gorm:"column:title;size:255" json:"title,omitempty"` + Comments *string `gorm:"column:comments;type:longtext" json:"comments,omitempty"` BaseModel } @@ -35,16 +37,22 @@ func (FileCommit) TableName() string { // FileCommitItem represents a single file change within a commit. type FileCommitItem struct { - ID string `gorm:"column:id;primaryKey;size:32" json:"id"` - CommitID string `gorm:"column:commit_id;size:32;not null;uniqueIndex:idx_commit_file" json:"commit_id"` - FileID string `gorm:"column:file_id;size:32;not null;uniqueIndex:idx_commit_file" json:"file_id"` - Operation string `gorm:"column:operation;size:16;not null;index" json:"operation"` - OldHash *string `gorm:"column:old_hash;size:64;index" json:"old_hash,omitempty"` - NewHash *string `gorm:"column:new_hash;size:64;index" json:"new_hash,omitempty"` - OldLocation *string `gorm:"column:old_location;size:255" json:"old_location,omitempty"` - NewLocation *string `gorm:"column:new_location;size:255" json:"new_location,omitempty"` - OldName *string `gorm:"column:old_name;size:255" json:"old_name,omitempty"` - NewName *string `gorm:"column:new_name;size:255" json:"new_name,omitempty"` + ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + CommitID string `gorm:"column:commit_id;size:32;not null;uniqueIndex:idx_commit_file" json:"commit_id"` + FileID string `gorm:"column:file_id;size:32;not null;uniqueIndex:idx_commit_file" json:"file_id"` + Operation string `gorm:"column:operation;size:16;not null;index" json:"operation"` + OldHash *string `gorm:"column:old_hash;size:64;index" json:"old_hash,omitempty"` + NewHash *string `gorm:"column:new_hash;size:64;index" json:"new_hash,omitempty"` + OldLocation *string `gorm:"column:old_location;size:255" json:"old_location,omitempty"` + NewLocation *string `gorm:"column:new_location;size:255" json:"new_location,omitempty"` + OldName *string `gorm:"column:old_name;size:255" json:"old_name,omitempty"` + NewName *string `gorm:"column:new_name;size:255" json:"new_name,omitempty"` + Diff *string `gorm:"column:diff;type:longtext" json:"diff,omitempty"` + ContentAfterStorage *string `gorm:"column:content_after_storage;size:16;index" json:"content_after_storage,omitempty"` + ContentAfterLocation *string `gorm:"column:content_after_location;size:512" json:"content_after_location,omitempty"` + SlugKwd *string `gorm:"column:slug_kwd;size:512;index" json:"slug_kwd,omitempty"` + PageTypeKwd *string `gorm:"column:page_type_kwd;size:32;index" json:"page_type_kwd,omitempty"` + BaseModel } diff --git a/internal/entity/llm.go b/internal/entity/llm.go index a2b956a42f..89609e4c2d 100644 --- a/internal/entity/llm.go +++ b/internal/entity/llm.go @@ -24,7 +24,7 @@ type LLM struct { MaxTokens int64 `gorm:"column:max_tokens;default:0" json:"max_tokens"` Tags string `gorm:"column:tags;size:255;not null;index" json:"tags"` IsTools bool `gorm:"column:is_tools;default:false" json:"is_tools"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"` BaseModel } diff --git a/internal/entity/tenant.go b/internal/entity/tenant.go index 9cbd324b64..844d82eb32 100644 --- a/internal/entity/tenant.go +++ b/internal/entity/tenant.go @@ -31,13 +31,13 @@ type Tenant struct { TenantImg2TxtID *string `gorm:"column:tenant_img2txt_id;size:32;index" json:"tenant_img2txt_id,omitempty"` RerankID string `gorm:"column:rerank_id;size:128;not null;index" json:"rerank_id"` TenantRerankID *string `gorm:"column:tenant_rerank_id;size:32;index" json:"tenant_rerank_id,omitempty"` - TTSID string `gorm:"column:tts_id;size:256;index" json:"tts_id,omitempty"` + TTSID *string `gorm:"column:tts_id;size:256;index" json:"tts_id,omitempty"` TenantTTSID *string `gorm:"column:tenant_tts_id;size:32;index" json:"tenant_tts_id,omitempty"` ParserIDs string `gorm:"column:parser_ids;size:256;not null;index" json:"parser_ids"` - OCRID string `gorm:"column:ocr_id;size:256" json:"ocr_id,omitempty"` - TenantOCRID *string `gorm:"column:tenant_ocr_id;size:32" json:"tenant_ocr_id,omitempty"` - Credit int64 `gorm:"column:credit;default:512;index" json:"credit"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + OCRID *string `gorm:"column:ocr_id;size:256;index" json:"ocr_id,omitempty"` + TenantOCRID *string `gorm:"column:tenant_ocr_id;size:32;index" json:"tenant_ocr_id,omitempty"` + Credit int `gorm:"column:credit;default:512;index" json:"credit"` + Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"` BaseModel } diff --git a/internal/entity/tenant_langfuse.go b/internal/entity/tenant_langfuse.go index f7cac41e60..03da5c7bc6 100644 --- a/internal/entity/tenant_langfuse.go +++ b/internal/entity/tenant_langfuse.go @@ -21,7 +21,7 @@ type TenantLangfuse struct { TenantID string `gorm:"column:tenant_id;primaryKey;size:32" json:"tenant_id"` SecretKey string `gorm:"column:secret_key;size:2048;not null" json:"secret_key"` PublicKey string `gorm:"column:public_key;size:2048;not null" json:"public_key"` - Host string `gorm:"column:host;size:128;not null;index" json:"host"` + Host string `gorm:"column:host;size:128;not null" json:"host"` BaseModel } diff --git a/internal/entity/tenant_llm.go b/internal/entity/tenant_llm.go index 319158c7fc..9b3829c6b9 100644 --- a/internal/entity/tenant_llm.go +++ b/internal/entity/tenant_llm.go @@ -20,14 +20,14 @@ package entity // Python uses PrimaryKeyField (auto-increment ID) with unique index on (tenant_id, llm_factory, llm_name) type TenantLLM struct { ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` - TenantID string `gorm:"column:tenant_id;size:32;not null;index:idx_tenant_llm_unique,unique" json:"tenant_id"` - LLMFactory string `gorm:"column:llm_factory;size:128;not null;index:idx_tenant_llm_unique,unique" json:"llm_factory"` + TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` + LLMFactory string `gorm:"column:llm_factory;size:128;not null" json:"llm_factory"` ModelType *string `gorm:"column:model_type;size:128;index" json:"model_type,omitempty"` - LLMName *string `gorm:"column:llm_name;size:128;index:idx_tenant_llm_unique,unique;default:\"\"" json:"llm_name,omitempty"` - APIKey *string `gorm:"column:api_key;type:longtext" json:"api_key,omitempty"` + LLMName *string `gorm:"column:llm_name;size:128;index;default:''" json:"llm_name,omitempty"` + APIKey *string `gorm:"column:api_key;size:8192" json:"api_key,omitempty"` APIBase *string `gorm:"column:api_base;size:255" json:"api_base,omitempty"` - MaxTokens int64 `gorm:"column:max_tokens;default:8192;index" json:"max_tokens"` - UsedTokens int64 `gorm:"column:used_tokens;default:0;index" json:"used_tokens"` + MaxTokens int64 `gorm:"column:max_tokens;default:8192" json:"max_tokens"` + UsedTokens int64 `gorm:"column:used_tokens;default:0" json:"used_tokens"` Status string `gorm:"column:status;size:1;not null;default:1;index" json:"status"` BaseModel } diff --git a/internal/entity/user.go b/internal/entity/user.go index d30246c0a1..7d70f9197b 100644 --- a/internal/entity/user.go +++ b/internal/entity/user.go @@ -26,9 +26,9 @@ type User struct { Password *string `gorm:"column:password;size:255;index" json:"-"` Email string `gorm:"column:email;size:255;not null;unique" json:"email"` Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"` - Language *string `gorm:"column:language;size:32;index" json:"language,omitempty"` - ColorSchema *string `gorm:"column:color_schema;size:32;index" json:"color_schema,omitempty"` - Timezone *string `gorm:"column:timezone;size:64;index" json:"timezone,omitempty"` + Language *string `gorm:"column:language;size:32;index;default:'English'" json:"language,omitempty"` + ColorSchema *string `gorm:"column:color_schema;size:32;index;default:'Bright'" json:"color_schema,omitempty"` + Timezone *string `gorm:"column:timezone;size:64;index;default:'UTC+8\tAsia/Shanghai'" json:"timezone,omitempty"` LastLoginTime *time.Time `gorm:"column:last_login_time;index" json:"last_login_time,omitempty"` IsAuthenticated string `gorm:"column:is_authenticated;size:1;not null;default:1;index" json:"is_authenticated"` IsActive string `gorm:"column:is_active;size:1;not null;default:1;index" json:"is_active"` diff --git a/internal/entity/user_tenant.go b/internal/entity/user_tenant.go index 7fbc719860..f46b47a74a 100644 --- a/internal/entity/user_tenant.go +++ b/internal/entity/user_tenant.go @@ -23,7 +23,7 @@ type UserTenant struct { TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"` Role string `gorm:"column:role;size:32;not null;index" json:"role"` InvitedBy string `gorm:"column:invited_by;size:32;not null;index" json:"invited_by"` - Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"` + Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"` BaseModel } diff --git a/internal/handler/document.go b/internal/handler/document.go index 583a6220b3..4f2dc1d5a0 100644 --- a/internal/handler/document.go +++ b/internal/handler/document.go @@ -1417,7 +1417,7 @@ func (h *DocumentHandler) ListIngestionTasks(c *gin.Context) { } type StartParseDocumentsRequest struct { - DatasetID string `json:"dataset_id" binding:"required"` + DatasetID string `json:"dataset_id"` Documents []string `json:"documents" binding:"required"` } diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go index 1386849a1a..b88fca9334 100644 --- a/internal/ingestion/component/dispatch_model.go +++ b/internal/ingestion/component/dispatch_model.go @@ -56,9 +56,9 @@ func defaultResolveTenantModelByType(tenantID string, modelType entity.ModelType case entity.ModelTypeImage2Text: modelID = tenant.Img2TxtID case entity.ModelTypeTTS: - modelID = tenant.TTSID + modelID = *tenant.TTSID case entity.ModelTypeOCR: - modelID = tenant.OCRID + modelID = *tenant.OCRID default: return nil, "", nil, 0, fmt.Errorf("invalid model type: %s", modelType) } diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go index 97f5509eb8..fc6c57b034 100644 --- a/internal/service/chunk/chunk_test.go +++ b/internal/service/chunk/chunk_test.go @@ -169,9 +169,12 @@ func TestParsePrevalidatesDocumentsBeforeMutating(t *testing.T) { if err != nil { t.Fatalf("get doc: %v", err) } - if doc.Run != nil { + if doc.Run == nil { t.Fatalf("expected doc run to remain nil, got %q", *doc.Run) } + if *doc.Run != "0" { + t.Fatalf("expected doc run status is '1', got %q", *doc.Run) + } if doc.ChunkNum != 7 { t.Fatalf("expected chunk_num to remain 7, got %d", doc.ChunkNum) } diff --git a/internal/service/model_service.go b/internal/service/model_service.go index 763f7b8f60..27ce1bd8e0 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -3019,9 +3019,9 @@ func defaultModelRefs(tenant *entity.Tenant, modelType entity.ModelType) (string case entity.ModelTypeImage2Text: return tenant.Img2TxtID, ptrStringValue(tenant.TenantImg2TxtID) case entity.ModelTypeTTS: - return tenant.TTSID, ptrStringValue(tenant.TenantTTSID) + return *tenant.TTSID, ptrStringValue(tenant.TenantTTSID) case entity.ModelTypeOCR: - return tenant.OCRID, ptrStringValue(tenant.TenantOCRID) + return *tenant.OCRID, ptrStringValue(tenant.TenantOCRID) default: return "", "" } diff --git a/internal/service/tenant.go b/internal/service/tenant.go index fda50f4066..d54fa19c4a 100644 --- a/internal/service/tenant.go +++ b/internal/service/tenant.go @@ -478,9 +478,9 @@ func (s *TenantService) GetDefaultModelName(tenantID string, modelType entity.Mo case entity.ModelTypeImage2Text: modelID = tenant.Img2TxtID case entity.ModelTypeTTS: - modelID = tenant.TTSID + modelID = *tenant.TTSID case entity.ModelTypeOCR: - modelID = tenant.OCRID + modelID = *tenant.OCRID default: return "", fmt.Errorf("invalid model type: %s", modelType) } diff --git a/internal/service/user.go b/internal/service/user.go index 42db4af8dd..7c29c20dec 100644 --- a/internal/service/user.go +++ b/internal/service/user.go @@ -199,8 +199,8 @@ func (s *UserService) Register(req *RegisterRequest) (*entity.User, common.Error ASRID: asrID, Img2TxtID: img2txtID, RerankID: rerankID, - TTSID: ttsID, - OCRID: ocrID, + TTSID: &ttsID, + OCRID: &ocrID, ParserIDs: "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag", Status: &status, }