mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-21 23:21:04 +08:00
274 lines
9.4 KiB
Go
274 lines
9.4 KiB
Go
//
|
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
package document
|
|
|
|
import (
|
|
"errors"
|
|
"ragflow/internal/service"
|
|
"regexp"
|
|
"time"
|
|
|
|
"ragflow/internal/dao"
|
|
"ragflow/internal/engine"
|
|
)
|
|
|
|
// DocumentService document service
|
|
type DocumentService struct {
|
|
documentDAO *dao.DocumentDAO
|
|
kbDAO *dao.KnowledgebaseDAO
|
|
ingestionTaskDAO *dao.IngestionTaskDAO
|
|
ingestionTaskLogDAO *dao.IngestionTaskLogDAO
|
|
ingestionTaskSvc *service.IngestionTaskService
|
|
docEngine engine.DocEngine
|
|
metadataSvc *service.MetadataService
|
|
taskDAO *dao.TaskDAO
|
|
file2DocumentDAO *dao.File2DocumentDAO
|
|
fileDAO *dao.FileDAO
|
|
canvasDAO *dao.UserCanvasDAO
|
|
api4ConvDAO *dao.API4ConversationDAO
|
|
}
|
|
|
|
// NewDocumentService create document service
|
|
func NewDocumentService() *DocumentService {
|
|
publisher := service.NewMessageQueueTaskPublisher()
|
|
ingestionTaskSvc := service.NewIngestionTaskService()
|
|
ingestionTaskSvc.SetTaskPublisher(publisher)
|
|
return &DocumentService{
|
|
documentDAO: dao.NewDocumentDAO(),
|
|
ingestionTaskDAO: dao.NewIngestionTaskDAO(),
|
|
ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
|
|
ingestionTaskSvc: ingestionTaskSvc,
|
|
kbDAO: dao.NewKnowledgebaseDAO(),
|
|
docEngine: engine.Get(),
|
|
metadataSvc: service.NewMetadataService(),
|
|
taskDAO: dao.NewTaskDAO(),
|
|
file2DocumentDAO: dao.NewFile2DocumentDAO(),
|
|
fileDAO: dao.NewFileDAO(),
|
|
canvasDAO: dao.NewUserCanvasDAO(),
|
|
api4ConvDAO: dao.NewAPI4ConversationDAO(),
|
|
}
|
|
}
|
|
|
|
// UpdateDocumentRequest update document request
|
|
type UpdateDocumentRequest struct {
|
|
Name *string `json:"name"`
|
|
Run *string `json:"run"`
|
|
TokenNum *int64 `json:"token_num"`
|
|
ChunkNum *int64 `json:"chunk_num"`
|
|
Progress *float64 `json:"progress"`
|
|
ProgressMsg *string `json:"progress_msg"`
|
|
}
|
|
|
|
// DocumentResponse document response
|
|
type DocumentResponse struct {
|
|
ID string `json:"id"`
|
|
Name *string `json:"name,omitempty"`
|
|
KbID string `json:"kb_id"`
|
|
ParserID string `json:"parser_id"`
|
|
PipelineID *string `json:"pipeline_id,omitempty"`
|
|
Type string `json:"type"`
|
|
SourceType string `json:"source_type"`
|
|
CreatedBy string `json:"created_by"`
|
|
Location *string `json:"location,omitempty"`
|
|
Size int64 `json:"size"`
|
|
TokenNum int64 `json:"token_num"`
|
|
ChunkNum int64 `json:"chunk_num"`
|
|
Progress float64 `json:"progress"`
|
|
ProgressMsg *string `json:"progress_msg,omitempty"`
|
|
ProcessDuration float64 `json:"process_duration"`
|
|
Suffix string `json:"suffix"`
|
|
Run *string `json:"run,omitempty"`
|
|
Status *string `json:"status,omitempty"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
}
|
|
|
|
type ThumbnailResponse struct {
|
|
ID string `json:"id"`
|
|
Thumbnail *string `json:"thumbnail,omitempty"`
|
|
KbID string `json:"kb_id"`
|
|
}
|
|
|
|
const imgBase64Prefix = "data:image/png;base64,"
|
|
|
|
type ArtifactResponse struct {
|
|
Data []byte
|
|
ContentType string
|
|
SafeFilename string
|
|
ForceAttachment bool
|
|
}
|
|
|
|
type UpdateDatasetDocumentRequest struct {
|
|
Name *string `json:"name"`
|
|
ParserID *string `json:"parser_id"`
|
|
ChunkCount *int64 `json:"chunk_count"`
|
|
TokenCount *int64 `json:"token_count"`
|
|
PipelineID *string `json:"pipeline_id"`
|
|
Enabled *int `json:"enabled"`
|
|
Progress *float64 `json:"progress"`
|
|
ParserConfig map[string]any `json:"parser_config"`
|
|
MetaFields map[string]any `json:"meta_fields"`
|
|
}
|
|
|
|
// PATCH /api/v1/datasets/:dataset_id/documents/:document_id.
|
|
type UpdateDatasetDocumentResponse struct {
|
|
ID string `json:"id"`
|
|
Thumbnail *string `json:"thumbnail,omitempty"`
|
|
DatasetID string `json:"dataset_id"`
|
|
ParserID string `json:"parser_id"`
|
|
PipelineID *string `json:"pipeline_id,omitempty"`
|
|
ParserConfig map[string]interface{} `json:"parser_config"`
|
|
SourceType string `json:"source_type"`
|
|
Type string `json:"type"`
|
|
CreatedBy string `json:"created_by"`
|
|
Name *string `json:"name,omitempty"`
|
|
Location *string `json:"location,omitempty"`
|
|
Size int64 `json:"size"`
|
|
TokenCount int64 `json:"token_count"`
|
|
ChunkCount int64 `json:"chunk_count"`
|
|
Progress float64 `json:"progress"`
|
|
ProgressMsg *string `json:"progress_msg,omitempty"`
|
|
ProcessBeginAt *time.Time `json:"process_begin_at,omitempty"`
|
|
ProcessDuration float64 `json:"process_duration"`
|
|
ContentHash *string `json:"content_hash,omitempty"`
|
|
MetaFields map[string]interface{} `json:"meta_fields,omitempty"`
|
|
Suffix string `json:"suffix"`
|
|
Run string `json:"run"`
|
|
Status *string `json:"status,omitempty"`
|
|
CreateTime *int64 `json:"create_time,omitempty"`
|
|
CreateDate *time.Time `json:"create_date,omitempty"`
|
|
UpdateTime *int64 `json:"update_time,omitempty"`
|
|
UpdateDate *time.Time `json:"update_date,omitempty"`
|
|
}
|
|
|
|
var (
|
|
ErrArtifactInvalidFilename = errors.New("Invalid filename.")
|
|
ErrArtifactInvalidFileType = errors.New("Invalid file type.")
|
|
ErrArtifactNotFound = errors.New("Artifact not found.")
|
|
)
|
|
|
|
var artifactContentTypes = map[string]string{
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".svg": "image/svg+xml",
|
|
".pdf": "application/pdf",
|
|
".csv": "text/csv",
|
|
".json": "application/json",
|
|
".html": "text/html",
|
|
}
|
|
|
|
var artifactForceAttachmentExtensions = map[string]struct{}{
|
|
".htm": {},
|
|
".html": {},
|
|
".shtml": {},
|
|
".xht": {},
|
|
".xhtml": {},
|
|
".xml": {},
|
|
".mhtml": {},
|
|
".svg": {},
|
|
}
|
|
var artifactForceAttachmentContentTypes = map[string]struct{}{
|
|
"text/html": {},
|
|
"image/svg+xml": {},
|
|
"application/xhtml+xml": {},
|
|
"text/xml": {},
|
|
"application/xml": {},
|
|
"multipart/related": {},
|
|
}
|
|
|
|
var artifactUnsafeFilenameChars = regexp.MustCompile(`[^\pL\pN_.-]`)
|
|
|
|
type DocumentPreview struct {
|
|
Data []byte
|
|
ContentType string
|
|
FileName string
|
|
}
|
|
|
|
type DownloadDocumentResp struct {
|
|
Data []byte
|
|
FileName string
|
|
ContentType string
|
|
}
|
|
|
|
type IngestDocumentRequest struct {
|
|
DocIDs []string `json:"doc_ids" binding:"required"`
|
|
Run interface{} `json:"run" binding:"required"`
|
|
Delete bool `json:"delete"`
|
|
ApplyKB bool `json:"apply_kb"`
|
|
}
|
|
|
|
// StartParseOptions controls StartParseDocuments behavior.
|
|
type StartParseOptions struct {
|
|
// ApplyKB merges the knowledgebase's parser_config (llm_id, metadata)
|
|
// into the document before parsing.
|
|
ApplyKB bool
|
|
// RerunWithDelete clears prior chunks/tasks/counters before re-parsing.
|
|
RerunWithDelete bool
|
|
}
|
|
|
|
// GetMetadataSummaryRequest request for metadata summary
|
|
type GetMetadataSummaryRequest struct {
|
|
KBID string `json:"kb_id" binding:"required"`
|
|
DocIDs []string `json:"doc_ids"`
|
|
}
|
|
|
|
// GetMetadataSummaryResponse response for metadata summary
|
|
type GetMetadataSummaryResponse struct {
|
|
Summary map[string]interface{} `json:"summary"`
|
|
}
|
|
|
|
// valueInfo holds count and order of first appearance
|
|
type valueInfo struct {
|
|
count int
|
|
firstOrder int
|
|
}
|
|
|
|
// knowledgebaseFolderName is the file-manager folder under each tenant's root
|
|
// that holds per-dataset subfolders, mirroring Python KNOWLEDGEBASE_FOLDER_NAME.
|
|
const knowledgebaseFolderName = ".knowledgebase"
|
|
|
|
// maxUploadDocSize bounds a single uploaded file held in memory, mirroring the
|
|
// Python DOC_MAXIMUM_SIZE default (128 MiB; overridable there via MAX_CONTENT_LENGTH).
|
|
const maxUploadDocSize = 128 * 1024 * 1024
|
|
|
|
// MetadataUpdate is one update item: set key to value.
|
|
type DocumentMetadataUpdate struct {
|
|
Key string `json:"key"`
|
|
Value interface{} `json:"value"`
|
|
Match interface{} `json:"match,omitempty"`
|
|
ValueType string `json:"valueType,omitempty"`
|
|
}
|
|
|
|
// MetadataDelete removes a whole key, or a specific value from a list field.
|
|
type DocumentMetadataDelete struct {
|
|
Key string `json:"key"`
|
|
Value interface{} `json:"value,omitempty"`
|
|
}
|
|
|
|
// MetadataSelector selects which documents to target.
|
|
type DocumentMetadataSelector struct {
|
|
DocumentIDs []string `json:"document_ids"`
|
|
MetadataCondition map[string]interface{} `json:"metadata_condition"`
|
|
}
|
|
|
|
// BatchUpdateDocumentMetadatasResponse summarises the operation.
|
|
type BatchUpdateDocumentMetadatasResponse struct {
|
|
Updated int `json:"updated"`
|
|
MatchedDocs int `json:"matched_docs"`
|
|
}
|