diff --git a/internal/ingestion/pipeline/builtin_registry.go b/internal/ingestion/pipeline/builtin_registry.go index 01aab3870a..d2381b771c 100644 --- a/internal/ingestion/pipeline/builtin_registry.go +++ b/internal/ingestion/pipeline/builtin_registry.go @@ -8,7 +8,6 @@ import ( "io/fs" "os" "path/filepath" - "regexp" "sort" "strings" "sync" @@ -188,39 +187,6 @@ func (r *Registry) IsValid(ref string) bool { return ok } -// fileTypeParserOverrides maps a file type value to the canonical parser_id that -// must be used for all files of that type. The type values are FileType constants -// defined in the utility package. -var fileTypeParserOverrides = map[string]string{ - "visual": "picture", - "aural": "audio", -} - -var ( - presentationExtPattern = regexp.MustCompile(`(?i)\.(ppt|pptx|pages)$`) - emailExtPattern = regexp.MustCompile(`(?i)\.(msg|eml)$`) -) - -// DefaultParserID returns the canonical parser_id for a document given its file -// type, filename and the dataset-level parser_id. File-type-based overrides -// (e.g. "visual" → "picture", "aural" → "audio") take precedence, followed by -// filename extension heuristics (presentation / email), then falling back to -// the dataset's parser_id. -func (r *Registry) DefaultParserID(fileType, filename, fallback string) string { - if override, ok := fileTypeParserOverrides[strings.ToLower(fileType)]; ok { - return override - } - base := filepath.Base(strings.TrimSpace(filename)) - switch { - case presentationExtPattern.MatchString(base): - return "presentation" - case emailExtPattern.MatchString(base): - return "email" - default: - return fallback - } -} - func parseTemplate(filename string, raw []byte) (*BuiltinPipeline, error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() diff --git a/internal/ingestion/pipeline/builtin_registry_test.go b/internal/ingestion/pipeline/builtin_registry_test.go index be645052a3..f35d33746e 100644 --- a/internal/ingestion/pipeline/builtin_registry_test.go +++ b/internal/ingestion/pipeline/builtin_registry_test.go @@ -241,37 +241,3 @@ func TestLoadBuiltinDSL_UnknownFails(t *testing.T) { t.Fatal("LoadBuiltinDSL(nonexistent) = nil, want error") } } - -func TestDefaultRegistry_DefaultParserID(t *testing.T) { - r, err := DefaultRegistry() - if err != nil { - t.Fatalf("DefaultRegistry: %v", err) - } - - tests := []struct { - name string - fileType string - filename string - fallback string - want string - }{ - {name: "visual", fileType: "visual", filename: "img.png", fallback: "naive", want: "picture"}, - {name: "aural", fileType: "aural", filename: "audio.mp3", fallback: "naive", want: "audio"}, - {name: "presentation by ext", fileType: "doc", filename: "deck.pptx", fallback: "naive", want: "presentation"}, - {name: "email by ext", fileType: "doc", filename: "mail.eml", fallback: "naive", want: "email"}, - {name: "pages as presentation", fileType: "doc", filename: "slides.pages", fallback: "naive", want: "presentation"}, - {name: "msg as email", fileType: "doc", filename: "message.msg", fallback: "naive", want: "email"}, - {name: "fallback default", fileType: "doc", filename: "notes.txt", fallback: "manual", want: "manual"}, - {name: "unknown filetype fallback", fileType: "unknown", filename: "file.bin", fallback: "general", want: "general"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := r.DefaultParserID(tt.fileType, tt.filename, tt.fallback) - if got != tt.want { - t.Fatalf("DefaultParserID(%q, %q, %q) = %q, want %q", - tt.fileType, tt.filename, tt.fallback, got, tt.want) - } - }) - } -} diff --git a/internal/service/document/document.go b/internal/service/document/document.go index be266f7193..3803a20c04 100644 --- a/internal/service/document/document.go +++ b/internal/service/document/document.go @@ -63,7 +63,6 @@ func NewDocumentService() *DocumentService { } } -// CreateDocumentRequest create document request // UpdateDocumentRequest update document request type UpdateDocumentRequest struct { Name *string `json:"name"` diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go index fa1813c4b8..2a7e35e99b 100644 --- a/internal/service/document/document_crud.go +++ b/internal/service/document/document_crud.go @@ -100,7 +100,6 @@ func (s *DocumentService) DownloadDocument(datasetID, docID string) (*DownloadDo }, nil } -// CreateDocument create document // GetDocumentByID get document by ID func (s *DocumentService) GetDocumentByID(id string) (*DocumentResponse, error) { document, err := s.documentDAO.GetByID(id) diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go index 44fae39faa..b636811f30 100644 --- a/internal/service/document/document_test.go +++ b/internal/service/document/document_test.go @@ -40,7 +40,6 @@ import ( "ragflow/internal/service" "ragflow/internal/service/file" "ragflow/internal/storage" - "ragflow/internal/utility" ) // recordingTaskPublisher implements service.TaskPublisher and records published messages. @@ -711,29 +710,6 @@ func TestDeleteDocumentFull_SharedFilePreserved(t *testing.T) { } } -func TestSelectUploadParser_MirrorsPython(t *testing.T) { - tests := []struct { - name string - docType utility.FileType - filename string - defaultValue string - want string - }{ - {name: "visual", docType: utility.FileTypeVISUAL, filename: "img.png", defaultValue: "naive", want: "picture"}, - {name: "aural", docType: utility.FileTypeAURAL, filename: "audio.mp3", defaultValue: "naive", want: "audio"}, - {name: "presentation by ext", docType: utility.FileTypeDOC, filename: "deck.pptx", defaultValue: "naive", want: "presentation"}, - {name: "email by ext", docType: utility.FileTypeDOC, filename: "mail.eml", defaultValue: "naive", want: "email"}, - {name: "fallback default", docType: utility.FileTypeDOC, filename: "notes.txt", defaultValue: "manual", want: "manual"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := selectUploadParser(tt.docType, tt.filename, tt.defaultValue); got != tt.want { - t.Fatalf("selectUploadParser(%q)=%q, want %q", tt.filename, got, tt.want) - } - }) - } -} - func TestContentHashHex_MatchesPythonXXH128(t *testing.T) { tests := []struct { data []byte diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.go index a653cc9b88..a374868e6c 100644 --- a/internal/service/document/document_upload.go +++ b/internal/service/document/document_upload.go @@ -6,15 +6,12 @@ import ( "mime/multipart" "net/http" "path/filepath" - "ragflow/internal/service" "strings" "ragflow/internal/common" "ragflow/internal/entity" "ragflow/internal/storage" "ragflow/internal/utility" - - "go.uber.org/zap" ) // UploadLocalDocuments stores each uploaded file in object storage and inserts a @@ -313,7 +310,7 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, if i := strings.LastIndex(filename, "."); i >= 0 { suffix = filename[i+1:] } - parserID := selectUploadParser(utility.FileType(filetype), filename, kb.ParserID) + parserID := kb.ParserID if kb.PipelineID != nil { parserID = "" // canvas pipeline mode — parser_id not applicable } @@ -338,19 +335,6 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, hash := contentHashHex(blob) doc.ContentHash = &hash } - - // When the document's builtin parser_id differs from the KB's (e.g. visual→picture, - // aural→audio), re-resolve component_params defaults from the document's own DSL - // template so cpnIDs and param keys match the pipeline that will actually execute. - if kb.PipelineID == nil && parserID != kb.ParserID { - if cp, err := service.ResolveComponentParamsDefaults(parserID, nil); err != nil { - common.Warn("newDatasetDocument: resolve component_params defaults", - zap.String("parserID", parserID), zap.Error(err)) - } else if cp != nil { - doc.ParserConfig = cp - } - } - return doc } diff --git a/internal/service/document/document_upload_helpers.go b/internal/service/document/document_upload_helpers.go index 96c4fcf853..35318f57ad 100644 --- a/internal/service/document/document_upload_helpers.go +++ b/internal/service/document/document_upload_helpers.go @@ -3,22 +3,9 @@ package document import ( "encoding/hex" - pipelinepkg "ragflow/internal/ingestion/pipeline" - "ragflow/internal/utility" - "github.com/zeebo/xxh3" ) -// selectUploadParser resolves the parser_id for a document by delegating to -// the builtin pipeline registry, which owns the file-type-to-parser mapping. -func selectUploadParser(docType utility.FileType, filename, defaultParser string) string { - registry, err := pipelinepkg.DefaultRegistry() - if err != nil || registry == nil { - return defaultParser - } - return registry.DefaultParserID(string(docType), filename, defaultParser) -} - // contentHashHex mirrors Python xxhash.xxh128(blob).hexdigest(). func contentHashHex(blob []byte) string { sum := xxh3.Hash128(blob).Bytes() diff --git a/internal/service/document/file2document.go b/internal/service/document/file2document.go index a4fe3116fb..dbdb9c3c51 100644 --- a/internal/service/document/file2document.go +++ b/internal/service/document/file2document.go @@ -198,7 +198,7 @@ func (s *File2DocumentService) convertFiles(fileIDs, kbIDs []string, userID stri continue } - parserID := selectUploadParser(utility.FileType(file.Type), file.Name, kb.ParserID) + parserID := kb.ParserID suffix := strings.TrimPrefix(filepath.Ext(file.Name), ".") doc := &entity.Document{ ID: utility.GenerateUUID(), @@ -219,19 +219,6 @@ func (s *File2DocumentService) convertFiles(fileIDs, kbIDs []string, userID stri doc.ParserID = "" // canvas pipeline mode — parser_id not applicable } - // When the document's builtin parser_id differs from the KB's - // (e.g. visual→picture, aural→audio), re-resolve component_params - // defaults from the document's own DSL template so cpnIDs and - // param keys match the pipeline that will actually execute. - if kb.PipelineID == nil && parserID != kb.ParserID { - if cp, err := service.ResolveComponentParamsDefaults(parserID, nil); err != nil { - common.Warn("convertFiles: resolve component_params defaults", - zap.String("parserID", parserID), zap.Error(err)) - } else if cp != nil { - doc.ParserConfig = cp - } - } - // InsertDocument creates the row and increments KB doc_num in one // transaction, so a failed insert never leaves a stale counter. if err := s.documentSvc.InsertDocument(doc); err != nil {