From 1fff48b656c8f197facd94b381441e6ba80f425c Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 27 Mar 2026 18:12:56 +0800 Subject: [PATCH] Add minio go test (#13800) ### What problem does this PR solve? 1. Add go test 2. Update CI process ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Signed-off-by: Jin Hai --- internal/admin/service.go | 12 +- internal/cli/cli.go | 20 +- internal/cli/user_parser.go | 4 +- internal/server/config.go | 25 +- internal/storage/minio.go | 78 ++-- internal/storage/minio_test.go | 658 +++++++++++++++++++++++++++++++++ internal/storage/oss.go | 6 +- internal/storage/s3.go | 6 +- internal/storage/types.go | 4 +- run_go_tests.sh | 31 ++ 10 files changed, 775 insertions(+), 69 deletions(-) create mode 100644 internal/storage/minio_test.go create mode 100755 run_go_tests.sh diff --git a/internal/admin/service.go b/internal/admin/service.go index e8fb31129a..cc90b9ce73 100644 --- a/internal/admin/service.go +++ b/internal/admin/service.go @@ -1137,7 +1137,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "timeout", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "message": err.Error(), }, nil } @@ -1148,7 +1148,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "timeout", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "message": err.Error(), }, nil } @@ -1156,7 +1156,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "alive", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "message": "MySQL connection successful", }, nil } @@ -1170,7 +1170,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "timeout", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "error": "Redis client not initialized", }, nil } @@ -1180,7 +1180,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "timeout", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "error": "Redis health check failed", }, nil } @@ -1188,7 +1188,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) { return map[string]interface{}{ "service_name": name, "status": "alive", - "elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()), + "elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()), "message": "Redis connection successful", }, nil } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index e87e70f97a..40765fd2b4 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -59,9 +59,9 @@ type ConnectionArgs struct { Password string APIToken string UserName string - Command string // Original command string (for SQL mode) - CommandArgs []string // Split command arguments (for ContextEngine mode) - IsSQLMode bool // true=SQL mode (quoted), false=ContextEngine mode (unquoted) + Command string // Original command string (for SQL mode) + CommandArgs []string // Split command arguments (for ContextEngine mode) + IsSQLMode bool // true=SQL mode (quoted), false=ContextEngine mode (unquoted) ShowHelp bool AdminMode bool OutputFormat OutputFormat // Output format: table, plain, json @@ -384,8 +384,7 @@ Configuration File: Commands: SQL commands (use quotes): "LIST USERS", "CREATE USER 'email' 'password'", etc. Context Engine commands (no quotes): ls datasets, search "keyword", cat path, etc. - If no command is provided, CLI runs in interactive mode. -`) + If no command is provided, CLI runs in interactive mode.`) } // HistoryFile returns the path to the history file @@ -921,10 +920,10 @@ func (c *CLI) printContextEngineResult(result *contextengine.Result, cmdType con break } } - fmt.Println(sep) - fmt.Printf("Total: %d\n", result.Total) - } -case contextengine.CommandCat: + fmt.Println(sep) + fmt.Printf("Total: %d\n", result.Total) + } + case contextengine.CommandCat: // Cat output is handled differently - it returns []byte, not *Result // This case should not be reached in normal flow since Cat returns []byte directly fmt.Println("Content retrieved") @@ -1138,7 +1137,8 @@ type ListCommandOptions struct { // parseSearchCommandArgs parses search command arguments // Format: search [-d dir1] [-d dir2] ... -q query [-k top_k] [-t threshold] -// search -h|--help (shows help) +// +// search -h|--help (shows help) func parseSearchCommandArgs(args []string) (*SearchCommandOptions, error) { opts := &SearchCommandOptions{ TopK: 10, diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go index 1f7fd5a057..c4ba7da358 100644 --- a/internal/cli/user_parser.go +++ b/internal/cli/user_parser.go @@ -446,7 +446,7 @@ func (p *Parser) parseCreateCommand() (*Command, error) { case TokenToken: return p.parseCreateToken() case TokenIndex: - return p.parseCreateIndex() + return p.parseCreateIndex() default: return nil, fmt.Errorf("unknown CREATE target: %s", p.curToken.Value) } @@ -691,7 +691,7 @@ func (p *Parser) parseDropCommand() (*Command, error) { case TokenToken: return p.parseDropToken() case TokenIndex: - return p.parseDropIndex() + return p.parseDropIndex() default: return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value) } diff --git a/internal/server/config.go b/internal/server/config.go index 3aaed8119a..4ed498f65c 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -18,6 +18,7 @@ package server import ( "fmt" + "net" "net/mail" "net/url" "os" @@ -213,7 +214,7 @@ var ( // Init initialize configuration func Init(configPath string) error { - err := FromConfigFile("") + err := FromConfigFile(configPath) if err != nil { return err } @@ -444,6 +445,26 @@ func FromEnvironments() error { return fmt.Errorf("invalid storage type: %s", storageType) } + // Minio + minioIP := strings.ToLower(os.Getenv("MINIO_IP")) + if minioIP != "" { + _, port, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host) + if err != nil { + return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err) + } + globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", minioIP, port) + } + + minioPort := strings.ToLower(os.Getenv("MINIO_PORT")) + println(fmt.Sprintf("MINIO ip and port from env: %s:%s", minioIP, minioPort)) + if minioPort != "" { + ip, _, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host) + if err != nil { + return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err) + } + globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", ip, minioPort) + } + // Language if globalConfig.Language == "" { globalConfig.Language = GetLanguage() @@ -464,8 +485,6 @@ func FromConfigFile(configPath string) error { v.SetConfigType("yaml") v.AddConfigPath("./conf") v.AddConfigPath(".") - v.AddConfigPath("./config") - v.AddConfigPath("./internal/config") v.AddConfigPath("/etc/ragflow/") } diff --git a/internal/storage/minio.go b/internal/storage/minio.go index 496f56c9d0..72e6d5d0df 100644 --- a/internal/storage/minio.go +++ b/internal/storage/minio.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "fmt" "net/http" + "ragflow/internal/logger" "ragflow/internal/server" "time" @@ -33,8 +34,8 @@ import ( // MinioStorage implements Storage interface for MinIO type MinioStorage struct { client *minio.Client - bucket string - prefixPath string + bucket string // default bucket + prefixPath string // default prefix path config *server.MinioConfig } @@ -81,7 +82,7 @@ func (m *MinioStorage) connect() error { func (m *MinioStorage) reconnect() { if err := m.connect(); err != nil { - zap.L().Error("Failed to reconnect to MinIO", zap.Error(err)) + logger.Fatal(fmt.Sprintf("Failed to reconnect to MinIO, %s", err.Error())) } } @@ -107,23 +108,17 @@ func (m *MinioStorage) resolveBucketAndPath(bucket, fnm string) (string, string) // Health checks MinIO service availability func (m *MinioStorage) Health() bool { - ctx := context.Background() - - if m.bucket != "" { - exists, err := m.client.BucketExists(ctx, m.bucket) - if err != nil { - zap.L().Warn("MinIO health check failed", zap.Error(err)) - return false - } - return exists + cancelFunction, err := m.client.HealthCheck(time.Second * 5) + if cancelFunction != nil { + defer cancelFunction() } - _, err := m.client.ListBuckets(ctx) if err != nil { - zap.L().Warn("MinIO health check failed", zap.Error(err)) + logger.Warn("Failed to check MinIO health", zap.Error(err)) return false } - return true + + return m.client.IsOnline() } // Put uploads an object to MinIO @@ -132,19 +127,22 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string ctx := context.Background() + var err error + for i := 0; i < 3; i++ { + var exists bool // Ensure bucket exists if m.bucket == "" { - exists, err := m.client.BucketExists(ctx, bucket) + exists, err = m.client.BucketExists(ctx, bucket) if err != nil { - zap.L().Error("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err)) + logger.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue } if !exists { - if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { - zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) + if err = m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + logger.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -153,9 +151,9 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string } reader := bytes.NewReader(binary) - _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}) + _, err = m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}) if err != nil { - zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -164,7 +162,7 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string return nil } - return fmt.Errorf("failed to put object after 3 retries") + return err } // Get retrieves an object from MinIO @@ -176,7 +174,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro for i := 0; i < 2; i++ { obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{}) if err != nil { - zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -185,7 +183,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro buf := new(bytes.Buffer) if _, err := buf.ReadFrom(obj); err != nil { - zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -197,14 +195,14 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro return nil, fmt.Errorf("failed to get object after retries") } -// Rm removes an object from MinIO -func (m *MinioStorage) Rm(bucket, fnm string, tenantID ...string) error { +// Remove removes an object from MinIO +func (m *MinioStorage) Remove(bucket, fnm string, tenantID ...string) error { bucket, fnm = m.resolveBucketAndPath(bucket, fnm) ctx := context.Background() if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil { - zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) return err } @@ -228,7 +226,7 @@ func (m *MinioStorage) ObjExist(bucket, fnm string, tenantID ...string) bool { if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" { return false } - zap.L().Error("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) return false } @@ -244,7 +242,7 @@ func (m *MinioStorage) GetPresignedURL(bucket, fnm string, expires time.Duration for i := 0; i < 10; i++ { url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil) if err != nil { - zap.L().Error("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + logger.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -267,7 +265,7 @@ func (m *MinioStorage) BucketExists(bucket string) bool { exists, err := m.client.BucketExists(ctx, actualBucket) if err != nil { - zap.L().Error("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err)) + logger.Warn("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err)) return false } @@ -304,7 +302,7 @@ func (m *MinioStorage) RemoveBucket(bucket string) error { Recursive: true, }) { if obj.Err != nil { - zap.L().Error("Error listing objects", zap.Error(obj.Err)) + logger.Warn("Failed to list objects", zap.Error(obj.Err)) return } objectsCh <- obj @@ -312,13 +310,13 @@ func (m *MinioStorage) RemoveBucket(bucket string) error { }() for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) { - zap.L().Error("Failed to remove object", zap.String("key", err.ObjectName), zap.Error(err.Err)) + logger.Warn(fmt.Sprintf("Failed to remove object, key: %s", err.ObjectName), zap.Error(err.Err)) } // Only remove the actual bucket if not in single-bucket mode if m.bucket == "" { if err := m.client.RemoveBucket(ctx, actualBucket); err != nil { - zap.L().Error("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err)) + logger.Warn("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err)) return err } } @@ -337,12 +335,12 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo if m.bucket == "" { exists, err := m.client.BucketExists(ctx, destBucket) if err != nil { - zap.L().Error("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err)) + logger.Warn("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err)) return false } if !exists { - if err := m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil { - zap.L().Error("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err)) + if err = m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil { + logger.Warn("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err)) return false } } @@ -351,7 +349,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo // Check if source object exists _, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{}) if err != nil { - zap.L().Error("Source object not found", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + logger.Warn("Failed to stat source object", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } @@ -367,7 +365,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo _, err = m.client.CopyObject(ctx, destOpts, srcOpts) if err != nil { - zap.L().Error("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) + logger.Warn("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) return false } @@ -377,8 +375,8 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo // Move moves an object from source to destination func (m *MinioStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool { if m.Copy(srcBucket, srcPath, destBucket, destPath) { - if err := m.Rm(srcBucket, srcPath); err != nil { - zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + if err := m.Remove(srcBucket, srcPath); err != nil { + logger.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } return true diff --git a/internal/storage/minio_test.go b/internal/storage/minio_test.go new file mode 100644 index 0000000000..1ce4248658 --- /dev/null +++ b/internal/storage/minio_test.go @@ -0,0 +1,658 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "bytes" + "fmt" + "log" + "os" + "ragflow/internal/utility" + "testing" + "time" + + "ragflow/internal/server" +) + +// getMinioConfig returns MinIO configuration for testing +// Configuration can be loaded from environment variables or config file +func getMinioConfig() (*server.MinioConfig, error) { + + // Initialize configuration + if err := server.Init(""); err != nil { + return nil, err + } + + // Try to get configuration from environment variables first + config := server.GetConfig().StorageEngine.Minio + + log.Printf("MinioConfig: %+v", config) + return config, nil +} + +// getEnv gets environment variable or returns default value +func getEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// getEnvBool gets environment variable as bool or returns default value +func getEnvBool(key string, defaultValue bool) bool { + if value := os.Getenv(key); value != "" { + return value == "true" || value == "1" || value == "yes" + } + return defaultValue +} + +// newTestMinioStorage creates a new MinIO storage instance for testing +func newTestMinioStorage(t *testing.T) *MinioStorage { + rootDir := utility.GetProjectRoot() + t.Chdir(rootDir) + t.Chdir(rootDir) + + config, err := getMinioConfig() + + if err != nil { + t.Skipf("Skipping test: failed to get MinIO configuration: %v", err) + return nil + } + storage, err := NewMinioStorage(config) + if err != nil { + t.Skipf("Skipping test: failed to connect to MinIO: %v", err) + } + return storage +} + +func TestNewMinioStorage(t *testing.T) { + rootDir := utility.GetProjectRoot() + t.Chdir(rootDir) + + config, err := getMinioConfig() + if err != nil { + t.Skipf("Skipping test: failed to get MinIO configuration: %v", err) + return + } + + storage, err := NewMinioStorage(config) + if err != nil { + t.Skipf("Skipping test: failed to connect to MinIO: %v", err) + } + + if storage == nil { + t.Error("Expected storage to be non-nil") + } + + if storage.client == nil { + t.Error("Expected client to be non-nil") + } + + if storage.config == nil { + t.Error("Expected config to be non-nil") + } +} + +func TestNewMinioStorage_InvalidConfig(t *testing.T) { + // Test with invalid host + config := &server.MinioConfig{ + Host: "invalid-host:99999", + User: "test", + Password: "test", + Secure: false, + } + + _, err := NewMinioStorage(config) + // Should return an error for invalid connection + if err == nil { + t.Log("Note: Connection may succeed but fail later depending on network timeout") + } +} + +func TestMinioStorage_Health(t *testing.T) { + storage := newTestMinioStorage(t) + + healthy := storage.Health() + // Health check should return true if connection is working + // Note: This depends on whether a default bucket is configured + t.Logf("Health check result: %v", healthy) + if !healthy { + t.Error("Expected storage to be healthy") + } +} + +func TestMinioStorage_PutAndGet(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "test-file.txt" + content := []byte("Hello, MinIO Test!") + + // Test Put + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + // Test Get + retrieved, err := storage.Get(bucket, key) + if err != nil { + t.Fatalf("Failed to get object: %v", err) + } + + if !bytes.Equal(retrieved, content) { + t.Errorf("Retrieved content does not match. Expected %s, got %s", content, retrieved) + } + + // Cleanup + err = storage.Remove(bucket, key) + if err != nil { + t.Logf("Warning: failed to cleanup test object: %v", err) + } +} + +func TestMinioStorage_Put_EmptyData(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "empty-file.txt" + content := []byte{} + + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put empty object: %v", err) + } + + // Verify object exists + exists := storage.ObjExist(bucket, key) + if !exists { + t.Error("Expected empty object to exist") + } + + // Cleanup + storage.Remove(bucket, key) +} + +func TestMinioStorage_Put_LargeData(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "large-file.bin" + // Create 1MB of data + content := make([]byte, 1024*1024) + for i := range content { + content[i] = byte(i % 256) + } + + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put large object: %v", err) + } + + retrieved, err := storage.Get(bucket, key) + if err != nil { + t.Fatalf("Failed to get large object: %v", err) + } + + if !bytes.Equal(retrieved, content) { + t.Error("Retrieved large content does not match original") + } + + // Cleanup + storage.Remove(bucket, key) +} + +func TestMinioStorage_Get_NonExistent(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "non-existent-file.txt" + + _, err := storage.Get(bucket, key) + if err == nil { + t.Error("Expected error when getting non-existent object") + } +} + +func TestMinioStorage_Remove(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "file-to-delete.txt" + content := []byte("Delete me") + + // First, put an object + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + // Verify it exists + exists := storage.ObjExist(bucket, key) + if !exists { + t.Fatal("Expected object to exist before removal") + } + + // Remove it + err = storage.Remove(bucket, key) + if err != nil { + t.Fatalf("Failed to remove object: %v", err) + } + + // Verify it's gone + exists = storage.ObjExist(bucket, key) + if exists { + t.Error("Expected object to not exist after removal") + } +} + +func TestMinioStorage_Remove_NonExistent(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "non-existent-file.txt" + + // Removing a non-existent object should not error + err := storage.Remove(bucket, key) + if err != nil { + t.Logf("Remove non-existent object returned error (may be acceptable): %v", err) + } +} + +func TestMinioStorage_ObjExist(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "existence-test.txt" + content := []byte("Test content") + + // Check non-existent object + exists := storage.ObjExist(bucket, key) + if exists { + t.Error("Expected non-existent object to return false") + } + + // Create object + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + // Check existing object + exists = storage.ObjExist(bucket, key) + if !exists { + t.Error("Expected existing object to return true") + } + + // Cleanup + storage.Remove(bucket, key) +} + +func TestMinioStorage_GetPresignedURL(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "presigned-test.txt" + content := []byte("Presigned URL test content") + + // Create object first + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + // Get presigned URL + url, err := storage.GetPresignedURL(bucket, key, 5*time.Minute) + if err != nil { + t.Fatalf("Failed to get presigned URL: %v", err) + } + + if url == "" { + t.Error("Expected presigned URL to be non-empty") + } + + // Verify URL contains expected components + if len(url) > 0 { + t.Logf("Generated presigned URL (first 100 chars): %s...", url[:min(100, len(url))]) + } + + // Cleanup + storage.Remove(bucket, key) +} + +func TestMinioStorage_GetPresignedURL_NonExistent(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "non-existent-presigned.txt" + + _, err := storage.GetPresignedURL(bucket, key, 5*time.Minute) + if err == nil { + t.Log("Note: Some MinIO versions may allow presigned URLs for non-existent objects") + } +} + +func TestMinioStorage_BucketExists(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := fmt.Sprintf("test-bucket-exists-%d", time.Now().Unix()) + + // Check non-existent bucket + exists := storage.BucketExists(bucket) + if exists { + t.Error("Expected non-existent bucket to return false") + } + + // Create bucket by putting an object + err := storage.Put(bucket, "test.txt", []byte("test")) + if err != nil { + t.Fatalf("Failed to create bucket: %v", err) + } + + // Check existing bucket + exists = storage.BucketExists(bucket) + if !exists { + t.Error("Expected existing bucket to return true") + } + + // Cleanup + storage.RemoveBucket(bucket) +} + +func TestMinioStorage_RemoveBucket(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := fmt.Sprintf("test-bucket-remove-%d", time.Now().Unix()) + + // Create bucket with some objects + err := storage.Put(bucket, "file1.txt", []byte("content1")) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + err = storage.Put(bucket, "file2.txt", []byte("content2")) + if err != nil { + t.Fatalf("Failed to put object: %v", err) + } + + // Verify bucket exists + exists := storage.BucketExists(bucket) + if !exists { + t.Fatal("Expected bucket to exist before removal") + } + + // Remove bucket + err = storage.RemoveBucket(bucket) + if err != nil { + t.Fatalf("Failed to remove bucket: %v", err) + } + + // Verify bucket is gone + exists = storage.BucketExists(bucket) + if exists { + t.Error("Expected bucket to not exist after removal") + } +} + +func TestMinioStorage_Copy(t *testing.T) { + storage := newTestMinioStorage(t) + + srcBucket := "test-bucket-src" + srcKey := "source-file.txt" + destBucket := "test-bucket-dest" + destKey := "copied-file.txt" + content := []byte("Content to copy") + + // Create source object + err := storage.Put(srcBucket, srcKey, content) + if err != nil { + t.Fatalf("Failed to put source object: %v", err) + } + + // Copy object + success := storage.Copy(srcBucket, srcKey, destBucket, destKey) + if !success { + t.Fatal("Failed to copy object") + } + + // Verify destination exists + exists := storage.ObjExist(destBucket, destKey) + if !exists { + t.Error("Expected copied object to exist") + } + + // Verify content matches + retrieved, err := storage.Get(destBucket, destKey) + if err != nil { + t.Fatalf("Failed to get copied object: %v", err) + } + + if !bytes.Equal(retrieved, content) { + t.Error("Copied content does not match original") + } + + // Cleanup + storage.Remove(srcBucket, srcKey) + storage.Remove(destBucket, destKey) +} + +func TestMinioStorage_Copy_NonExistentSource(t *testing.T) { + storage := newTestMinioStorage(t) + + srcBucket := "test-bucket-src" + srcKey := "non-existent-source.txt" + destBucket := "test-bucket-dest" + destKey := "should-not-exist.txt" + + success := storage.Copy(srcBucket, srcKey, destBucket, destKey) + if success { + t.Error("Expected copy of non-existent object to fail") + } + + // Verify destination does not exist + exists := storage.ObjExist(destBucket, destKey) + if exists { + t.Error("Expected destination object to not exist after failed copy") + storage.Remove(destBucket, destKey) + } +} + +func TestMinioStorage_Move(t *testing.T) { + storage := newTestMinioStorage(t) + + srcBucket := "test-bucket-src" + srcKey := "file-to-move.txt" + destBucket := "test-bucket-dest" + destKey := "moved-file.txt" + content := []byte("Content to move") + + // Create source object + err := storage.Put(srcBucket, srcKey, content) + if err != nil { + t.Fatalf("Failed to put source object: %v", err) + } + + // Move object + success := storage.Move(srcBucket, srcKey, destBucket, destKey) + if !success { + t.Fatal("Failed to move object") + } + + // Verify source is gone + exists := storage.ObjExist(srcBucket, srcKey) + if exists { + t.Error("Expected source object to not exist after move") + } + + // Verify destination exists + exists = storage.ObjExist(destBucket, destKey) + if !exists { + t.Error("Expected moved object to exist") + } + + // Verify content matches + retrieved, err := storage.Get(destBucket, destKey) + if err != nil { + t.Fatalf("Failed to get moved object: %v", err) + } + + if !bytes.Equal(retrieved, content) { + t.Error("Moved content does not match original") + } + + // Cleanup + storage.Remove(destBucket, destKey) +} + +func TestMinioStorage_Move_NonExistentSource(t *testing.T) { + storage := newTestMinioStorage(t) + + srcBucket := "test-bucket-src" + srcKey := "non-existent-source.txt" + destBucket := "test-bucket-dest" + destKey := "should-not-exist.txt" + + success := storage.Move(srcBucket, srcKey, destBucket, destKey) + if success { + t.Error("Expected move of non-existent object to fail") + } +} + +func TestMinioStorage_MultipleObjectsInBucket(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := fmt.Sprintf("test-multi-%d", time.Now().Unix()) + numObjects := 10 + + // Create multiple objects + for i := 0; i < numObjects; i++ { + key := fmt.Sprintf("file-%d.txt", i) + content := []byte(fmt.Sprintf("Content %d", i)) + err := storage.Put(bucket, key, content) + if err != nil { + t.Fatalf("Failed to put object %d: %v", i, err) + } + } + + // Verify all objects exist + for i := 0; i < numObjects; i++ { + key := fmt.Sprintf("file-%d.txt", i) + exists := storage.ObjExist(bucket, key) + if !exists { + t.Errorf("Expected object %s to exist", key) + } + } + + // Verify content + for i := 0; i < numObjects; i++ { + key := fmt.Sprintf("file-%d.txt", i) + expectedContent := []byte(fmt.Sprintf("Content %d", i)) + retrieved, err := storage.Get(bucket, key) + if err != nil { + t.Errorf("Failed to get object %s: %v", key, err) + continue + } + if !bytes.Equal(retrieved, expectedContent) { + t.Errorf("Content mismatch for object %s", key) + } + } + + // Cleanup - remove bucket with all objects + err := storage.RemoveBucket(bucket) + if err != nil { + t.Logf("Warning: failed to cleanup bucket: %v", err) + } +} + +func TestMinioStorage_SpecialCharactersInKey(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + specialKeys := []string{ + "file with spaces.txt", + "file-with-dashes.txt", + "file_with_underscores.txt", + "file.multiple.dots.txt", + "path/to/nested/file.txt", + "unicode-文件.txt", + } + + for _, key := range specialKeys { + content := []byte(fmt.Sprintf("Content for %s", key)) + + err := storage.Put(bucket, key, content) + if err != nil { + t.Errorf("Failed to put object with key '%s': %v", key, err) + continue + } + + retrieved, err := storage.Get(bucket, key) + if err != nil { + t.Errorf("Failed to get object with key '%s': %v", key, err) + continue + } + + if !bytes.Equal(retrieved, content) { + t.Errorf("Content mismatch for key '%s'", key) + } + + // Cleanup + storage.Remove(bucket, key) + } +} + +func TestMinioStorage_TenantID(t *testing.T) { + storage := newTestMinioStorage(t) + + bucket := "test-bucket" + key := "tenant-test.txt" + content := []byte("Tenant test content") + tenantID := "tenant-123" + + // Put with tenant ID + err := storage.Put(bucket, key, content, tenantID) + if err != nil { + t.Fatalf("Failed to put object with tenant ID: %v", err) + } + + // Get with tenant ID + retrieved, err := storage.Get(bucket, key, tenantID) + if err != nil { + t.Fatalf("Failed to get object with tenant ID: %v", err) + } + + if !bytes.Equal(retrieved, content) { + t.Error("Content mismatch for tenant-specific object") + } + + // Check existence with tenant ID + exists := storage.ObjExist(bucket, key, tenantID) + if !exists { + t.Error("Expected object to exist with tenant ID") + } + + // Cleanup + storage.Remove(bucket, key, tenantID) +} + +// min is a helper function to get the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/storage/oss.go b/internal/storage/oss.go index 11146cb1ef..8c3c52bb5a 100644 --- a/internal/storage/oss.go +++ b/internal/storage/oss.go @@ -218,8 +218,8 @@ func (o *OSSStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) return nil, fmt.Errorf("failed to get object after retries") } -// Rm removes an object from OSS -func (o *OSSStorage) Rm(bucket, fnm string, tenantID ...string) error { +// Remove removes an object from OSS +func (o *OSSStorage) Remove(bucket, fnm string, tenantID ...string) error { bucket, fnm = o.resolveBucketAndPath(bucket, fnm) ctx := context.Background() @@ -381,7 +381,7 @@ func (o *OSSStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool // Move moves an object from source to destination func (o *OSSStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool { if o.Copy(srcBucket, srcPath, destBucket, destPath) { - if err := o.Rm(srcBucket, srcPath); err != nil { + if err := o.Remove(srcBucket, srcPath); err != nil { zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 5c3addfc15..45b2347263 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -226,8 +226,8 @@ func (s *S3Storage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) return nil, fmt.Errorf("failed to get object after retries") } -// Rm removes an object from S3 -func (s *S3Storage) Rm(bucket, fnm string, tenantID ...string) error { +// Remove removes an object from S3 +func (s *S3Storage) Remove(bucket, fnm string, tenantID ...string) error { bucket, fnm = s.resolveBucketAndPath(bucket, fnm) ctx := context.Background() @@ -389,7 +389,7 @@ func (s *S3Storage) Copy(srcBucket, srcPath, destBucket, destPath string) bool { // Move moves an object from source to destination func (s *S3Storage) Move(srcBucket, srcPath, destBucket, destPath string) bool { if s.Copy(srcBucket, srcPath, destBucket, destPath) { - if err := s.Rm(srcBucket, srcPath); err != nil { + if err := s.Remove(srcBucket, srcPath); err != nil { zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } diff --git a/internal/storage/types.go b/internal/storage/types.go index fc777373af..0d15ba5556 100644 --- a/internal/storage/types.go +++ b/internal/storage/types.go @@ -78,8 +78,8 @@ type Storage interface { // Returns the data or nil if not found Get(bucket, fnm string, tenantID ...string) ([]byte, error) - // Rm removes an object from storage - Rm(bucket, fnm string, tenantID ...string) error + // Remove removes an object from storage + Remove(bucket, fnm string, tenantID ...string) error // ObjExist checks if an object exists ObjExist(bucket, fnm string, tenantID ...string) bool diff --git a/run_go_tests.sh b/run_go_tests.sh new file mode 100755 index 0000000000..f633d5fbfd --- /dev/null +++ b/run_go_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +PACKAGES=( + "./internal/admin/..." +# "./internal/binding/..." + "./internal/cache/..." + "./internal/cli/..." + "./internal/common/..." + "./internal/dao/..." + "./internal/engine/..." + "./internal/handler/..." + "./internal/logger/..." + "./internal/model/..." + "./internal/router/..." + "./internal/server/..." +# "./internal/service/..." + "./internal/storage/..." + "./internal/tokenizer/..." +# "./internal/utility/..." +) + +echo "Running tests for specific packages..." +for pkg in "${PACKAGES[@]}"; do + echo "=== Testing $pkg ===" + go test $pkg -v -cover -test.v + echo "" +done + +#echo "Running all tests except failed packages..." +#go test $(go list ./internal/... | grep -v -E '(cli|service|binding)$') -v \ No newline at end of file