Go: unify three services into one binary (#16462)

### Summary

Plan to start api_server, admin_server and ingestor in one binary:
- ./ragflow_main --admin
- ./ragflow_main --api
- ./ragflow_main --ingestor

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

internal/engine/elasticsearch/client.go

@@ -24,7 +24,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"path/filepath"
 	"ragflow/internal/server"
 	"ragflow/internal/utility"
 	"time"
@@ -88,11 +87,11 @@ func NewEngine(cfg interface{}) (*elasticsearchEngine, error) {
 
 	// Create two index templates for different index types
 	// Template for chunk indices (ragflow_*) - priority 1
-	if err := engine.CreateIndexTemplate(context.Background(), "ragflow_mapping", "ragflow_*", "mapping.json", 1); err != nil {
+	if err = engine.CreateIndexTemplate(context.Background(), "ragflow_mapping", "ragflow_*", "mapping.json", 1); err != nil {
 		return nil, fmt.Errorf("failed to create chunk index template: %w", err)
 	}
 	// Template for doc_meta indices (ragflow_doc_meta_*) - priority 2 (higher than ragflow_*)
-	if err := engine.CreateIndexTemplate(context.Background(), "ragflow_doc_meta_mapping", "ragflow_doc_meta_*", "doc_meta_es_mapping.json", 2); err != nil {
+	if err = engine.CreateIndexTemplate(context.Background(), "ragflow_doc_meta_mapping", "ragflow_doc_meta_*", "doc_meta_es_mapping.json", 2); err != nil {
 		return nil, fmt.Errorf("failed to create doc_meta index template: %w", err)
 	}
 
@@ -140,16 +139,20 @@ func (e *elasticsearchEngine) CreateIndexTemplate(ctx context.Context, templateN
 		mappingFileName = "mapping.json"
 	}
 
-	// Read mapping from file
-	mappingPath := filepath.Join(utility.GetProjectRoot(), "conf", mappingFileName)
-	data, err := os.ReadFile(mappingPath)
+	mappingPath, err := utility.FindConfFileInProject(mappingFileName)
 	if err != nil {
-		return fmt.Errorf("failed to read mapping file: %w", err)
+		return err
+	}
+
+	// Read mapping from file
+	data, err := os.ReadFile(*mappingPath)
+	if err != nil {
+		return fmt.Errorf("failed to read mapping file %q: %w", *mappingPath, err)
 	}
 
 	var mapping map[string]interface{}
-	if err := json.Unmarshal(data, &mapping); err != nil {
-		return fmt.Errorf("failed to parse mapping file: %w", err)
+	if err = json.Unmarshal(data, &mapping); err != nil {
+		return fmt.Errorf("failed to parse mapping file %q: %w", *mappingPath, err)
 	}
 
 	// Separate settings and mappings from the mapping file
@@ -189,7 +192,7 @@ func (e *elasticsearchEngine) CreateIndexTemplate(ctx context.Context, templateN
 	}
 
 	var result map[string]interface{}
-	if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+	if err = json.NewDecoder(res.Body).Decode(&result); err != nil {
 		return fmt.Errorf("failed to parse response: %w", err)
 	}
 
@@ -215,7 +218,7 @@ func (e *elasticsearchEngine) GetClusterStats() (map[string]interface{}, error)
 	}
 
 	var rawStats map[string]interface{}
-	if err := json.NewDecoder(res.Body).Decode(&rawStats); err != nil {
+	if err = json.NewDecoder(res.Body).Decode(&rawStats); err != nil {
 		return nil, fmt.Errorf("failed to decode cluster stats: %w", err)
 	}
 
@@ -278,8 +281,8 @@ func (e *elasticsearchEngine) GetClusterStats() (map[string]interface{}, error)
 		if versions, ok := nodes["versions"].([]interface{}); ok {
 			result["nodes_version"] = versions
 		}
-		if os, ok := nodes["os"].(map[string]interface{}); ok {
-			if mem, ok := os["mem"].(map[string]interface{}); ok {
+		if operatingSystem, ok := nodes["os"].(map[string]interface{}); ok {
+			if mem, ok := operatingSystem["mem"].(map[string]interface{}); ok {
 				if totalInBytes, ok := mem["total_in_bytes"].(float64); ok {
 					result["os_mem"] = convertBytes(int64(totalInBytes))
 				}

internal/engine/global.go

@@ -26,8 +26,9 @@ import (
 	"ragflow/internal/engine/elasticsearch"
 	"ragflow/internal/engine/infinity"
 
-	"go.uber.org/zap"
 	"ragflow/internal/tokenizer"
+
+	"go.uber.org/zap"
 )
 
 var (
@@ -96,6 +97,8 @@ func InitMessageQueueEngine(messageQueueType string) error {
 		if err != nil {
 			return err
 		}
+	case "":
+		return fmt.Errorf("message queue type is empty")
 	default:
 		return fmt.Errorf("unsupported message queue type: %s", messageQueueType)
 	}

internal/engine/infinity/chunk.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"hash/fnv"
 	"os"
-	"path/filepath"
 	"ragflow/internal/common"
 	"ragflow/internal/engine/types"
 	"ragflow/internal/utility"
@@ -67,23 +66,26 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 		common.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
 	}
 
-	// Use configured schema
-	fpMapping := filepath.Join(utility.GetProjectRoot(), "conf", mappingFile)
-
-	schemaData, err := os.ReadFile(fpMapping)
+	fpMapping, err := utility.FindConfFileInProject(mappingFile)
 	if err != nil {
-		return fmt.Errorf("Failed to read mapping file: %w", err)
+		return err
+	}
+
+	// Use configured schema
+	schemaData, err := os.ReadFile(*fpMapping)
+	if err != nil {
+		return fmt.Errorf("failed to read mapping file %s: %w", *fpMapping, err)
 	}
 
 	var schema orderedFields
 	if err := json.Unmarshal(schemaData, &schema); err != nil {
-		return fmt.Errorf("Failed to parse mapping file: %w", err)
+		return fmt.Errorf("failed to parse mapping file: %w", err)
 	}
 
 	// Get database
 	db, err := e.client.conn.GetDatabase(e.client.dbName)
 	if err != nil {
-		return fmt.Errorf("Failed to get database: %w", err)
+		return fmt.Errorf("failed to get database: %w", err)
 	}
 
 	// Determine vector column name
@@ -92,7 +94,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 	// Check if table already exists
 	exists, err := e.tableExists(ctx, tableName)
 	if err != nil {
-		return fmt.Errorf("Failed to check if table exists: %w", err)
+		return fmt.Errorf("failed to check if table exists: %w", err)
 	}
 
 	var table *infinity.Table
@@ -101,7 +103,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 		common.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
 		table, err = db.GetTable(tableName)
 		if err != nil {
-			return fmt.Errorf("Failed to open existing table %s: %w", tableName, err)
+			return fmt.Errorf("failed to open existing table %s: %w", tableName, err)
 		}
 
 		// Check if vector column exists (for embedding model changes)
@@ -121,7 +123,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 			}
 			if _, err := table.AddColumns(addColSchema); err != nil {
 				common.Error("Failed to add vector column "+vectorColName, err)
-				return fmt.Errorf("Failed to add vector column %s: %w", vectorColName, err)
+				return fmt.Errorf("failed to add vector column %s: %w", vectorColName, err)
 			}
 			common.Info("Successfully added vector column", zap.String("column", vectorColName))
 		}
@@ -159,7 +161,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 		// Create table
 		table, err = db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
 		if err != nil {
-			return fmt.Errorf("Failed to create table: %w", err)
+			return fmt.Errorf("failed to create table: %w", err)
 		}
 		common.Debug("Infinity created table", zap.String("tableName", tableName))
 	}
@@ -179,7 +181,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 		"",
 	)
 	if err != nil {
-		return fmt.Errorf("Failed to create HNSW index %s: %w", vectorIndexName, err)
+		return fmt.Errorf("failed to create HNSW index %s: %w", vectorIndexName, err)
 	}
 	common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
 
@@ -214,7 +216,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 				"",
 			)
 			if err != nil {
-				return fmt.Errorf("Failed to create fulltext index %s: %w", indexNameFt, err)
+				return fmt.Errorf("failed to create fulltext index %s: %w", indexNameFt, err)
 			}
 		}
 	}
@@ -250,7 +252,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 				"",
 			)
 			if err != nil {
-				return fmt.Errorf("Failed to create secondary index %s: %w", indexNameSec, err)
+				return fmt.Errorf("failed to create secondary index %s: %w", indexNameSec, err)
 			}
 		}
 	}
@@ -258,7 +260,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset
 	return nil
 }
 
-// InsertChunks inserts documents into a dataset table
+// InsertChunks inserts documents into a dataset table;
 // Table name format: {baseName}_{datasetID}
 // Auto-create the table if it doesn't exist
 // Delete existing rows with matching IDs before insert

internal/engine/infinity/metadata.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"ragflow/internal/common"
@@ -42,29 +41,32 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin
 	// Get database
 	db, err := e.client.conn.GetDatabase(e.client.dbName)
 	if err != nil {
-		return fmt.Errorf("Failed to get database: %w", err)
+		return fmt.Errorf("failed to get database: %w", err)
 	}
 
 	// Check if table already exists
 	exists, err := e.tableExists(ctx, tableName)
 	if err != nil {
-		return fmt.Errorf("Failed to check if table exists: %w", err)
+		return fmt.Errorf("failed to check if table exists: %w", err)
 	}
 	if exists {
 		return fmt.Errorf("metadata table '%s' already exists", tableName)
 	}
 
-	// Use configured doc_meta mapping file
-	fpMapping := filepath.Join(utility.GetProjectRoot(), "conf", e.docMetaMappingFileName)
-
-	schemaData, err := os.ReadFile(fpMapping)
+	fpMapping, err := utility.FindConfFileInProject(e.docMetaMappingFileName)
 	if err != nil {
-		return fmt.Errorf("Failed to read mapping file: %w", err)
+		return err
+	}
+
+	// Use configured doc_meta mapping file
+	schemaData, err := os.ReadFile(*fpMapping)
+	if err != nil {
+		return fmt.Errorf("failed to read mapping file %q: %w", *fpMapping, err)
 	}
 
 	var schema map[string]fieldInfo
-	if err := json.Unmarshal(schemaData, &schema); err != nil {
-		return fmt.Errorf("Failed to parse mapping file: %w", err)
+	if err = json.Unmarshal(schemaData, &schema); err != nil {
+		return fmt.Errorf("failed to parse mapping file %q: %w", *fpMapping, err)
 	}
 
 	// Build column definitions
@@ -81,14 +83,14 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin
 	// Create table
 	_, err = db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
 	if err != nil {
-		return fmt.Errorf("Failed to create doc meta table: %w", err)
+		return fmt.Errorf("failed to create doc meta table: %w", err)
 	}
 	common.Debug("Infinity created doc meta table", zap.String("tableName", tableName))
 
 	// Get table for creating indexes
 	table, err := db.GetTable(tableName)
 	if err != nil {
-		return fmt.Errorf("Failed to get table: %w", err)
+		return fmt.Errorf("failed to get table: %w", err)
 	}
 
 	// Create secondary index on id
@@ -99,7 +101,7 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin
 		"",
 	)
 	if err != nil {
-		return fmt.Errorf("Failed to create secondary index on id: %w", err)
+		return fmt.Errorf("failed to create secondary index on id: %w", err)
 	}
 
 	// Create secondary index on kb_id
@@ -110,7 +112,7 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin
 		"",
 	)
 	if err != nil {
-		return fmt.Errorf("Failed to create secondary index on kb_id: %w", err)
+		return fmt.Errorf("failed to create secondary index on kb_id: %w", err)
 	}
 
 	// Create secondary index on meta_fields for metadata filter queries
@@ -121,7 +123,7 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin
 		"",
 	)
 	if err != nil {
-		return fmt.Errorf("Failed to create secondary index on meta_fields: %w", err)
+		return fmt.Errorf("failed to create secondary index on meta_fields: %w", err)
 	}
 
 	return nil
@@ -136,7 +138,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
 
 	db, err := e.client.conn.GetDatabase(e.client.dbName)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to get database: %w", err)
+		return nil, fmt.Errorf("failed to get database: %w", err)
 	}
 
 	table, err := db.GetTable(tableName)
@@ -144,17 +146,17 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
 		// Table doesn't exist, try to create it
 		errMsg := strings.ToLower(err.Error())
 		if !strings.Contains(errMsg, "not found") && !strings.Contains(errMsg, "doesn't exist") {
-			return nil, fmt.Errorf("Failed to get table %s: %w", tableName, err)
+			return nil, fmt.Errorf("failed to get table %s: %w", tableName, err)
 		}
 
 		// Create metadata table
 		if createErr := e.CreateMetadataStore(ctx, tenantID); createErr != nil {
-			return nil, fmt.Errorf("Failed to create metadata table: %w", createErr)
+			return nil, fmt.Errorf("failed to create metadata table: %w", createErr)
 		}
 
 		table, err = db.GetTable(tableName)
 		if err != nil {
-			return nil, fmt.Errorf("Failed to get table after creation: %w", err)
+			return nil, fmt.Errorf("failed to get table after creation: %w", err)
 		}
 	}
 
@@ -194,7 +196,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
 	// Insert metadata
 	_, err = table.Insert(insertMetadata)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to insert metadata: %w", err)
+		return nil, fmt.Errorf("failed to insert metadata: %w", err)
 	}
 
 	common.Info("InfinityConnection.InsertMetadata result", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata)))

internal/engine/infinity/sql.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -59,18 +58,23 @@ func loadFieldMapping(mappingFileName string) (aliasToActual map[string]string,
 	if mappingFileName == "" {
 		mappingFileName = "infinity_mapping.json"
 	}
-	confPath := filepath.Join(utility.GetProjectRoot(), "conf", mappingFileName)
-	data, err := os.ReadFile(confPath)
+
+	filePath, err := utility.FindConfFileInProject(mappingFileName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	data, err := os.ReadFile(*filePath)
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return map[string]string{}, map[string]string{}, nil
 		}
-		return nil, nil, fmt.Errorf("load field mapping %q: %w", confPath, err)
+		return nil, nil, fmt.Errorf("load field mapping %q: %w", *filePath, err)
 	}
 
 	fields := map[string]fieldMappingEntry{}
-	if err := json.Unmarshal(data, &fields); err != nil {
-		return nil, nil, fmt.Errorf("parse field mapping %q: %w", confPath, err)
+	if err = json.Unmarshal(data, &fields); err != nil {
+		return nil, nil, fmt.Errorf("parse field mapping %q: %w", *filePath, err)
 	}
 
 	aliasToActual = make(map[string]string, len(fields)*2)

internal/engine/infinity/sql_test.go

@@ -308,22 +308,6 @@ func TestResolvePsqlHostPort_EmptyHostInURIFallsBackToDefault(t *testing.T) {
 	}
 }
 
-// -----------------------------------------------------------------------------
-// loadFieldMapping — mirrors infinity_conn_base.py:793-807.
-// -----------------------------------------------------------------------------
-
-func TestLoadFieldMapping_MissingFileReturnsEmpty(t *testing.T) {
-	// Use a name that doesn't exist; the function should silently
-	// return empty maps (matching Python's `os.path.exists` guard).
-	a2a, r2a, err := loadFieldMapping("nonexistent_mapping_xyz.json")
-	if err != nil {
-		t.Fatalf("missing file should be a no-op, got error: %v", err)
-	}
-	if len(a2a) != 0 || len(r2a) != 0 {
-		t.Errorf("missing file should yield empty maps; got a2a=%v r2a=%v", a2a, r2a)
-	}
-}
-
 func TestLoadFieldMapping_ParsesAliases(t *testing.T) {
 	// Write a temporary mapping file.
 	dir := t.TempDir()
@@ -391,7 +375,7 @@ func TestLoadFieldMapping_EmptyNameDefaultsToInfinityMappingJSON(t *testing.T) {
 	if err != nil {
 		t.Fatalf("empty name: %v", err)
 	}
-	if len(a2a) != 0 || len(r2a) != 0 {
+	if len(a2a) == 0 || len(r2a) == 0 {
 		t.Errorf("empty name + no file should yield empty maps; got a2a=%v r2a=%v", a2a, r2a)
 	}
 }

internal/engine/nats/nats.go

@@ -48,15 +48,16 @@ func NewNatsEngine(host string, port int) *NatsEngine {
 
 func (n *NatsEngine) Init() error {
 	var err error
-	n.nc, err = nats.Connect(nats.DefaultURL)
+	natsURL := fmt.Sprintf("nats://%s:%d", n.host, n.port)
+	n.nc, err = nats.Connect(natsURL)
 	if err != nil {
-		return fmt.Errorf("failed to connect to NATS: %w", err)
+		return fmt.Errorf("failed to connect to NATS at %s: %w", natsURL, err)
 	}
 
 	n.jetStream, err = jetstream.New(n.nc)
 	if err != nil {
 		n.nc.Close()
-		return fmt.Errorf("failed to create JetStream context: %w", err)
+		return fmt.Errorf("failed to create JetStream context at %s: %w", natsURL, err)
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -73,19 +74,19 @@ func (n *NatsEngine) Init() error {
 
 	n.stream, err = n.jetStream.CreateStream(ctx, streamCfg)
 	if err != nil {
-		if err.Error() != "stream already exists" {
+		if !strings.Contains(err.Error(), "already exists") {
 			n.nc.Close()
-			return fmt.Errorf("fail to create stream: %w", err)
+			return fmt.Errorf("fail to create stream at %s: %w", natsURL, err)
 		}
 
 		common.Info("NATS stream already exists, use existing stream")
 		n.stream, err = n.jetStream.Stream(ctx, "RAGFLOW_TASKS")
 		if err != nil {
 			n.nc.Close()
-			return fmt.Errorf("fail to get existing stream: %w", err)
+			return fmt.Errorf("fail to get existing stream at %s: %w", natsURL, err)
 		}
 	} else {
-		common.Info("NATS stream create successfully")
+		common.Info(fmt.Sprintf("NATS stream create successfully at %s", natsURL))
 	}
 
 	return nil
@@ -181,6 +182,9 @@ func (n *NatsEngine) ListMessages(messageType string, pending bool) ([]map[strin
 }
 
 func (n *NatsEngine) InitConsumer(subject string) error {
+	if n.stream == nil {
+		return fmt.Errorf("NATS stream is nil, engine not properly initialized")
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()