mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-31 04:59:24 +08:00
Go: introduce clickhouse (#16996)
### Summary Introduce Clickhouse for data statistics. Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
@@ -27,6 +27,7 @@ import (
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/engine"
|
||||
"ragflow/internal/engine/clickhouse"
|
||||
"ragflow/internal/engine/elasticsearch"
|
||||
"ragflow/internal/engine/redis"
|
||||
"ragflow/internal/entity"
|
||||
@@ -970,6 +971,14 @@ func (s *Service) ListServices() ([]map[string]interface{}, error) {
|
||||
configDict["status"] = "timeout"
|
||||
}
|
||||
if serviceDetail != nil {
|
||||
// delete element of configDict
|
||||
delete(configDict, "extra")
|
||||
delete(configDict, "database")
|
||||
delete(configDict, "password")
|
||||
delete(configDict, "sample_ratio")
|
||||
delete(configDict, "secure")
|
||||
delete(configDict, "stdout")
|
||||
delete(configDict, "user")
|
||||
results = append(results, configDict)
|
||||
}
|
||||
}
|
||||
@@ -1042,6 +1051,10 @@ func (s *Service) GetServiceDetails(configDict map[string]interface{}) (map[stri
|
||||
return s.checkRAGFlowServerAlive(name)
|
||||
case "file_store":
|
||||
return s.checkMinioAlive(name)
|
||||
case "olap":
|
||||
return s.checkOlapAlive(name)
|
||||
case "tracing":
|
||||
return s.checkTracingAlive(name)
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1368,6 +1381,41 @@ func (s *Service) checkNatsAlive(name string, ip string, port int) (map[string]i
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkTracingAlive checks if tracing is alive
|
||||
func (s *Service) checkTracingAlive(name string) (map[string]interface{}, error) {
|
||||
return map[string]interface{}{
|
||||
"service_name": name,
|
||||
"status": "unknown",
|
||||
"message": "Tracing health check not implemented",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkOlapAlive checks if ClickHouse is alive
|
||||
func (s *Service) checkOlapAlive(name string) (map[string]interface{}, error) {
|
||||
clickhouseDriver := clickhouse.GetDriver()
|
||||
if clickhouseDriver == nil {
|
||||
return map[string]interface{}{
|
||||
"service_name": name,
|
||||
"status": "ClickHouse engine not initialized",
|
||||
}, nil
|
||||
}
|
||||
|
||||
status, err := clickhouseDriver.Status()
|
||||
if err != nil {
|
||||
return map[string]interface{}{
|
||||
"service_name": name,
|
||||
"status": "timeout",
|
||||
"message": fmt.Sprintf("error: %s", err.Error()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"service_name": name,
|
||||
"status": "alive",
|
||||
"message": status,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ShutdownService shutdown service
|
||||
func (s *Service) ShutdownService(serviceID string) (map[string]interface{}, error) {
|
||||
// TODO: Implement with proper service manager
|
||||
|
||||
102
internal/engine/clickhouse/clickhouse.go
Normal file
102
internal/engine/clickhouse/clickhouse.go
Normal file
@@ -0,0 +1,102 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
//"context"
|
||||
//"crypto/tls"
|
||||
//"fmt"
|
||||
//"log"
|
||||
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/server"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
var (
|
||||
globalDriver *Driver
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// GetDriver gets global ClickHouse client instance
|
||||
func GetDriver() *Driver {
|
||||
return globalDriver
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
conn driver.Conn
|
||||
config *clickhouse.Options
|
||||
}
|
||||
|
||||
func Init(config *server.ClickhouseConfig, ctx context.Context) error {
|
||||
var err error
|
||||
once.Do(func() {
|
||||
address := fmt.Sprintf("%s:%d", config.Host, config.Port)
|
||||
|
||||
globalDriver = &Driver{}
|
||||
globalDriver.config = &clickhouse.Options{
|
||||
Addr: []string{address},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: config.Database,
|
||||
Username: config.User,
|
||||
Password: config.Password,
|
||||
},
|
||||
DialTimeout: 5 * time.Second,
|
||||
MaxOpenConns: 10,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
}
|
||||
|
||||
globalDriver.conn, err = clickhouse.Open(globalDriver.config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalDriver.conn.Ping(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Driver) Ping(ctx context.Context) error {
|
||||
return d.conn.Ping(ctx)
|
||||
}
|
||||
|
||||
func (d *Driver) Status() (map[string]interface{}, error) {
|
||||
stats := d.conn.Stats()
|
||||
version, err := d.conn.ServerVersion()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get server version: %w", err)
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"max_open_conns": stats.MaxOpenConns,
|
||||
"max_idle_conns": stats.MaxIdleConns,
|
||||
"open": stats.Open,
|
||||
"idle": stats.Idle,
|
||||
"server_version": version.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func Close() error {
|
||||
return globalDriver.conn.Close()
|
||||
}
|
||||
@@ -54,6 +54,7 @@ type Config struct {
|
||||
TaskExecutor TaskExecutorConfig `mapstructure:"task_executor"`
|
||||
FileSyncer FileSyncerConfig `mapstructure:"file_syncer"`
|
||||
OTel OtelConfig `mapstructure:"otel"`
|
||||
Clickhouse ClickhouseConfig `mapstructure:"clickhouse"`
|
||||
}
|
||||
|
||||
// AdminConfig admin server configuration
|
||||
@@ -90,7 +91,14 @@ type OtelConfig struct {
|
||||
Stdout bool `mapstructure:"stdout"`
|
||||
}
|
||||
|
||||
// UserDefaultLLMConfig user default LLM configuration
|
||||
type ClickhouseConfig struct {
|
||||
Host string `mapstructure:"host"`
|
||||
Port int `mapstructure:"port"`
|
||||
User string `mapstructure:"user"`
|
||||
Password string `mapstructure:"password"`
|
||||
Database string `mapstructure:"database"`
|
||||
}
|
||||
|
||||
type UserDefaultLLMConfig struct {
|
||||
DefaultModels DefaultModelsConfig `mapstructure:"default_models"`
|
||||
}
|
||||
@@ -427,13 +435,17 @@ func Init(configPath string) error {
|
||||
}
|
||||
delete(configDict, "message_queue_type")
|
||||
case "nats":
|
||||
host := getString(configDict, "host")
|
||||
port := getInt(configDict, "port")
|
||||
configDict["id"] = id
|
||||
configDict["name"] = "nats"
|
||||
configDict["host"] = host
|
||||
configDict["port"] = port
|
||||
configDict["service_type"] = "message_queue"
|
||||
case "otel":
|
||||
configDict["id"] = id
|
||||
configDict["name"] = "jaeger"
|
||||
configDict["service_type"] = "tracing"
|
||||
case "clickhouse":
|
||||
configDict["id"] = id
|
||||
configDict["name"] = "clickhouse"
|
||||
configDict["service_type"] = "olap"
|
||||
case "admin":
|
||||
// Skip admin section
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user