Go: introduce clickhouse (#16996)

### Summary

Introduce Clickhouse for data statistics.

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-07-16 18:36:56 +08:00
committed by GitHub
parent caa76fb68e
commit c7a623ff81
11 changed files with 297 additions and 26 deletions

View File

@@ -27,6 +27,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/clickhouse"
"ragflow/internal/engine/elasticsearch"
"ragflow/internal/engine/redis"
"ragflow/internal/entity"
@@ -970,6 +971,14 @@ func (s *Service) ListServices() ([]map[string]interface{}, error) {
configDict["status"] = "timeout"
}
if serviceDetail != nil {
// delete element of configDict
delete(configDict, "extra")
delete(configDict, "database")
delete(configDict, "password")
delete(configDict, "sample_ratio")
delete(configDict, "secure")
delete(configDict, "stdout")
delete(configDict, "user")
results = append(results, configDict)
}
}
@@ -1042,6 +1051,10 @@ func (s *Service) GetServiceDetails(configDict map[string]interface{}) (map[stri
return s.checkRAGFlowServerAlive(name)
case "file_store":
return s.checkMinioAlive(name)
case "olap":
return s.checkOlapAlive(name)
case "tracing":
return s.checkTracingAlive(name)
default:
return nil, nil
}
@@ -1368,6 +1381,41 @@ func (s *Service) checkNatsAlive(name string, ip string, port int) (map[string]i
}, nil
}
// checkTracingAlive checks if tracing is alive
func (s *Service) checkTracingAlive(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "unknown",
"message": "Tracing health check not implemented",
}, nil
}
// checkOlapAlive checks if ClickHouse is alive
func (s *Service) checkOlapAlive(name string) (map[string]interface{}, error) {
clickhouseDriver := clickhouse.GetDriver()
if clickhouseDriver == nil {
return map[string]interface{}{
"service_name": name,
"status": "ClickHouse engine not initialized",
}, nil
}
status, err := clickhouseDriver.Status()
if err != nil {
return map[string]interface{}{
"service_name": name,
"status": "timeout",
"message": fmt.Sprintf("error: %s", err.Error()),
}, nil
}
return map[string]interface{}{
"service_name": name,
"status": "alive",
"message": status,
}, nil
}
// ShutdownService shutdown service
func (s *Service) ShutdownService(serviceID string) (map[string]interface{}, error) {
// TODO: Implement with proper service manager

View File

@@ -0,0 +1,102 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package clickhouse
import (
//"context"
//"crypto/tls"
//"fmt"
//"log"
"context"
"fmt"
"ragflow/internal/server"
"sync"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
var (
globalDriver *Driver
once sync.Once
)
// GetDriver gets global ClickHouse client instance
func GetDriver() *Driver {
return globalDriver
}
type Driver struct {
conn driver.Conn
config *clickhouse.Options
}
func Init(config *server.ClickhouseConfig, ctx context.Context) error {
var err error
once.Do(func() {
address := fmt.Sprintf("%s:%d", config.Host, config.Port)
globalDriver = &Driver{}
globalDriver.config = &clickhouse.Options{
Addr: []string{address},
Auth: clickhouse.Auth{
Database: config.Database,
Username: config.User,
Password: config.Password,
},
DialTimeout: 5 * time.Second,
MaxOpenConns: 10,
MaxIdleConns: 5,
ConnMaxLifetime: time.Hour,
}
globalDriver.conn, err = clickhouse.Open(globalDriver.config)
if err != nil {
return
}
if err = globalDriver.conn.Ping(ctx); err != nil {
return
}
})
return err
}
func (d *Driver) Ping(ctx context.Context) error {
return d.conn.Ping(ctx)
}
func (d *Driver) Status() (map[string]interface{}, error) {
stats := d.conn.Stats()
version, err := d.conn.ServerVersion()
if err != nil {
return nil, fmt.Errorf("failed to get server version: %w", err)
}
return map[string]interface{}{
"max_open_conns": stats.MaxOpenConns,
"max_idle_conns": stats.MaxIdleConns,
"open": stats.Open,
"idle": stats.Idle,
"server_version": version.String(),
}, nil
}
func Close() error {
return globalDriver.conn.Close()
}

View File

@@ -54,6 +54,7 @@ type Config struct {
TaskExecutor TaskExecutorConfig `mapstructure:"task_executor"`
FileSyncer FileSyncerConfig `mapstructure:"file_syncer"`
OTel OtelConfig `mapstructure:"otel"`
Clickhouse ClickhouseConfig `mapstructure:"clickhouse"`
}
// AdminConfig admin server configuration
@@ -90,7 +91,14 @@ type OtelConfig struct {
Stdout bool `mapstructure:"stdout"`
}
// UserDefaultLLMConfig user default LLM configuration
type ClickhouseConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
}
type UserDefaultLLMConfig struct {
DefaultModels DefaultModelsConfig `mapstructure:"default_models"`
}
@@ -427,13 +435,17 @@ func Init(configPath string) error {
}
delete(configDict, "message_queue_type")
case "nats":
host := getString(configDict, "host")
port := getInt(configDict, "port")
configDict["id"] = id
configDict["name"] = "nats"
configDict["host"] = host
configDict["port"] = port
configDict["service_type"] = "message_queue"
case "otel":
configDict["id"] = id
configDict["name"] = "jaeger"
configDict["service_type"] = "tracing"
case "clickhouse":
configDict["id"] = id
configDict["name"] = "clickhouse"
configDict["service_type"] = "olap"
case "admin":
// Skip admin section
continue