mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-06-29 15:31:05 +08:00
feat[Go]: implement /system/stats and refactor /system/config/log (#15407)
### What problem does this PR solve? As title ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Refactoring
This commit is contained in:
@@ -18,7 +18,9 @@ package common
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
@@ -30,23 +32,70 @@ var (
|
|||||||
Sugar *zap.SugaredLogger
|
Sugar *zap.SugaredLogger
|
||||||
levelMu sync.RWMutex
|
levelMu sync.RWMutex
|
||||||
atomicLevel zap.AtomicLevel
|
atomicLevel zap.AtomicLevel
|
||||||
|
pkgLevels map[string]string
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func parseZapLevel(level string) (zapcore.Level, error) {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(level)) {
|
||||||
|
case "debug":
|
||||||
|
return zapcore.DebugLevel, nil
|
||||||
|
case "info":
|
||||||
|
return zapcore.InfoLevel, nil
|
||||||
|
case "warn", "warning":
|
||||||
|
return zapcore.WarnLevel, nil
|
||||||
|
case "error":
|
||||||
|
return zapcore.ErrorLevel, nil
|
||||||
|
case "fatal":
|
||||||
|
return zapcore.FatalLevel, nil
|
||||||
|
case "panic":
|
||||||
|
return zapcore.PanicLevel, nil
|
||||||
|
default:
|
||||||
|
return zapcore.InfoLevel, fmt.Errorf("unknown log level: %s", level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func logLevelName(level zapcore.Level) string {
|
||||||
|
if level == zapcore.WarnLevel {
|
||||||
|
return "WARNING"
|
||||||
|
}
|
||||||
|
return strings.ToUpper(level.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func initPackageLogLevels(rootLevel zapcore.Level) {
|
||||||
|
levels := make(map[string]string)
|
||||||
|
for _, item := range strings.Split(os.Getenv("LOG_LEVELS"), ",") {
|
||||||
|
terms := strings.SplitN(item, "=", 2)
|
||||||
|
if len(terms) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pkgName := strings.TrimSpace(terms[0])
|
||||||
|
if pkgName == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
level, err := parseZapLevel(terms[1])
|
||||||
|
if err != nil {
|
||||||
|
level = zapcore.InfoLevel
|
||||||
|
}
|
||||||
|
levels[pkgName] = logLevelName(level)
|
||||||
|
}
|
||||||
|
// I set it to align with python for now, we shall change it later before ragflow 1.0
|
||||||
|
if _, ok := levels["peewee"]; !ok {
|
||||||
|
levels["peewee"] = logLevelName(zapcore.WarnLevel)
|
||||||
|
}
|
||||||
|
if _, ok := levels["pdfminer"]; !ok {
|
||||||
|
levels["pdfminer"] = logLevelName(zapcore.WarnLevel)
|
||||||
|
}
|
||||||
|
if _, ok := levels["root"]; !ok {
|
||||||
|
levels["root"] = logLevelName(rootLevel)
|
||||||
|
}
|
||||||
|
pkgLevels = levels
|
||||||
|
}
|
||||||
|
|
||||||
// Init initializes the global logger
|
// Init initializes the global logger
|
||||||
// Note: This requires zap to be installed: go get go.uber.org/zap
|
// Note: This requires zap to be installed: go get go.uber.org/zap
|
||||||
func Init(level string) error {
|
func Init(level string) error {
|
||||||
// Parse log level
|
zapLevel, err := parseZapLevel(level)
|
||||||
var zapLevel zapcore.Level
|
if err != nil {
|
||||||
switch level {
|
|
||||||
case "debug":
|
|
||||||
zapLevel = zapcore.DebugLevel
|
|
||||||
case "info":
|
|
||||||
zapLevel = zapcore.InfoLevel
|
|
||||||
case "warn":
|
|
||||||
zapLevel = zapcore.WarnLevel
|
|
||||||
case "error":
|
|
||||||
zapLevel = zapcore.ErrorLevel
|
|
||||||
default:
|
|
||||||
zapLevel = zapcore.InfoLevel
|
zapLevel = zapcore.InfoLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +137,10 @@ func Init(level string) error {
|
|||||||
Logger = logger
|
Logger = logger
|
||||||
Sugar = logger.Sugar()
|
Sugar = logger.Sugar()
|
||||||
|
|
||||||
|
levelMu.Lock()
|
||||||
|
initPackageLogLevels(zapLevel)
|
||||||
|
levelMu.Unlock()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,29 +208,51 @@ func GetLevel() string {
|
|||||||
return atomicLevel.String()
|
return atomicLevel.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetLogLevels returns Python-compatible package log levels.
|
||||||
|
func GetLogLevels() map[string]string {
|
||||||
|
levelMu.RLock()
|
||||||
|
defer levelMu.RUnlock()
|
||||||
|
|
||||||
|
levels := make(map[string]string, len(pkgLevels))
|
||||||
|
for pkgName, level := range pkgLevels {
|
||||||
|
levels[pkgName] = level
|
||||||
|
}
|
||||||
|
return levels
|
||||||
|
}
|
||||||
|
|
||||||
// SetLevel sets the log level at runtime
|
// SetLevel sets the log level at runtime
|
||||||
func SetLevel(level string) error {
|
func SetLevel(level string) error {
|
||||||
levelMu.Lock()
|
levelMu.Lock()
|
||||||
defer levelMu.Unlock()
|
defer levelMu.Unlock()
|
||||||
|
|
||||||
var zapLevel zapcore.Level
|
zapLevel, err := parseZapLevel(level)
|
||||||
switch level {
|
if err != nil {
|
||||||
case "debug":
|
return err
|
||||||
zapLevel = zapcore.DebugLevel
|
|
||||||
case "info":
|
|
||||||
zapLevel = zapcore.InfoLevel
|
|
||||||
case "warn", "warning":
|
|
||||||
zapLevel = zapcore.WarnLevel
|
|
||||||
case "error":
|
|
||||||
zapLevel = zapcore.ErrorLevel
|
|
||||||
case "fatal":
|
|
||||||
zapLevel = zapcore.FatalLevel
|
|
||||||
case "panic":
|
|
||||||
zapLevel = zapcore.PanicLevel
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("unknown log level: %s", level)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
atomicLevel.SetLevel(zapLevel)
|
atomicLevel.SetLevel(zapLevel)
|
||||||
|
if pkgLevels == nil {
|
||||||
|
pkgLevels = make(map[string]string)
|
||||||
|
}
|
||||||
|
pkgLevels["root"] = logLevelName(zapLevel)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetPackageLogLevel sets a Python-compatible package log level at runtime.
|
||||||
|
func SetPackageLogLevel(pkgName, level string) error {
|
||||||
|
zapLevel, err := parseZapLevel(level)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
levelMu.Lock()
|
||||||
|
defer levelMu.Unlock()
|
||||||
|
|
||||||
|
if pkgLevels == nil {
|
||||||
|
pkgLevels = make(map[string]string)
|
||||||
|
}
|
||||||
|
pkgLevels[pkgName] = logLevelName(zapLevel)
|
||||||
|
if pkgName == "root" {
|
||||||
|
atomicLevel.SetLevel(zapLevel)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,6 +79,46 @@ func NewAPI4ConversationDAO() *API4ConversationDAO {
|
|||||||
return &API4ConversationDAO{}
|
return &API4ConversationDAO{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConversationStatsRow is one daily aggregate row for api_4_conversation.
|
||||||
|
type ConversationStatsRow struct {
|
||||||
|
Dt string `gorm:"column:dt"`
|
||||||
|
PV int64 `gorm:"column:pv"`
|
||||||
|
UV int64 `gorm:"column:uv"`
|
||||||
|
Tokens float64 `gorm:"column:tokens"`
|
||||||
|
Duration float64 `gorm:"column:duration"`
|
||||||
|
Round float64 `gorm:"column:round"`
|
||||||
|
ThumbUp int64 `gorm:"column:thumb_up"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats returns daily conversation aggregates for a tenant.
|
||||||
|
func (dao *API4ConversationDAO) Stats(tenantID, fromDate, toDate string, source *string) ([]ConversationStatsRow, error) {
|
||||||
|
var rows []ConversationStatsRow
|
||||||
|
dateExpr := "DATE_FORMAT(a.create_date, '%Y-%m-%d 00:00:00')"
|
||||||
|
db := DB.Table("api_4_conversation AS a").
|
||||||
|
Select(`
|
||||||
|
DATE_FORMAT(a.create_date, '%Y-%m-%d 00:00:00') AS dt,
|
||||||
|
COUNT(a.id) AS pv,
|
||||||
|
COUNT(DISTINCT a.user_id) AS uv,
|
||||||
|
COALESCE(SUM(a.tokens), 0) AS tokens,
|
||||||
|
COALESCE(SUM(a.duration), 0) AS duration,
|
||||||
|
COALESCE(AVG(a.round), 0) AS round,
|
||||||
|
COALESCE(SUM(a.thumb_up), 0) AS thumb_up
|
||||||
|
`).
|
||||||
|
Joins("JOIN dialog AS d ON a.dialog_id = d.id AND d.tenant_id = ?", tenantID).
|
||||||
|
Where("a.create_date >= ? AND a.create_date <= ?", fromDate, toDate)
|
||||||
|
|
||||||
|
if source == nil {
|
||||||
|
db = db.Where("a.source IS NULL")
|
||||||
|
} else {
|
||||||
|
db = db.Where("a.source = ?", *source)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := db.Group(dateExpr).
|
||||||
|
Order(dateExpr).
|
||||||
|
Scan(&rows).Error
|
||||||
|
return rows, err
|
||||||
|
}
|
||||||
|
|
||||||
// DeleteByDialogIDs deletes API4Conversations by dialog IDs (hard delete)
|
// DeleteByDialogIDs deletes API4Conversations by dialog IDs (hard delete)
|
||||||
func (dao *API4ConversationDAO) DeleteByDialogIDs(dialogIDs []string) (int64, error) {
|
func (dao *API4ConversationDAO) DeleteByDialogIDs(dialogIDs []string) (int64, error) {
|
||||||
if len(dialogIDs) == 0 {
|
if len(dialogIDs) == 0 {
|
||||||
|
|||||||
69
internal/handler/stats.go
Normal file
69
internal/handler/stats.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
//
|
||||||
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"ragflow/internal/common"
|
||||||
|
"ragflow/internal/service"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetStats returns API conversation statistics for the current user's tenant.
|
||||||
|
func (h *SystemHandler) GetStats(c *gin.Context) {
|
||||||
|
user, errorCode, errorMessage := GetUser(c)
|
||||||
|
if errorCode != common.CodeSuccess {
|
||||||
|
jsonError(c, errorCode, errorMessage)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
fromDate := c.DefaultQuery("from_date", now.AddDate(0, 0, -7).Format("2006-01-02 00:00:00"))
|
||||||
|
toDate := c.DefaultQuery("to_date", now.Format("2006-01-02 15:04:05"))
|
||||||
|
if len(toDate) == 10 {
|
||||||
|
toDate += " 23:59:59"
|
||||||
|
}
|
||||||
|
|
||||||
|
var source *string
|
||||||
|
if _, ok := c.GetQuery("canvas_id"); ok {
|
||||||
|
agentSource := "agent"
|
||||||
|
source = &agentSource
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := h.systemService.GetStats(user.ID, fromDate, toDate, source)
|
||||||
|
if err != nil {
|
||||||
|
code := common.CodeExceptionError
|
||||||
|
if errors.Is(err, service.ErrTenantNotFound) {
|
||||||
|
code = common.CodeDataError
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusOK, gin.H{
|
||||||
|
"code": code,
|
||||||
|
"message": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{
|
||||||
|
"code": common.CodeSuccess,
|
||||||
|
"message": "success",
|
||||||
|
"data": stats,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -164,44 +164,46 @@ func (h *SystemHandler) GetVersion(c *gin.Context) {
|
|||||||
|
|
||||||
// GetLogLevel returns the current log level
|
// GetLogLevel returns the current log level
|
||||||
func (h *SystemHandler) GetLogLevel(c *gin.Context) {
|
func (h *SystemHandler) GetLogLevel(c *gin.Context) {
|
||||||
level := common.GetLevel()
|
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"code": 0,
|
"code": 0,
|
||||||
"message": "success",
|
"message": "success",
|
||||||
"data": gin.H{"level": level},
|
"data": common.GetLogLevels(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetLogLevelRequest set log level request
|
// SetLogLevelRequest set log level request
|
||||||
type SetLogLevelRequest struct {
|
type SetLogLevelRequest struct {
|
||||||
Level string `json:"level" binding:"required"`
|
PkgName string `json:"pkg_name" binding:"required"`
|
||||||
|
Level string `json:"level" binding:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetLogLevel sets the log level at runtime
|
// SetLogLevel sets the log level at runtime
|
||||||
func (h *SystemHandler) SetLogLevel(c *gin.Context) {
|
func (h *SystemHandler) SetLogLevel(c *gin.Context) {
|
||||||
var req SetLogLevelRequest
|
var req SetLogLevelRequest
|
||||||
if err := c.ShouldBindJSON(&req); err != nil {
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
c.JSON(http.StatusBadRequest, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"code": 400,
|
"code": common.CodeDataError,
|
||||||
"message": "level is required",
|
"message": "pkg_name and level are required",
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := common.SetLevel(req.Level); err != nil {
|
if err := common.SetPackageLogLevel(req.PkgName, req.Level); err != nil {
|
||||||
c.JSON(http.StatusBadRequest, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"code": 400,
|
"code": common.CodeDataError,
|
||||||
"message": err.Error(),
|
"message": "Invalid log level: " + req.Level,
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
config := server.GetConfig()
|
config := server.GetConfig()
|
||||||
config.Log.Level = req.Level
|
if req.PkgName == "root" && config != nil {
|
||||||
|
config.Log.Level = common.GetLevel()
|
||||||
|
}
|
||||||
|
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"code": 0,
|
"code": 0,
|
||||||
"message": "Log level updated successfully",
|
"message": "success",
|
||||||
"data": gin.H{"level": req.Level},
|
"data": gin.H{"pkg_name": req.PkgName, "level": req.Level},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -343,14 +343,22 @@ func (r *Router) Setup(engine *gin.Engine) {
|
|||||||
{
|
{
|
||||||
system.GET("/configs", r.systemHandler.GetConfigs)
|
system.GET("/configs", r.systemHandler.GetConfigs)
|
||||||
system.GET("/status", r.systemHandler.GetStatus)
|
system.GET("/status", r.systemHandler.GetStatus)
|
||||||
log := system.Group("/log")
|
system.GET("/stats", r.systemHandler.GetStats)
|
||||||
|
|
||||||
|
config := system.Group("/config")
|
||||||
{
|
{
|
||||||
// /api/v1/system/log GET
|
config.GET("/log", r.systemHandler.GetLogLevel)
|
||||||
log.GET("", r.systemHandler.GetLogLevel)
|
config.PUT("/log", r.systemHandler.SetLogLevel)
|
||||||
// /api/v1/system/log PUT
|
|
||||||
log.PUT("", r.systemHandler.SetLogLevel)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//log := system.Group("/log")
|
||||||
|
//{
|
||||||
|
// // /api/v1/system/log GET
|
||||||
|
// log.GET("", r.systemHandler.GetLogLevel)
|
||||||
|
// // /api/v1/system/log PUT
|
||||||
|
// log.PUT("", r.systemHandler.SetLogLevel)
|
||||||
|
//}
|
||||||
|
|
||||||
tokens := system.Group("/tokens")
|
tokens := system.Group("/tokens")
|
||||||
{
|
{
|
||||||
// list tokens /api/v1/system/tokens GET
|
// list tokens /api/v1/system/tokens GET
|
||||||
|
|||||||
73
internal/service/stats.go
Normal file
73
internal/service/stats.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
//
|
||||||
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
|
||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"ragflow/internal/dao"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrTenantNotFound indicates the current user has no tenant relation.
|
||||||
|
var ErrTenantNotFound = errors.New("Tenant not found!")
|
||||||
|
|
||||||
|
// StatPoint matches the frontend [date, value] tuple shape.
|
||||||
|
type StatPoint [2]interface{}
|
||||||
|
|
||||||
|
// StatsResponse matches Python GET /api/v1/system/stats response data.
|
||||||
|
type StatsResponse struct {
|
||||||
|
PV []StatPoint `json:"pv"`
|
||||||
|
UV []StatPoint `json:"uv"`
|
||||||
|
Speed []StatPoint `json:"speed"`
|
||||||
|
Tokens []StatPoint `json:"tokens"`
|
||||||
|
Round []StatPoint `json:"round"`
|
||||||
|
ThumbUp []StatPoint `json:"thumb_up"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns daily API conversation statistics for the first tenant of a user.
|
||||||
|
func (s *SystemService) GetStats(userID, fromDate, toDate string, source *string) (*StatsResponse, error) {
|
||||||
|
userTenantDAO := dao.NewUserTenantDAO()
|
||||||
|
tenants, err := userTenantDAO.GetByUserID(userID)
|
||||||
|
if err != nil || len(tenants) == 0 {
|
||||||
|
return nil, ErrTenantNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := dao.NewAPI4ConversationDAO().Stats(tenants[0].TenantID, fromDate, toDate, source)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
response := &StatsResponse{
|
||||||
|
PV: make([]StatPoint, 0, len(rows)),
|
||||||
|
UV: make([]StatPoint, 0, len(rows)),
|
||||||
|
Speed: make([]StatPoint, 0, len(rows)),
|
||||||
|
Tokens: make([]StatPoint, 0, len(rows)),
|
||||||
|
Round: make([]StatPoint, 0, len(rows)),
|
||||||
|
ThumbUp: make([]StatPoint, 0, len(rows)),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, row := range rows {
|
||||||
|
response.PV = append(response.PV, StatPoint{row.Dt, row.PV})
|
||||||
|
response.UV = append(response.UV, StatPoint{row.Dt, row.UV})
|
||||||
|
response.Speed = append(response.Speed, StatPoint{row.Dt, row.Tokens / (row.Duration + 0.1)})
|
||||||
|
response.Tokens = append(response.Tokens, StatPoint{row.Dt, row.Tokens / 1000.0})
|
||||||
|
response.Round = append(response.Round, StatPoint{row.Dt, row.Round})
|
||||||
|
response.ThumbUp = append(response.ThumbUp, StatPoint{row.Dt, row.ThumbUp})
|
||||||
|
}
|
||||||
|
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user