Go: add audit log framework (#17129)

### Summary

Prepare for audit log

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

internal/admin/handler_ee.go

@@ -1968,3 +1968,29 @@ func (h *Handler) GetTokenStatsSummary(c *gin.Context) {
 
 	common.SuccessWithData(c, stats, "success")
 }
+
+func (h *Handler) ListLogs(c *gin.Context) {
+	userName := c.Query("user_name")
+	if userName == "" {
+		common.ErrorWithCode(c, common.CodeBadRequest, "User name is required")
+		return
+	}
+	days := c.Query("days")
+	if days == "" {
+		days = "7"
+	}
+	daysInt, err := strconv.Atoi(days)
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeBadRequest, "Invalid days")
+		return
+	}
+
+	stats, err := h.service.ListLogs(userName, daysInt)
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeDataError, err.Error())
+		return
+	}
+
+	common.SuccessWithData(c, stats, "success")
+
+}

internal/admin/router_ee.go

@@ -80,6 +80,9 @@ func RegisterEERouter(protected *gin.RouterGroup, r *Router) {
 	protected.GET("/stats/token/users", r.handler.GetTokenUsersStats)
 	protected.GET("/stats/token/summary", r.handler.GetTokenStatsSummary)
 
+	// Logs
+	protected.GET("/logs", r.handler.ListLogs)
+
 	// Stats data info
 	protected.GET("/users/:username/activity", r.handler.ShowUserActivity)
 	protected.GET("/users/:username/dataset", r.handler.ShowUserDatasetSummary)

internal/admin/service_ee.go

@@ -1328,3 +1328,16 @@ func (s *Service) GetTokenStatsSummary(fromDate, toDate string) (map[string]inte
 
 	return result, nil
 }
+
+// ListLogs lists operation logs for the user.
+func (s *Service) ListLogs(userName string, days int) ([]map[string]interface{}, error) {
+	result := []map[string]interface{}{
+		{
+			"command":   "list_logs",
+			"user_name": userName,
+			"days":      days,
+			"error":     "'List operation logs' is not supported",
+		},
+	}
+	return result, nil
+}

internal/cli/admin_command.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net/url"
 	"ragflow/internal/common"
+	"strconv"
 )
 
 // PingServer pings the server to check if it's alive
@@ -2550,6 +2551,37 @@ func (c *CLI) AdminListUserDefaultModelsCommand(cmd *Command) (ResponseIf, error
 	return HandleCommonResponse(resp, fmt.Sprintf("list user %s default models", userName))
 }
 
+// AdminListUserLogsCommand list user operation logs
+func (c *CLI) AdminListUserLogsCommand(cmd *Command) (ResponseIf, error) {
+
+	if c.Config.CLIMode != AdminMode || c.AdminServerClient.LoginToken == nil {
+		return nil, fmt.Errorf("this command is only allowed in ADMIN mode or already login")
+	}
+
+	userName, ok := cmd.Params["user_name"].(string)
+	if !ok {
+		return nil, fmt.Errorf("user_name not provided")
+	}
+
+	days, ok := cmd.Params["days"].(int)
+	if !ok {
+		return nil, fmt.Errorf("days not provided")
+	}
+
+	q := url.Values{}
+	q.Set("user_name", userName)
+	q.Set("days", strconv.Itoa(days))
+
+	baseUrl := fmt.Sprintf("/admin/logs?%s", q.Encode())
+
+	resp, err := c.AdminServerClient.Request("GET", baseUrl, "admin", nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list user %s logs: %w", userName, err)
+	}
+
+	return HandleCommonResponse(resp, fmt.Sprintf("list user %s logs", userName))
+}
+
 func (c *CLI) AdminStopUserIngestionTasksCommand(cmd *Command) (ResponseIf, error) {
 
 	if c.Config.CLIMode != AdminMode || c.AdminServerClient.LoginToken == nil {

internal/cli/admin_parser.go

@@ -3191,10 +3191,12 @@ func (p *Parser) parseAdminListUserCommand() (*Command, error) {
 	case TokenProviders:
 		p.nextToken()
 		cmd = NewCommand("admin_list_user_providers")
+	case TokenLogs:
+		return p.parseAdminListUserLogs(userName)
 	case TokenDefault:
 		return p.parseAdminListUserDefaultModels(userName)
 	default:
-		return nil, fmt.Errorf("expected INGESTION or DATASETS or AGENTS or CHATS or SEARCHES or MODELS or FILES or KEYS after USER")
+		return nil, fmt.Errorf("expected INGESTION or DATASETS or AGENTS or CHATS or SEARCHES or MODELS or FILES or KEYS or LOGS after USER")
 	}
 
 	// Semicolon is optional
@@ -3299,6 +3301,31 @@ func (p *Parser) parseAdminListUserDefaultModels(userName string) (*Command, err
 	return cmd, nil
 }
 
+// LIST USER 'user_name' LOGS;
+// LIST USER 'user_name' LOGS 10;
+func (p *Parser) parseAdminListUserLogs(userName string) (*Command, error) {
+	p.nextToken() // consume LOGS
+
+	dayCount := 7
+
+	var err error
+	cmd := NewCommand("admin_list_user_operation_logs")
+	if p.curToken.Type == TokenNumber {
+		dayCount, err = p.parseNumber()
+		if err != nil {
+			return nil, err
+		}
+		p.nextToken()
+	}
+	cmd.Params["user_name"] = userName
+	cmd.Params["days"] = dayCount
+
+	if p.curToken.Type == TokenSemicolon {
+		p.nextToken()
+	}
+	return cmd, nil
+}
+
 // STOP USER 'user@example.com' INGESTION TASKS 'created';
 func (p *Parser) parseAdminStopUserCommand() (*Command, error) {
 	p.nextToken() // consume USER

internal/cli/cli_http.go

@@ -259,6 +259,8 @@ func (c *CLI) ExecuteAdminCommand(cmd *Command) (ResponseIf, error) {
 		return c.AdminListUserProviderInstanceModelsCommand(cmd)
 	case "admin_list_user_default_models":
 		return c.AdminListUserDefaultModelsCommand(cmd)
+	case "admin_list_user_operation_logs":
+		return c.AdminListUserLogsCommand(cmd)
 	case "admin_stop_user_ingestion_tasks_command":
 		return c.AdminStopUserIngestionTasksCommand(cmd)
 	case "admin_remove_user_ingestion_tasks_command":

internal/cli/lexer.go

@@ -545,6 +545,8 @@ func (l *Lexer) lookupIdent(ident string) Token {
 		return Token{Type: TokenData, Value: ident}
 	case "LOG":
 		return Token{Type: TokenLog, Value: ident}
+	case "LOGS":
+		return Token{Type: TokenLogs, Value: ident}
 	case "LEVEL":
 		return Token{Type: TokenLevel, Value: ident}
 	case "DEBUG":

internal/cli/types.go

@@ -204,6 +204,7 @@ const (
 	TokenPreview
 	TokenOpenaiChat
 	TokenLog
+	TokenLogs
 	TokenLevel
 	TokenDebug
 	TokenInfo

internal/common/model_type.go

@@ -49,3 +49,19 @@ type ModelUsage struct {
 func (m *ModelUsage) String() string {
 	return fmt.Sprintf("%#v", m)
 }
+
+type OperationLog struct {
+	EventTime     time.Time `json:"event_time"`
+	UserID        string    `json:"user_id"`
+	IPAddress     string    `json:"ip_address"`
+	Operation     string    `json:"operation"`
+	APIPath       string    `json:"api_path"`
+	HTTPMethod    string    `json:"http_method"`
+	ResourceType  string    `json:"resource_type"`
+	ResourceID    string    `json:"resource_id"`
+	ResourceName  string    `json:"resource_name"`
+	RequestParams string    `json:"request_params"`
+	ErrorCode     uint16    `json:"error_code"`
+	Message       string    `json:"message"`
+	DurationMS    int64     `json:"duration_ms"`
+}

internal/engine/clickhouse/clickhouse_ee.go

@@ -41,6 +41,8 @@ func (d *Driver) CollectModelUsage(modelUsage *common.ModelUsage) error {
 	return nil
 }
 
+func (d *Driver) SaveOperationLog(operationLog *common.OperationLog) error { return nil }
+
 func (d *Driver) Status() (map[string]interface{}, error) {
 	return nil, nil
 }

internal/handler/user.go

@@ -20,16 +20,18 @@ import (
 	"fmt"
 	"net/http"
 	"ragflow/internal/common"
+	"ragflow/internal/engine/clickhouse"
 	"ragflow/internal/engine/redis"
 	"ragflow/internal/server"
 	"ragflow/internal/server/local"
 	"ragflow/internal/utility"
 	"strconv"
+	"time"
+
+	"ragflow/internal/service"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"
-
-	"ragflow/internal/service"
 )
 
 // UserHandler user handler
@@ -102,27 +104,60 @@ func (h *UserHandler) Register(c *gin.Context) {
 // @Success 200 {object} map[string]interface{}
 // @Router /api/v1/users/login [post]
 func (h *UserHandler) Login(c *gin.Context) {
+	startAt := time.Now()
+	operationLog := &common.OperationLog{
+		EventTime:  startAt,
+		Operation:  "login",
+		APIPath:    c.FullPath(),
+		HTTPMethod: c.Request.Method,
+		IPAddress:  c.ClientIP(),
+	}
+	defer func() {
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
+		clickhouseDriver := clickhouse.GetDriver()
+		err := clickhouseDriver.SaveOperationLog(operationLog)
+		if err != nil {
+			common.ResponseWithCodeData(c, common.CodeServerError, false, err.Error())
+			return
+		}
+	}()
+
 	var req service.LoginRequest
 	if err := c.ShouldBindJSON(&req); err != nil {
 		common.ResponseWithCodeData(c, common.CodeBadRequest, false, err.Error())
+		operationLog.ErrorCode = uint16(common.CodeBadRequest)
+		operationLog.Message = err.Error()
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
+	operationLog.ResourceName = req.Username
 
 	user, code, err := h.userService.Login(&req)
 	if err != nil {
 		common.ResponseWithCodeData(c, code, false, err.Error())
+		operationLog.ErrorCode = uint16(code)
+		operationLog.Message = err.Error()
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
+	operationLog.UserID = user.ID
 
 	// Sign the access_token using itsdangerous (compatible with Python)
 	secretKey, err := server.GetSecretKey(redis.Get())
 	if err != nil {
-		common.ResponseWithCodeData(c, common.CodeServerError, false, fmt.Sprintf("Failed to get secret key: %s", err.Error()))
+		errMessage := fmt.Sprintf("Failed to get secret key: %s", err.Error())
+		common.ResponseWithCodeData(c, common.CodeServerError, false, errMessage)
+		operationLog.ErrorCode = uint16(common.CodeServerError)
+		operationLog.Message = errMessage
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 	authToken, err := utility.DumpAccessToken(*user.AccessToken, secretKey)
 	if err != nil {
 		common.ResponseWithCodeData(c, common.CodeServerError, false, "Failed to generate auth token")
+		operationLog.ErrorCode = uint16(common.CodeServerError)
+		operationLog.Message = "Failed to generate auth token"
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 
@@ -149,32 +184,68 @@ func (h *UserHandler) Login(c *gin.Context) {
 // @Success 200 {object} map[string]interface{}
 // @Router /v1/user/login [post]
 func (h *UserHandler) LoginByEmail(c *gin.Context) {
+	startAt := time.Now()
+	operationLog := &common.OperationLog{
+		EventTime:  startAt,
+		Operation:  "login",
+		APIPath:    c.FullPath(),
+		HTTPMethod: c.Request.Method,
+		IPAddress:  c.ClientIP(),
+	}
+	defer func() {
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
+		clickhouseDriver := clickhouse.GetDriver()
+		err := clickhouseDriver.SaveOperationLog(operationLog)
+		if err != nil {
+			common.ResponseWithCodeData(c, common.CodeServerError, false, err.Error())
+			return
+		}
+	}()
+
 	var req service.EmailLoginRequest
 	if err := c.ShouldBindJSON(&req); err != nil {
 		common.ResponseWithCodeData(c, common.CodeBadRequest, false, err.Error())
+		operationLog.ErrorCode = uint16(common.CodeBadRequest)
+		operationLog.Message = err.Error()
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
+	operationLog.ResourceName = req.Email
 
 	if !local.IsAdminAvailable() {
 		license := local.GetAdminStatus()
 		common.ResponseWithCodeData(c, common.CodeAuthenticationError, "No", license.Reason)
+		operationLog.ErrorCode = uint16(common.CodeAuthenticationError)
+		operationLog.Message = license.Reason
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 
 	user, code, err := h.userService.LoginByEmail(&req)
 	if err != nil {
 		common.ResponseWithCodeData(c, code, false, err.Error())
+		operationLog.ErrorCode = uint16(code)
+		operationLog.Message = err.Error()
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
+	operationLog.UserID = user.ID
 
 	secretKey, err := server.GetSecretKey(redis.Get())
 	if err != nil {
-		common.ResponseWithCodeData(c, common.CodeServerError, false, fmt.Sprintf("Failed to get secret key: %s", err.Error()))
+		errorMessage := fmt.Sprintf("Failed to get secret key: %s", err.Error())
+		common.ResponseWithCodeData(c, common.CodeServerError, false, errorMessage)
+		operationLog.ErrorCode = uint16(common.CodeServerError)
+		operationLog.Message = errorMessage
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 	authToken, err := utility.DumpAccessToken(*user.AccessToken, secretKey)
 	if err != nil {
 		common.ResponseWithCodeData(c, common.CodeServerError, false, "Failed to generate auth token")
+		operationLog.ErrorCode = uint16(common.CodeServerError)
+		operationLog.Message = "Failed to generate auth token"
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 	setOAuthAuthCookie(c, authToken)
@@ -224,6 +295,24 @@ func (h *UserHandler) GetUserByID(c *gin.Context) {
 // @Success 200 {object} map[string]interface{}
 // @Router /v1/user/logout [post]
 func (h *UserHandler) Logout(c *gin.Context) {
+	startAt := time.Now()
+	operationLog := &common.OperationLog{
+		EventTime:  startAt,
+		Operation:  "logout",
+		APIPath:    c.FullPath(),
+		HTTPMethod: c.Request.Method,
+		IPAddress:  c.ClientIP(),
+	}
+	defer func() {
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
+		clickhouseDriver := clickhouse.GetDriver()
+		err := clickhouseDriver.SaveOperationLog(operationLog)
+		if err != nil {
+			common.ResponseWithCodeData(c, common.CodeServerError, false, err.Error())
+			return
+		}
+	}()
+
 	http.SetCookie(c.Writer, &http.Cookie{
 		Name:     oauthAuthCookie,
 		Value:    "",
@@ -237,8 +326,11 @@ func (h *UserHandler) Logout(c *gin.Context) {
 	// Same as AuthMiddleware@auth.go
 	token := c.GetHeader("Authorization")
 	if token == "" {
-		common.ResponseWithHttpCodeData(c, http.StatusUnauthorized, 401, nil, "Missing Authorization header")
+		common.ResponseWithHttpCodeData(c, http.StatusUnauthorized, common.CodeUnauthorized, nil, "Missing Authorization header")
 		c.Abort()
+		operationLog.ErrorCode = uint16(common.CodeUnauthorized)
+		operationLog.Message = "Missing Authorization header"
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
 
@@ -247,13 +339,20 @@ func (h *UserHandler) Logout(c *gin.Context) {
 	if err != nil {
 		common.ResponseWithHttpCodeData(c, http.StatusUnauthorized, code, nil, "Invalid access token")
 		c.Abort()
+		operationLog.ErrorCode = uint16(code)
+		operationLog.Message = "Invalid access token"
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}
+	operationLog.UserID = user.ID
 
 	// Logout user
 	code, err = h.userService.Logout(user)
 	if err != nil {
 		common.ResponseWithCodeData(c, code, false, err.Error())
+		operationLog.ErrorCode = uint16(code)
+		operationLog.Message = err.Error()
+		operationLog.DurationMS = time.Since(startAt).Milliseconds()
 		return
 	}