mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-07 12:00:44 +08:00
feat(go-api): implement connector (data source) management endpoints (#15274)
## Summary Ports the connector (data source) management endpoints that power `web/src/pages/user-setting/data-source/` from Python (`api/apps/restful_apis/connector_api.py`) to Go. Previously only `GET /connectors` (list) was implemented in Go; this adds the rest of the lifecycle. Closes #15273 (subtask of #15240). ## Endpoints implemented All under base path `/api/v1` (mirrors the Python routes): | Method | Path | Description | |--------|------|-------------| | POST | `/connectors/{connector_id}/test` | Validate stored credentials | `GET /connectors` (list) was already present and is unchanged. --------- Co-authored-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
@@ -17,29 +17,31 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/entity"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/service"
|
||||
)
|
||||
|
||||
type connectorService interface {
|
||||
type connectorServiceIface interface {
|
||||
ListConnectors(userID string) (*service.ListConnectorsResponse, error)
|
||||
CreateConnector(userID string, req *service.CreateConnectorRequest) (*entity.Connector, error)
|
||||
GetConnector(connectorID, userID string) (*entity.Connector, common.ErrorCode, error)
|
||||
ListLog(connectorID, userID string, page, pageSize int) ([]*entity.ConnectorSyncLog, int64, common.ErrorCode, error)
|
||||
DeleteConnector(connectorID, userID string) (bool, common.ErrorCode, error)
|
||||
RebuildConnector(connectorID, userID, kbID string) (bool, common.ErrorCode, error)
|
||||
TestConnector(connectorID, userID string) error
|
||||
}
|
||||
|
||||
// ConnectorHandler connector handler
|
||||
type ConnectorHandler struct {
|
||||
connectorService connectorService
|
||||
connectorService connectorServiceIface
|
||||
userService *service.UserService
|
||||
}
|
||||
|
||||
@@ -84,6 +86,25 @@ func (h *ConnectorHandler) ListConnectors(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// connectorErrorResponse maps service sentinel errors to the response codes used
|
||||
// by the Python connector_api, and writes the JSON response. It returns true when
|
||||
// the error was handled.
|
||||
func connectorErrorResponse(c *gin.Context, err error) bool {
|
||||
switch {
|
||||
case err == nil:
|
||||
return false
|
||||
case errors.Is(err, service.ErrConnectorNoAuth):
|
||||
c.JSON(http.StatusOK, gin.H{"code": common.CodeAuthenticationError, "data": false, "message": "No authorization."})
|
||||
case errors.Is(err, service.ErrConnectorNotFound):
|
||||
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": nil, "message": "Can't find this Connector!"})
|
||||
case errors.Is(err, service.ErrConnectorTestUnsupported):
|
||||
c.JSON(http.StatusOK, gin.H{"code": common.CodeArgumentError, "data": false, "message": err.Error()})
|
||||
default:
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"code": common.CodeServerError, "data": nil, "message": err.Error()})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// GetConnector get connector
|
||||
// @Summary Get Connector
|
||||
// @Description Get connector details for the current user
|
||||
@@ -227,6 +248,43 @@ func (h *ConnectorHandler) CreateConnector(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestConnector validates an accessible connector's stored credentials.
|
||||
// @Summary Test Connector
|
||||
// @Description Validate connector credentials / connection (equivalent to Python's test_connector)
|
||||
// @Tags connector
|
||||
// @Produce json
|
||||
// @Param connector_id path string true "connector ID"
|
||||
// @Router /api/v1/connectors/{connector_id}/test [post]
|
||||
func (h *ConnectorHandler) TestConnector(c *gin.Context) {
|
||||
user, errorCode, errorMessage := GetUser(c)
|
||||
if errorCode != common.CodeSuccess {
|
||||
jsonError(c, errorCode, errorMessage)
|
||||
return
|
||||
}
|
||||
|
||||
connectorID := c.Param("connector_id")
|
||||
if connectorID == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"code": common.CodeBadRequest, "data": nil, "message": "connector_id is required"})
|
||||
return
|
||||
}
|
||||
|
||||
err := h.connectorService.TestConnector(connectorID, user.ID)
|
||||
if errors.Is(err, service.ErrConnectorTestUnsupported) {
|
||||
connectorErrorResponse(c, err)
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, service.ErrConnectorNoAuth) && !errors.Is(err, service.ErrConnectorNotFound) {
|
||||
// Validation failure (e.g. missing credentials): mirror Python's DATA_ERROR with data=false.
|
||||
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
|
||||
return
|
||||
}
|
||||
if connectorErrorResponse(c, err) {
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"code": common.CodeSuccess, "data": true, "message": "success"})
|
||||
}
|
||||
|
||||
// DeleteConnector delete connector
|
||||
// @Description Detele Connector
|
||||
// @Tags connector
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/service"
|
||||
)
|
||||
@@ -25,11 +24,18 @@ type fakeConnectorService struct {
|
||||
}
|
||||
|
||||
func (s fakeConnectorService) ListConnectors(string) (*service.ListConnectorsResponse, error) {
|
||||
return &service.ListConnectorsResponse{Connectors: []*dao.ConnectorListItem{}}, nil
|
||||
return &service.ListConnectorsResponse{}, nil
|
||||
}
|
||||
|
||||
func (s fakeConnectorService) TestConnector(string, string) error {
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s fakeConnectorService) CreateConnector(string, *service.CreateConnectorRequest) (*entity.Connector, error) {
|
||||
return s.connector, s.err
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
return s.connector, nil
|
||||
}
|
||||
|
||||
func (s fakeConnectorService) GetConnector(string, string) (*entity.Connector, common.ErrorCode, error) {
|
||||
@@ -60,48 +66,52 @@ func (s fakeConnectorService) RebuildConnector(string, string, string) (bool, co
|
||||
return true, common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
func TestConnectorHandlerGetConnector(t *testing.T) {
|
||||
func TestConnectorHandlerTestConnector(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
service fakeConnectorService
|
||||
err error
|
||||
wantCode common.ErrorCode
|
||||
wantID string
|
||||
wantMsg string
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
service: fakeConnectorService{connector: &entity.Connector{
|
||||
ID: "connector-1",
|
||||
TenantID: "tenant-1",
|
||||
Name: "Docs",
|
||||
Source: "google_drive",
|
||||
Status: "unstart",
|
||||
Config: entity.JSONMap{"folder": "docs"},
|
||||
}},
|
||||
name: "success",
|
||||
err: nil,
|
||||
wantCode: common.CodeSuccess,
|
||||
wantID: "connector-1",
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
err: service.ErrConnectorNotFound,
|
||||
wantCode: common.CodeDataError,
|
||||
},
|
||||
{
|
||||
name: "unauthorized",
|
||||
service: fakeConnectorService{code: common.CodeAuthenticationError, err: fmt.Errorf("No authorization.")},
|
||||
err: service.ErrConnectorNoAuth,
|
||||
wantCode: common.CodeAuthenticationError,
|
||||
wantMsg: "No authorization.",
|
||||
},
|
||||
{
|
||||
name: "unsupported source",
|
||||
err: service.ErrConnectorTestUnsupported,
|
||||
wantCode: common.CodeArgumentError,
|
||||
},
|
||||
{
|
||||
name: "validation failure",
|
||||
err: fmt.Errorf("connector credentials are missing"),
|
||||
wantCode: common.CodeDataError,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
h := &ConnectorHandler{connectorService: tt.service}
|
||||
h := &ConnectorHandler{connectorService: fakeConnectorService{err: tt.err}}
|
||||
router := gin.New()
|
||||
router.GET("/api/v1/connectors/:connector_id", func(c *gin.Context) {
|
||||
router.POST("/api/v1/connectors/:connector_id/test", func(c *gin.Context) {
|
||||
c.Set("user", &entity.User{ID: "tenant-1"})
|
||||
h.GetConnector(c)
|
||||
h.TestConnector(c)
|
||||
})
|
||||
|
||||
resp := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/connectors/connector-1", nil)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/connectors/connector-1/test", nil)
|
||||
router.ServeHTTP(resp, req)
|
||||
if resp.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", resp.Code, resp.Body.String())
|
||||
@@ -112,13 +122,7 @@ func TestConnectorHandlerGetConnector(t *testing.T) {
|
||||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
if body["code"] != float64(tt.wantCode) {
|
||||
t.Fatalf("code=%v body=%v", body["code"], body)
|
||||
}
|
||||
if tt.wantMsg != "" && body["message"] != tt.wantMsg {
|
||||
t.Fatalf("message=%v", body["message"])
|
||||
}
|
||||
if tt.wantID != "" && body["data"].(map[string]interface{})["id"] != tt.wantID {
|
||||
t.Fatalf("data=%v", body["data"])
|
||||
t.Fatalf("code=%v want=%v body=%v", body["code"], tt.wantCode, body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -322,6 +322,7 @@ func (r *Router) Setup(engine *gin.Engine) {
|
||||
connector.GET("/:connector_id/logs", r.connectorHandler.ListLogs)
|
||||
connector.DELETE("/:connector_id", r.connectorHandler.DeleteConnector)
|
||||
connector.POST("/:connector_id/rebuild", r.connectorHandler.RebuildConnector)
|
||||
connector.POST("/:connector_id/test", r.connectorHandler.TestConnector)
|
||||
}
|
||||
|
||||
system := v1.Group("/system")
|
||||
|
||||
@@ -20,12 +20,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/engine"
|
||||
"ragflow/internal/entity"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -35,17 +36,24 @@ const (
|
||||
defaultConnectorTimeout = 60 * 29
|
||||
)
|
||||
|
||||
// Sentinel errors so handlers can map to the proper response codes,
|
||||
// mirroring the Python connector_api responses.
|
||||
var (
|
||||
// ErrConnectorNotFound mirrors Python's "Can't find this Connector!".
|
||||
ErrConnectorNotFound = errors.New("can't find this Connector")
|
||||
// ErrConnectorNoAuth mirrors Python's "No authorization." denial.
|
||||
ErrConnectorNoAuth = errors.New("no authorization")
|
||||
// ErrConnectorTestUnsupported is returned for connector sources whose
|
||||
// validation path is not yet ported to Go.
|
||||
ErrConnectorTestUnsupported = errors.New("test endpoint currently supports only REST API connectors")
|
||||
)
|
||||
|
||||
// ConnectorService connector service
|
||||
type ConnectorService struct {
|
||||
connectorDAO *dao.ConnectorDAO
|
||||
userTenantDAO *dao.UserTenantDAO
|
||||
}
|
||||
|
||||
func (s *ConnectorService) Rebuild() {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// NewConnectorService create connector service
|
||||
func NewConnectorService() *ConnectorService {
|
||||
return &ConnectorService{
|
||||
@@ -190,6 +198,69 @@ func (s *ConnectorService) ListConnectors(userID string) (*ListConnectorsRespons
|
||||
}, nil
|
||||
}
|
||||
|
||||
// accessible reports whether the user can access the connector's tenant.
|
||||
// Mirrors Python's ConnectorService.accessible: owner access plus joined tenants.
|
||||
func (s *ConnectorService) accessible(connectorID, userID string) (bool, error) {
|
||||
conn, err := s.connectorDAO.GetByID(connectorID)
|
||||
if err != nil {
|
||||
return false, ErrConnectorNotFound
|
||||
}
|
||||
|
||||
if conn.TenantID == userID {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
tenantIDs, err := s.userTenantDAO.GetTenantIDsByUserID(userID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, tid := range tenantIDs {
|
||||
if tid == conn.TenantID {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// TestConnector validates a connector's stored configuration.
|
||||
// Equivalent to Python's test_connector. Per-connector credential validation
|
||||
// lives in the Python common.data_source package and is not yet available in
|
||||
// Go; for now this verifies access, that the connector exists, that the source
|
||||
// is REST_API (the only source Python currently tests), and that credentials
|
||||
// are present in the stored config. It returns ErrConnectorTestUnsupported for
|
||||
// other sources.
|
||||
func (s *ConnectorService) TestConnector(connectorID, userID string) error {
|
||||
ok, err := s.accessible(connectorID, userID)
|
||||
if err != nil && errors.Is(err, ErrConnectorNotFound) {
|
||||
return ErrConnectorNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrConnectorNoAuth
|
||||
}
|
||||
|
||||
conn, err := s.connectorDAO.GetByID(connectorID)
|
||||
if err != nil {
|
||||
return ErrConnectorNotFound
|
||||
}
|
||||
|
||||
if conn.Source != "rest_api" {
|
||||
return ErrConnectorTestUnsupported
|
||||
}
|
||||
|
||||
config := conn.Config
|
||||
if config == nil {
|
||||
return fmt.Errorf("connector configuration is missing")
|
||||
}
|
||||
creds, ok := config["credentials"].(map[string]interface{})
|
||||
if !ok || len(creds) == 0 {
|
||||
return fmt.Errorf("connector credentials are missing")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ConnectorService) DeleteConnector(connectorID, userID string) (bool, common.ErrorCode, error) {
|
||||
if connectorID == "" {
|
||||
return false, common.CodeDataError, fmt.Errorf("connector_id is required")
|
||||
|
||||
Reference in New Issue
Block a user