zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-09-08 10:14:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: add cli command, list dataset documents (#14948)

Browse Source 
### What problem does this PR solve?
```
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
| created_at          | id                               | meta_fields | name            | size    | status | type |
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
| 2026-05-08 19:35:08 | f6aa38bb4ad111f1ba6338a74640adcc | map[]       |  abc.pdf        | 3387987 | 1      | pdf  |
+---------------------+----------------------------------+-------------+-----------------+---------+--------+------+
```

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-05-15 14:00:45 +08:00
committed by GitHub
parent 1a25191b13
commit 335dd5a263
12 changed files with 225 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/service/datasets.go → internal/service/dataset.go
+16 -11
View File
@@ -57,8 +57,8 @@ var (
datasetChunkMethodErrorMessage = "Input should be 'naive', 'book', 'email', 'laws', 'manual', 'one', 'paper', 'picture', 'presentation', 'qa', 'resume', 'table' or 'tag'"
)
// DatasetsService implements the RESTful dataset APIs from dataset_api.py.
type DatasetsService struct {
// DatasetService implements the RESTful dataset APIs from dataset_api.py.
type DatasetService struct {
kbDAO *dao.KnowledgebaseDAO
documentDAO *dao.DocumentDAO
connectorDAO *dao.ConnectorDAO
@@ -66,9 +66,9 @@ type DatasetsService struct {
tenantLLMDAO *dao.TenantLLMDAO
}
// NewDatasetsService creates a new datasets service.
func NewDatasetsService() *DatasetsService {
return &DatasetsService{
// NewDatasetService creates a new datasets service.
func NewDatasetService() *DatasetService {
return &DatasetService{
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
connectorDAO: dao.NewConnectorDAO(),
@@ -108,7 +108,7 @@ type CreateDatasetRequest struct {
}
// ListDatasets lists datasets with pagination and filtering.
func (s *DatasetsService) ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
func (s *DatasetService) ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
id = strings.TrimSpace(id)
if id != "" {
normalizedID, err := normalizeDatasetUUID1(id)
@@ -190,7 +190,7 @@ func (s *DatasetsService) ListDatasets(id, name string, page, pageSize int, orde
}
// CreateDataset creates a new dataset.
func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
func (s *DatasetService) CreateDataset(req *CreateDatasetRequest, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
if !isValidString(req.Name) {
return nil, common.CodeDataError, errors.New("Dataset name must be string.")
}
@@ -441,7 +441,7 @@ func (s *DatasetsService) CreateDataset(req *CreateDatasetRequest, tenantID stri
}
// DeleteDatasets deletes multiple datasets.
func (s *DatasetsService) DeleteDatasets(ids []string, deleteAll bool, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
func (s *DatasetService) DeleteDatasets(ids []string, deleteAll bool, tenantID string) (map[string]interface{}, common.ErrorCode, error) {
normalizedIDs := make([]string, 0, len(ids))
seenIDs := make(map[string]struct{}, len(ids))
@@ -521,7 +521,7 @@ func (s *DatasetsService) DeleteDatasets(ids []string, deleteAll bool, tenantID
}
// GetDataset gets a single dataset with its size and linked connectors.
func (s *DatasetsService) GetDataset(datasetID, userID string) (map[string]interface{}, common.ErrorCode, error) {
func (s *DatasetService) GetDataset(datasetID, userID string) (map[string]interface{}, common.ErrorCode, error) {
datasetID = strings.TrimSpace(datasetID)
if datasetID == "" {
return nil, common.CodeDataError, errors.New("Lack of \"Dataset ID\"")
@@ -559,7 +559,12 @@ func (s *DatasetsService) GetDataset(datasetID, userID string) (map[string]inter
return data, common.CodeSuccess, nil
}
func (s *DatasetsService) deleteDataset(tenantID string, kb *entity.Knowledgebase) error {
// Accessible checks if a knowledge base is accessible by a user
func (s *DatasetService) Accessible(kbID, userID string) bool {
return s.kbDAO.Accessible(kbID, userID)
}
func (s *DatasetService) deleteDataset(tenantID string, kb *entity.Knowledgebase) error {
return dao.DB.Transaction(func(tx *gorm.DB) error {
var documents []entity.Document
if err := tx.Where("kb_id = ?", kb.ID).Find(&documents).Error; err != nil {
@@ -706,7 +711,7 @@ func normalizeDatasetUUID1(id string) (string, error) {
return strings.ReplaceAll(parsedUUID.String(), "-", ""), nil
}
func (s *DatasetsService) verifyEmbeddingAvailability(embdID string, tenantID string) (bool, string) {
func (s *DatasetService) verifyEmbeddingAvailability(embdID string, tenantID string) (bool, string) {
modelName, _, provider, err := parseModelName(embdID)
if err != nil {
return false, "Embedding model identifier must follow <model_name>@<provider> format"
internal/service/document.go
+9 -2
View File
@@ -175,8 +175,8 @@ func (s *DocumentService) ListDocuments(page, pageSize int) ([]*DocumentResponse
return responses, total, nil
}
// ListDocumentsByKBID list documents by knowledge base ID
func (s *DocumentService) ListDocumentsByKBID(kbID string, page, pageSize int) ([]*DocumentResponse, int64, error) {
// ListDocumentsByDatasetID list documents by knowledge base ID
func (s *DocumentService) ListDocumentsByDatasetID(kbID string, page, pageSize int) ([]*DocumentResponse, int64, error) {
offset := (page - 1) * pageSize
documents, total, err := s.documentDAO.ListByKBID(kbID, offset, pageSize)
if err != nil {
@@ -207,6 +207,13 @@ func (s *DocumentService) GetDocumentsByAuthorID(authorID, page, pageSize int) (
return responses, total, nil
}
func (s *DocumentService) ParseDocuments(datasetID, userID string, docIDs []string) error {
// create document parse id
// save to task table
// send to message queue
return nil
}
// toResponse convert model.Document to DocumentResponse
func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
createdAt := ""