zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-03 17:21:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: implement provider: MinerU (#14990)

Browse Source 
### What problem does this PR solve?

Implement MinerU Provider

**The following functionalities are now supported:**

**MinerU**
----
- [x] Parse file
- [x] Show task
- [ ] ~~List tasks~~

**Verified examples from the CLI:**
```plaintext
RAGFlow(user)> parse with 'vlm@test@mineru' file 'https://arxiv.org/pdf/2505.09358'
+--------------------------------------+
| task_id                              |
+--------------------------------------+
| 142ac8ea-d9d0-4a68-a2d1-d3af67635dc9 |
+--------------------------------------+

RAGFlow(user)> show 'test@mineru' task '142ac8ea-d9d0-4a68-a2d1-d3af67635dc9'
+--------------------------------------------+-------+
| content                                    | index |
+--------------------------------------------+-------+
| Task is running... Progress: 17 / 18 pages | 0     |
+--------------------------------------------+-------+

RAGFlow(user)> show 'test@mineru' task '142ac8ea-d9d0-4a68-a2d1-d3af67635dc9'
+--------------------------------------------------------------------------------------------+-------+
| content                                                                                    | index |
+--------------------------------------------------------------------------------------------+-------+
| https://cdn-mineru.openxlab.org.cn/pdf/2026-05-18/142ac8ea-d9d0-4a68-a2d1-d3af67635dc9.zip | 0     |
+--------------------------------------------------------------------------------------------+-------+

```


### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring
This commit is contained in:
Haruko386
2026-05-19 10:49:33 +08:00
committed by GitHub
parent 525a87be0f
commit db9e782747
8 changed files with 310 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
conf/models/baidu.json
View File

@@ -1,5 +1,5 @@
{
"Name": "Baidu",
"name": "Baidu",
"url": {
"default": "https://qianfan.baidubce.com/v2"
},

25
conf/models/mineru.json Normal file
View File

@@ -0,0 +1,25 @@
{
"name": "MinerU",
"url": {
"default": "https://mineru.net"
},
"url_suffix": {
"doc_parse": "v4/extract/task",
"tasks": ""
},
"class": "mineru",
"models": [
{
"name": "vlm",
"model_types": [
"doc_parse"
]
},
{
"name": "MinerU-HTML",
"model_types": [
"doc_parse"
]
}
]
}

4
conf/models/xunfei.json
View File

@@ -1,10 +1,10 @@
{
"name": "XunFei",
"url": {
"default": "https://"
"default": "https://spark-api-open.xf-yun.com"
},
"url_suffix": {
"chat": "spark-api-open.xf-yun.com/v2/chat/completions"
"chat": "v2/chat/completions"
},
"class": "xunfei",
"models": [

15
internal/cli/user_command.go
View File

@@ -2381,11 +2381,16 @@ func (c *RAGFlowClient) ParseFileUserCommand(cmd *Command) (ResponseIf, error) {
filename, ok = cmd.Params["file"].(string)
if ok {
// read file and convert to base64
var err error
fileContent, err = os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
// For online file
if strings.HasPrefix(filename, "http://") || strings.HasPrefix(filename, "https://") {
fileURL = filename
} else {
// read file and convert to base64
var err error
fileContent, err = os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
}
} else {
fileURL, ok = cmd.Params["url"].(string)

2
internal/entity/models/factory.go
View File

@@ -101,6 +101,8 @@ func (f *ModelFactory) CreateModelDriver(providerName string, baseURL map[string
return NewXunFeiModel(baseURL, urlSuffix), nil
case "deepinfra":
return NewDeepInfraModel(baseURL, urlSuffix), nil
case "mineru":
return NewMinerUModel(baseURL, urlSuffix), nil
default:
return NewDummyModel(baseURL, urlSuffix), nil
}

8
internal/entity/models/gitee.go
View File

@@ -874,7 +874,7 @@ func (g *GiteeModel) getParseFile(baseURL *string, apiKey, taskID *string, timeO
func (g *GiteeModel) ListModels(apiConfig *APIConfig) ([]string, error) {
var region = "default"
if apiConfig.Region != nil {
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -931,7 +931,7 @@ func (g *GiteeModel) ListModels(apiConfig *APIConfig) ([]string, error) {
func (g *GiteeModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
var region = "default"
if apiConfig.Region != nil {
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -1058,7 +1058,7 @@ type giteeTaskURLs struct {
func (g *GiteeModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
var region = "default"
if apiConfig.Region != nil {
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
@@ -1113,7 +1113,7 @@ func (g *GiteeModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
func (g *GiteeModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
var region = "default"
if apiConfig.Region != nil {
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}

265
internal/entity/models/mineru.go Normal file
View File

@@ -0,0 +1,265 @@
package models
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
type MinerUModel struct {
BaseURL map[string]string
URLSuffix URLSuffix
httpClient *http.Client
}
func NewMinerUModel(baseURL map[string]string, urlSuffix URLSuffix) *MinerUModel {
return &MinerUModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
httpClient: &http.Client{
Timeout: time.Second * 120,
Transport: &http.Transport{
MaxIdleConns: 10,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: time.Second * 90,
DisableCompression: false,
},
},
}
}
func (m *MinerUModel) NewInstance(baseURL map[string]string) ModelDriver {
return &MinerUModel{
BaseURL: baseURL,
URLSuffix: m.URLSuffix,
httpClient: &http.Client{
Timeout: time.Second * 120,
Transport: &http.Transport{
MaxIdleConns: 10,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: time.Second * 90,
DisableCompression: false,
},
},
}
}
func (m *MinerUModel) Name() string {
return "mineru"
}
func (m *MinerUModel) ChatWithMessages(modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig) (*ChatResponse, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) ChatStreamlyWithSender(modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) Embed(modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig) ([]EmbeddingData, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) Rerank(modelName *string, query string, documents []string, apiConfig *APIConfig, rerankConfig *RerankConfig) (*RerankResponse, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) TranscribeAudio(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig) (*ASRResponse, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) AudioSpeech(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*TTSResponse, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, sender func(*string, *string) error) error {
return fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) OCRFile(modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig) (*OCRFileResponse, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) ListModels(apiConfig *APIConfig) ([]string, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) Balance(apiConfig *APIConfig) (map[string]interface{}, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
func (m *MinerUModel) CheckConnection(apiConfig *APIConfig) error {
return fmt.Errorf("%s no such method", m.Name())
}
type mineruTaskSubmitResponse struct {
Code int `json:"code"`
Data struct {
TaskID string `json:"task_id"`
} `json:"data"`
Msg string `json:"msg"`
TraceID string `json:"trace_id"`
}
func (m *MinerUModel) ParseFile(modelName *string, content []byte, documentURL *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig) (*ParseFileResponse, error) {
if documentURL == nil || *documentURL == "" {
return nil, fmt.Errorf("MinerU API requires a valid public document URL; direct file upload is not supported")
}
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is required")
}
var region = "default"
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
apiURL := fmt.Sprintf("%s/api/%s", m.BaseURL[region], m.URLSuffix.DocumentParse)
reqBody := map[string]interface{}{
"url": *documentURL,
}
if modelName != nil && *modelName != "" {
reqBody["model_version"] = *modelName
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
resp, err := m.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("MinerU API failed with status %d: %s", resp.StatusCode, string(body))
}
var taskResp mineruTaskSubmitResponse
if err := json.Unmarshal(body, &taskResp); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
if taskResp.Code != 0 {
return nil, fmt.Errorf("MinerU task creation failed (code %d): %s", taskResp.Code, taskResp.Msg)
}
return &ParseFileResponse{
TaskID: taskResp.Data.TaskID,
}, nil
}
func (m *MinerUModel) ListTasks(apiConfig *APIConfig) ([]ListTaskStatus, error) {
return nil, fmt.Errorf("%s no such method", m.Name())
}
type mineruTaskQueryResponse struct {
Code int `json:"code"`
Data struct {
TaskID string `json:"task_id"`
State string `json:"state"` // including: pending, running, done, failed, converting
FullZipURL string `json:"full_zip_url"`
ErrMsg string `json:"err_msg"`
ExtractProgress struct {
ExtractedPages int `json:"extracted_pages"`
TotalPages int `json:"total_pages"`
} `json:"extract_progress"`
} `json:"data"`
Msg string `json:"msg"`
}
func (m *MinerUModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskResponse, error) {
if apiConfig == nil || apiConfig.ApiKey == nil || *apiConfig.ApiKey == "" {
return nil, fmt.Errorf("api key is required")
}
var region = "default"
if apiConfig.Region != nil && *apiConfig.Region != "" {
region = *apiConfig.Region
}
// URL: https://mineru.net/api/v4/extract/task/{task_id}
apiURL := fmt.Sprintf("%s/api/%s/%s", m.BaseURL[region], m.URLSuffix.DocumentParse, taskID)
req, err := http.NewRequest("GET", apiURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
resp, err := m.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("MinerU query API failed with status %d: %s", resp.StatusCode, string(body))
}
var queryResp mineruTaskQueryResponse
if err := json.Unmarshal(body, &queryResp); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
if queryResp.Code != 0 {
return nil, fmt.Errorf("MinerU task query failed: %s", queryResp.Msg)
}
// failed state
if queryResp.Data.State == "failed" {
return nil, fmt.Errorf("MinerU task failed: %s", queryResp.Data.ErrMsg)
}
content := ""
if queryResp.Data.State == "done" {
content = queryResp.Data.FullZipURL
} else if queryResp.Data.State == "running" {
content = fmt.Sprintf("Task is running... Progress: %d / %d pages",
queryResp.Data.ExtractProgress.ExtractedPages,
queryResp.Data.ExtractProgress.TotalPages)
} else {
// queue or formating
content = fmt.Sprintf("Task state: %s", queryResp.Data.State)
}
return &TaskResponse{
Segments: []TaskSegment{
{
Index: 0,
Content: content,
},
},
}, nil
}

1
internal/entity/models/types.go
View File

@@ -79,6 +79,7 @@ type OCRFileResponse struct {
}
type ParseFileResponse struct {
TaskID string `json:"task_id"`
}
type ListTaskStatus struct {