zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-26 10:23:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: merge duplicate codes (#16783)

Browse Source 
### Summary

1. merge heartbeat function.
2. introduce all environments

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-07-10 11:58:32 +08:00
committed by GitHub
parent 289ca28ce2
commit add7b9486f
67 changed files with 479 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/parser/parser/pdf_parser_common.go
View File

@@ -23,7 +23,7 @@ import (
"fmt"
"image"
"log/slog"
"os"
"ragflow/internal/common"
"sort"
"strings"
"time"
@@ -294,7 +294,7 @@ func emptyPDFResult(filename string) ParseResult {
}
func deepDocAnalyzerFromEnv() deepdoctype.DocAnalyzer {
baseURL := strings.TrimSpace(os.Getenv("DEEPDOC_URL"))
baseURL := strings.TrimSpace(common.GetEnv(common.EnvDeepDocURL))
if baseURL == "" {
return &deepdocpdf.MockDocAnalyzer{Healthy: true}
}

6
internal/parser/parser/pdf_parser_docling.go
View File

@@ -6,7 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -42,14 +42,14 @@ func parsePDFWithDocling(filename string, data []byte, parser *PDFParser) ParseR
}
serverURL := strings.TrimSpace(parser.DoclingServerURL)
if serverURL == "" {
serverURL = strings.TrimSpace(os.Getenv("DOCLING_SERVER_URL"))
serverURL = strings.TrimSpace(common.GetEnv(common.EnvDoclingServerURL))
}
if serverURL == "" {
return ParseResult{Err: fmt.Errorf("parser: Docling requires docling_server_url or DOCLING_SERVER_URL")}
}
apiKey := strings.TrimSpace(parser.DoclingAPIKey)
if apiKey == "" {
apiKey = strings.TrimSpace(os.Getenv("DOCLING_API_KEY"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvDoclingApiKey))
}
baseURL := strings.TrimRight(serverURL, "/")

8
internal/parser/parser/pdf_parser_mineru.go
View File

@@ -2,7 +2,7 @@ package parser
import (
"fmt"
"os"
"ragflow/internal/common"
"strings"
"time"
@@ -18,18 +18,18 @@ func parsePDFWithMinerU(filename string, data []byte, parser *PDFParser) ParseRe
}
apiServer := strings.TrimSpace(parser.MinerUAPIServer)
if apiServer == "" {
apiServer = strings.TrimSpace(os.Getenv("MINERU_APISERVER"))
apiServer = strings.TrimSpace(common.GetEnv(common.EnvMineruApiServer))
}
if apiServer == "" {
return ParseResult{Err: fmt.Errorf("parser: MinerU requires mineru_apiserver or MINERU_APISERVER")}
}
apiKey := parser.MinerUAPIKey
if strings.TrimSpace(apiKey) == "" {
apiKey = strings.TrimSpace(os.Getenv("MINERU_API_KEY"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruApiKey))
}
backend := strings.TrimSpace(parser.MinerUBackend)
if backend == "" {
backend = strings.TrimSpace(os.Getenv("MINERU_BACKEND"))
backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend))
}
if backend == "" {
backend = "pipeline"

12
internal/parser/parser/pdf_parser_opendataloader.go
View File

@@ -7,7 +7,7 @@ import (
"io"
"mime/multipart"
"net/http"
"os"
"ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -19,14 +19,14 @@ func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser)
}
baseURL := strings.TrimSpace(parser.OpenDataLoaderAPIServer)
if baseURL == "" {
baseURL = strings.TrimSpace(os.Getenv("OPENDATALOADER_APISERVER"))
baseURL = strings.TrimSpace(common.GetEnv(common.EnvOpenDataLoaderApiServer))
}
if baseURL == "" {
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader requires opendataloader_apiserver or OPENDATALOADER_APISERVER")}
}
apiKey := strings.TrimSpace(parser.OpenDataLoaderAPIKey)
if apiKey == "" {
apiKey = strings.TrimSpace(os.Getenv("OPENDATALOADER_API_KEY"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvOpenDataLoaderApiKey))
}
bodyReader, contentType, err := openDataLoaderMultipart(filename, data, parser)
@@ -57,7 +57,7 @@ func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser)
JSONDoc any `json:"json_doc"`
MDText string `json:"md_text"`
}
if err := json.Unmarshal(raw, &payload); err != nil {
if err = json.Unmarshal(raw, &payload); err != nil {
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader decode: %w", err)}
}
if payload.JSONDoc != nil {
@@ -79,7 +79,7 @@ func openDataLoaderMultipart(filename string, data []byte, parser *PDFParser) (i
if err != nil {
return nil, "", fmt.Errorf("parser: OpenDataLoader create form file: %w", err)
}
if _, err := part.Write(data); err != nil {
if _, err = part.Write(data); err != nil {
return nil, "", fmt.Errorf("parser: OpenDataLoader write PDF: %w", err)
}
if parser.OpenDataLoaderHybrid != "" {
@@ -95,7 +95,7 @@ func openDataLoaderMultipart(filename string, data []byte, parser *PDFParser) (i
_ = writer.WriteField("sanitize", "false")
}
}
if err := writer.Close(); err != nil {
if err = writer.Close(); err != nil {
return nil, "", fmt.Errorf("parser: OpenDataLoader finalize form: %w", err)
}
return strings.NewReader(body.String()), writer.FormDataContentType(), nil

10
internal/parser/parser/pdf_parser_paddleocr.go
View File

@@ -2,7 +2,7 @@ package parser
import (
"fmt"
"os"
"ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -14,21 +14,21 @@ func parsePDFWithPaddleOCR(filename string, data []byte, parser *PDFParser) Pars
}
baseURL := strings.TrimSpace(parser.PaddleOCRBaseURL)
if baseURL == "" {
baseURL = strings.TrimSpace(os.Getenv("PADDLEOCR_BASE_URL"))
baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRBaseUrl))
}
if baseURL == "" {
baseURL = strings.TrimSpace(os.Getenv("PADDLEOCR_API_URL"))
baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRApiURL))
}
if baseURL == "" {
return ParseResult{Err: fmt.Errorf("parser: PaddleOCR requires paddleocr_base_url or PADDLEOCR_BASE_URL")}
}
apiKey := parser.PaddleOCRAPIKey
if strings.TrimSpace(apiKey) == "" {
apiKey = strings.TrimSpace(os.Getenv("PADDLEOCR_ACCESS_TOKEN"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAccessToken))
}
algorithm := strings.TrimSpace(parser.PaddleOCRAlgorithm)
if algorithm == "" {
algorithm = strings.TrimSpace(os.Getenv("PADDLEOCR_ALGORITHM"))
algorithm = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAlgorithm))
}
if algorithm == "" {
algorithm = "PaddleOCR-VL"

38
internal/parser/parser/pdf_parser_somark.go
View File

@@ -9,7 +9,7 @@ import (
"mime/multipart"
"net/http"
"net/url"
"os"
"ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -21,14 +21,14 @@ func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseRe
}
baseURL := strings.TrimSpace(parser.SoMarkBaseURL)
if baseURL == "" {
baseURL = strings.TrimSpace(os.Getenv("SOMARK_BASE_URL"))
baseURL = strings.TrimSpace(common.GetEnv(common.EnvSOMarkBaseUrl))
}
if baseURL == "" {
return ParseResult{Err: fmt.Errorf("parser: SoMark requires somark_base_url or SOMARK_BASE_URL")}
}
apiKey := parser.SoMarkAPIKey
if strings.TrimSpace(apiKey) == "" {
apiKey = strings.TrimSpace(os.Getenv("SOMARK_API_KEY"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvSOMarkApiKey))
}
taskID, err := soMarkSubmit(strings.TrimRight(baseURL, "/"), filename, data, parser, apiKey)
if err != nil {
@@ -57,26 +57,26 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiK
}
_ = writer.WriteField("output_formats", "json")
elementFormats, _ := json.Marshal(map[string]any{
"image": envOrDefault("SOMARK_IMAGE_FORMAT", parser.SoMarkImageFormat, "url"),
"formula": envOrDefault("SOMARK_FORMULA_FORMAT", parser.SoMarkFormulaFormat, "latex"),
"table": envOrDefault("SOMARK_TABLE_FORMAT", parser.SoMarkTableFormat, "html"),
"cs": envOrDefault("SOMARK_CS_FORMAT", parser.SoMarkCSFormat, "image"),
"image": envOrDefault(common.EnvSOMarkImageFormat, parser.SoMarkImageFormat, "url"),
"formula": envOrDefault(common.EnvSOMarkFormulaFormat, parser.SoMarkFormulaFormat, "latex"),
"table": envOrDefault(common.EnvSOMarkTableFormat, parser.SoMarkTableFormat, "html"),
"cs": envOrDefault(common.EnvSOMarkCSFormat, parser.SoMarkCSFormat, "image"),
})
featureConfig, _ := json.Marshal(map[string]any{
"enable_text_cross_page": envOrBool("SOMARK_ENABLE_TEXT_CROSS_PAGE", parser.SoMarkEnableTextCrossPage),
"enable_table_cross_page": envOrBool("SOMARK_ENABLE_TABLE_CROSS_PAGE", parser.SoMarkEnableTableCrossPage),
"enable_title_level_recognition": envOrBool("SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION", parser.SoMarkEnableTitleLevelRecognition),
"enable_inline_image": envOrBool("SOMARK_ENABLE_INLINE_IMAGE", parser.SoMarkEnableInlineImage),
"enable_table_image": envOrBool("SOMARK_ENABLE_TABLE_IMAGE", parser.SoMarkEnableTableImage),
"enable_image_understanding": envOrBool("SOMARK_ENABLE_IMAGE_UNDERSTANDING", parser.SoMarkEnableImageUnderstanding),
"keep_header_footer": envOrBool("SOMARK_KEEP_HEADER_FOOTER", parser.SoMarkKeepHeaderFooter),
"enable_text_cross_page": envOrBool(common.EnvSOMarkEnableTextCrossPage, parser.SoMarkEnableTextCrossPage),
"enable_table_cross_page": envOrBool(common.EnvSOMarkEnableTableCrossPage, parser.SoMarkEnableTableCrossPage),
"enable_title_level_recognition": envOrBool(common.EnvSOMarkEnableTitleLevelRecognition, parser.SoMarkEnableTitleLevelRecognition),
"enable_inline_image": envOrBool(common.EnvSOMarkEnableInlineImage, parser.SoMarkEnableInlineImage),
"enable_table_image": envOrBool(common.EnvSOMarkEnableTableImage, parser.SoMarkEnableTableImage),
"enable_image_understanding": envOrBool(common.EnvSOMarkEnableImageUnderstanding, parser.SoMarkEnableImageUnderstanding),
"keep_header_footer": envOrBool(common.EnvSOMarkKeepHeaderFooter, parser.SoMarkKeepHeaderFooter),
})
_ = writer.WriteField("element_formats", string(elementFormats))
_ = writer.WriteField("feature_config", string(featureConfig))
if apiKey != "" {
_ = writer.WriteField("api_key", apiKey)
}
if err := writer.Close(); err != nil {
if err = writer.Close(); err != nil {
return "", fmt.Errorf("parser: SoMark finalize form: %w", err)
}
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, baseURL+"/parse/async", &body)
@@ -100,7 +100,7 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiK
TaskID string `json:"task_id"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &payload); err != nil {
if err = json.Unmarshal(raw, &payload); err != nil {
return "", fmt.Errorf("parser: SoMark decode submit: %w", err)
}
if payload.Code != 0 {
@@ -136,7 +136,7 @@ func soMarkPoll(baseURL, taskID, apiKey string) (map[string]any, error) {
Message string `json:"message"`
Data map[string]any `json:"data"`
}
if err := json.Unmarshal(raw, &payload); err != nil {
if err = json.Unmarshal(raw, &payload); err != nil {
return nil, fmt.Errorf("parser: SoMark decode poll: %w", err)
}
if payload.Code != 0 {
@@ -225,14 +225,14 @@ func envOrDefault(envKey, configured, fallback string) string {
if configured != "" {
return configured
}
if raw := strings.TrimSpace(os.Getenv(envKey)); raw != "" {
if raw := strings.TrimSpace(common.GetEnv(envKey)); raw != "" {
return raw
}
return fallback
}
func envOrBool(envKey string, configured bool) bool {
if raw := strings.TrimSpace(os.Getenv(envKey)); raw != "" {
if raw := strings.TrimSpace(common.GetEnv(envKey)); raw != "" {
switch strings.ToLower(raw) {
case "1", "true", "yes", "on":
return true

6
internal/parser/parser/pdf_parser_tcadp.go
View File

@@ -9,7 +9,7 @@ import (
"fmt"
"io"
"net/http"
"os"
"ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -21,14 +21,14 @@ func parsePDFWithTCADP(filename string, data []byte, parser *PDFParser) ParseRes
}
baseURL := strings.TrimSpace(parser.TCADPAPIServer)
if baseURL == "" {
baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER"))
baseURL = strings.TrimSpace(common.GetEnv(common.EnvTCADPApiServerURL))
}
if baseURL == "" {
return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}
}
apiKey := strings.TrimSpace(parser.TCADPAPIKey)
if apiKey == "" {
apiKey = strings.TrimSpace(os.Getenv("TCADP_API_KEY"))
apiKey = strings.TrimSpace(common.GetEnv(common.EnvTCADPApiKey))
}
requestBody := map[string]any{
"file_type": "PDF",