mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-12 11:43:39 +08:00
Go: merge duplicate codes (#16783)
### Summary 1. merge heartbeat function. 2. introduce all environments --------- Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -44,14 +45,14 @@ func TestBatchResults(t *testing.T) {
|
||||
pdfDir := filepath.Join("testdata", "real_pdfs")
|
||||
all := listRealPDFs(t, pdfDir)
|
||||
|
||||
count := countFromEnv("BATCH_COUNT", len(all))
|
||||
if single := os.Getenv("BATCH_SINGLE"); single != "" {
|
||||
count := countFromEnv(common.EnvBatchCount, len(all))
|
||||
if single := common.GetEnv(common.EnvBatchSingle); single != "" {
|
||||
all = filterSingle(all, single, t)
|
||||
count = 1
|
||||
}
|
||||
pdfs := all[:min(count, len(all))]
|
||||
|
||||
ddClient, err := inf.NewClient(os.Getenv("DEEPDOC_URL"))
|
||||
ddClient, err := inf.NewClient(common.GetEnv(common.EnvDeepDocURL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -73,7 +74,7 @@ func TestBatchResults(t *testing.T) {
|
||||
|
||||
func setupLogger() {
|
||||
level := slog.LevelInfo
|
||||
switch os.Getenv("BATCH_LOG_LEVEL") {
|
||||
switch common.GetEnv(common.EnvBatchLogLevel) {
|
||||
case "debug":
|
||||
level = slog.LevelDebug
|
||||
case "warn":
|
||||
@@ -83,7 +84,7 @@ func setupLogger() {
|
||||
}
|
||||
|
||||
func variantFromEnv() string {
|
||||
if os.Getenv("BATCH_SKIP_OCR") == "1" {
|
||||
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
|
||||
return "noocr"
|
||||
}
|
||||
return "ocr"
|
||||
@@ -106,7 +107,7 @@ func mkOutputDirs(variant string) outputDirs {
|
||||
}
|
||||
|
||||
func countFromEnv(key string, ceiling int) int {
|
||||
if s := os.Getenv(key); s != "" {
|
||||
if s := common.GetEnv(key); s != "" {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err == nil && n > 0 && n < ceiling {
|
||||
return n
|
||||
@@ -178,7 +179,7 @@ func processPDFs(t *testing.T, pdfDir string, pdfs []string, deepDoc pdf.DocAnal
|
||||
t.Helper()
|
||||
var results []tool.BatchResult
|
||||
totalChars := 0
|
||||
skipOCR := os.Getenv("BATCH_SKIP_OCR") == "1"
|
||||
skipOCR := common.GetEnv(common.EnvBatchSkipOCR) == "1"
|
||||
|
||||
for i, name := range pdfs {
|
||||
label := fmt.Sprintf("[%d/%d] %s", i+1, len(pdfs), name)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/deepdoc/parser/pdf/tool"
|
||||
@@ -17,19 +18,19 @@ import (
|
||||
// compare the noocr variant; PY_OCR_SUFFIX to override the Python variant.
|
||||
func TestBatchCompareWithPython(t *testing.T) {
|
||||
level := slog.LevelInfo
|
||||
if os.Getenv("BATCH_LOG_LEVEL") == "debug" {
|
||||
if common.GetEnv(common.EnvBatchLogLevel) == "debug" {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
if os.Getenv("BATCH_LOG_LEVEL") == "warn" {
|
||||
if common.GetEnv(common.EnvBatchLogLevel) == "warn" {
|
||||
level = slog.LevelWarn
|
||||
}
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
|
||||
|
||||
goVariant := "ocr"
|
||||
if os.Getenv("BATCH_SKIP_OCR") == "1" {
|
||||
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
|
||||
goVariant = "noocr"
|
||||
}
|
||||
pyVariant := os.Getenv("PY_OCR_SUFFIX")
|
||||
pyVariant := common.GetEnv(common.EnvPYOCRSuffix)
|
||||
if pyVariant == "" {
|
||||
pyVariant = goVariant
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ package pdf
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"testing"
|
||||
|
||||
inf "ragflow/internal/deepdoc/parser/pdf/inference"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
// mustConnectInferenceClient returns a InferenceClient for the OSS DeepDoc service.
|
||||
func mustConnectInferenceClient(t *testing.T) *inf.Client {
|
||||
t.Helper()
|
||||
url := os.Getenv("OSSDEEPDOC_URL")
|
||||
url := common.GetEnv(common.EnvOSSDeepDocURL)
|
||||
if url == "" {
|
||||
url = "http://localhost:9390"
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
// instead of real text. This validates that detect+merge+recognize
|
||||
// produces readable English from the scan.
|
||||
func TestOCR_mergeChars_RealScanned(t *testing.T) {
|
||||
url := os.Getenv("DEEPDOC_URL")
|
||||
url := common.GetEnv(common.EnvDeepDocURL)
|
||||
if url == "" {
|
||||
t.Skip("DEEPDOC_URL not set")
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ func writeGolden(t *testing.T, path string, v any) {
|
||||
}
|
||||
|
||||
func updateGolden() bool {
|
||||
return os.Getenv("UPDATE_GOLDEN") == "1"
|
||||
return common.GetEnv(common.EnvUpdateGolden) == "1"
|
||||
}
|
||||
|
||||
// sectionsToGolden converts []pdf.Section to the snapshot format.
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestPipelineParity(t *testing.T) {
|
||||
t.Skipf("charspy/ not found: %v", err)
|
||||
}
|
||||
|
||||
filter := os.Getenv("BATCH_PARITY_FILTER")
|
||||
filter := common.GetEnv(common.EnvBatchParityFilter)
|
||||
|
||||
total, passed := 0, 0
|
||||
for _, e := range entries {
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestTableRotation_Integration(t *testing.T) {
|
||||
t.Skipf("test PDF not found: %s (run tools/generate_rotated_table_pdf.py first)", pdfPath)
|
||||
}
|
||||
|
||||
baseURL := os.Getenv("DEEPDOC_URL")
|
||||
baseURL := common.GetEnv(common.EnvDeepDocURL)
|
||||
if baseURL == "" {
|
||||
baseURL = "http://localhost:9390"
|
||||
}
|
||||
@@ -126,7 +126,7 @@ func TestTableRotation_Integration(t *testing.T) {
|
||||
// TestTableRotation_Stability runs rotation detection on a sample real PDF
|
||||
// and verifies the pipeline doesn't crash. Set BATCH_COUNT to limit.
|
||||
func TestTableRotation_Stability(t *testing.T) {
|
||||
baseURL := os.Getenv("DEEPDOC_URL")
|
||||
baseURL := common.GetEnv(common.EnvDeepDocURL)
|
||||
if baseURL == "" {
|
||||
baseURL = "http://localhost:9390"
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestDumpTextOutput(t *testing.T) {
|
||||
}
|
||||
|
||||
count := len(entries)
|
||||
if n := os.Getenv("DUMP_COUNT"); n != "" {
|
||||
if n := common.GetEnv(common.EnvDumpCount); n != "" {
|
||||
c := 0
|
||||
for _, ch := range n {
|
||||
c = c*10 + int(ch-'0')
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -191,7 +192,7 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
|
||||
}
|
||||
|
||||
// Also write CSV if BATCH_CSV env is set (backward compat).
|
||||
if csvPath := os.Getenv("BATCH_CSV"); csvPath != "" {
|
||||
if csvPath := common.GetEnv(common.EnvBatchCSV); csvPath != "" {
|
||||
if err := WriteCSV(csvPath, diffs); err != nil {
|
||||
log.Logf("CSV write error: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -15,19 +15,19 @@ import (
|
||||
// compare the noocr variant; PY_OCR_SUFFIX to override the Python variant.
|
||||
func TestBatchCompareWithPython(t *testing.T) {
|
||||
level := slog.LevelInfo
|
||||
if os.Getenv("BATCH_LOG_LEVEL") == "debug" {
|
||||
if common.GetEnv(common.EnvBatchLogLevel) == "debug" {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
if os.Getenv("BATCH_LOG_LEVEL") == "warn" {
|
||||
if common.GetEnv(common.EnvBatchLogLevel) == "warn" {
|
||||
level = slog.LevelWarn
|
||||
}
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
|
||||
|
||||
goVariant := "ocr"
|
||||
if os.Getenv("BATCH_SKIP_OCR") == "1" {
|
||||
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
|
||||
goVariant = "noocr"
|
||||
}
|
||||
pyVariant := os.Getenv("PY_OCR_SUFFIX")
|
||||
pyVariant := common.GetEnv(common.EnvPYOCRSuffix)
|
||||
if pyVariant == "" {
|
||||
pyVariant = goVariant
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@@ -26,12 +27,12 @@ func LoadConfig() Config {
|
||||
pyVariant := "ocr"
|
||||
td := filepath.Join("testdata")
|
||||
return Config{
|
||||
Count: envInt("BATCH_COUNT", 0),
|
||||
Single: os.Getenv("BATCH_SINGLE"),
|
||||
SkipOCR: os.Getenv("BATCH_SKIP_OCR") == "1",
|
||||
CompareOnly: os.Getenv("BATCH_COMPARE_ONLY") == "1",
|
||||
CompareFilter: os.Getenv("BATCH_COMPARE_FILTER"),
|
||||
CSVOutput: envStr("BATCH_COMPARE_CSV", filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))),
|
||||
Count: envInt(common.EnvBatchCount, 0),
|
||||
Single: common.GetEnv(common.EnvBatchSingle),
|
||||
SkipOCR: common.GetEnv(common.EnvBatchSkipOCR) == "1",
|
||||
CompareOnly: common.GetEnv(common.EnvBatchCompareOnly) == "1",
|
||||
CompareFilter: common.GetEnv(common.EnvBatchCompareFilter),
|
||||
CSVOutput: envStr(common.EnvBatchCompareCSV, filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))),
|
||||
GoTextDir: filepath.Join(td, "output", "go", goVariant, "text"),
|
||||
PyTextDir: filepath.Join(td, "output", "py", pyVariant, "text"),
|
||||
TablesDir: filepath.Join(td, "output", "go", goVariant, "tables"),
|
||||
@@ -40,7 +41,7 @@ func LoadConfig() Config {
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
v := common.GetEnv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
@@ -52,7 +53,7 @@ func envInt(key string, def int) int {
|
||||
}
|
||||
|
||||
func envStr(key, def string) string {
|
||||
v := os.Getenv(key)
|
||||
v := common.GetEnv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user