Go: merge duplicate codes (#16783)

### Summary

1. merge heartbeat function.
2. introduce all environments

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-07-10 11:58:32 +08:00
committed by GitHub
parent 289ca28ce2
commit add7b9486f
67 changed files with 479 additions and 329 deletions

View File

@@ -10,6 +10,7 @@ import (
"math"
"os"
"path/filepath"
"ragflow/internal/common"
"regexp"
"sort"
"strconv"
@@ -44,14 +45,14 @@ func TestBatchResults(t *testing.T) {
pdfDir := filepath.Join("testdata", "real_pdfs")
all := listRealPDFs(t, pdfDir)
count := countFromEnv("BATCH_COUNT", len(all))
if single := os.Getenv("BATCH_SINGLE"); single != "" {
count := countFromEnv(common.EnvBatchCount, len(all))
if single := common.GetEnv(common.EnvBatchSingle); single != "" {
all = filterSingle(all, single, t)
count = 1
}
pdfs := all[:min(count, len(all))]
ddClient, err := inf.NewClient(os.Getenv("DEEPDOC_URL"))
ddClient, err := inf.NewClient(common.GetEnv(common.EnvDeepDocURL))
if err != nil {
t.Fatal(err)
}
@@ -73,7 +74,7 @@ func TestBatchResults(t *testing.T) {
func setupLogger() {
level := slog.LevelInfo
switch os.Getenv("BATCH_LOG_LEVEL") {
switch common.GetEnv(common.EnvBatchLogLevel) {
case "debug":
level = slog.LevelDebug
case "warn":
@@ -83,7 +84,7 @@ func setupLogger() {
}
func variantFromEnv() string {
if os.Getenv("BATCH_SKIP_OCR") == "1" {
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
return "noocr"
}
return "ocr"
@@ -106,7 +107,7 @@ func mkOutputDirs(variant string) outputDirs {
}
func countFromEnv(key string, ceiling int) int {
if s := os.Getenv(key); s != "" {
if s := common.GetEnv(key); s != "" {
n, err := strconv.Atoi(s)
if err == nil && n > 0 && n < ceiling {
return n
@@ -178,7 +179,7 @@ func processPDFs(t *testing.T, pdfDir string, pdfs []string, deepDoc pdf.DocAnal
t.Helper()
var results []tool.BatchResult
totalChars := 0
skipOCR := os.Getenv("BATCH_SKIP_OCR") == "1"
skipOCR := common.GetEnv(common.EnvBatchSkipOCR) == "1"
for i, name := range pdfs {
label := fmt.Sprintf("[%d/%d] %s", i+1, len(pdfs), name)

View File

@@ -6,6 +6,7 @@ import (
"log/slog"
"os"
"path/filepath"
"ragflow/internal/common"
"testing"
"ragflow/internal/deepdoc/parser/pdf/tool"
@@ -17,19 +18,19 @@ import (
// compare the noocr variant; PY_OCR_SUFFIX to override the Python variant.
func TestBatchCompareWithPython(t *testing.T) {
level := slog.LevelInfo
if os.Getenv("BATCH_LOG_LEVEL") == "debug" {
if common.GetEnv(common.EnvBatchLogLevel) == "debug" {
level = slog.LevelDebug
}
if os.Getenv("BATCH_LOG_LEVEL") == "warn" {
if common.GetEnv(common.EnvBatchLogLevel) == "warn" {
level = slog.LevelWarn
}
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
goVariant := "ocr"
if os.Getenv("BATCH_SKIP_OCR") == "1" {
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
goVariant = "noocr"
}
pyVariant := os.Getenv("PY_OCR_SUFFIX")
pyVariant := common.GetEnv(common.EnvPYOCRSuffix)
if pyVariant == "" {
pyVariant = goVariant
}

View File

@@ -5,6 +5,7 @@ package pdf
import (
"os"
"path/filepath"
"ragflow/internal/common"
"testing"
inf "ragflow/internal/deepdoc/parser/pdf/inference"
@@ -14,7 +15,7 @@ import (
// mustConnectInferenceClient returns a InferenceClient for the OSS DeepDoc service.
func mustConnectInferenceClient(t *testing.T) *inf.Client {
t.Helper()
url := os.Getenv("OSSDEEPDOC_URL")
url := common.GetEnv(common.EnvOSSDeepDocURL)
if url == "" {
url = "http://localhost:9390"
}

View File

@@ -18,7 +18,7 @@ import (
// instead of real text. This validates that detect+merge+recognize
// produces readable English from the scan.
func TestOCR_mergeChars_RealScanned(t *testing.T) {
url := os.Getenv("DEEPDOC_URL")
url := common.GetEnv(common.EnvDeepDocURL)
if url == "" {
t.Skip("DEEPDOC_URL not set")
}

View File

@@ -66,7 +66,7 @@ func writeGolden(t *testing.T, path string, v any) {
}
func updateGolden() bool {
return os.Getenv("UPDATE_GOLDEN") == "1"
return common.GetEnv(common.EnvUpdateGolden) == "1"
}
// sectionsToGolden converts []pdf.Section to the snapshot format.

View File

@@ -31,7 +31,7 @@ func TestPipelineParity(t *testing.T) {
t.Skipf("charspy/ not found: %v", err)
}
filter := os.Getenv("BATCH_PARITY_FILTER")
filter := common.GetEnv(common.EnvBatchParityFilter)
total, passed := 0, 0
for _, e := range entries {

View File

@@ -29,7 +29,7 @@ func TestTableRotation_Integration(t *testing.T) {
t.Skipf("test PDF not found: %s (run tools/generate_rotated_table_pdf.py first)", pdfPath)
}
baseURL := os.Getenv("DEEPDOC_URL")
baseURL := common.GetEnv(common.EnvDeepDocURL)
if baseURL == "" {
baseURL = "http://localhost:9390"
}
@@ -126,7 +126,7 @@ func TestTableRotation_Integration(t *testing.T) {
// TestTableRotation_Stability runs rotation detection on a sample real PDF
// and verifies the pipeline doesn't crash. Set BATCH_COUNT to limit.
func TestTableRotation_Stability(t *testing.T) {
baseURL := os.Getenv("DEEPDOC_URL")
baseURL := common.GetEnv(common.EnvDeepDocURL)
if baseURL == "" {
baseURL = "http://localhost:9390"
}

View File

@@ -24,7 +24,7 @@ func TestDumpTextOutput(t *testing.T) {
}
count := len(entries)
if n := os.Getenv("DUMP_COUNT"); n != "" {
if n := common.GetEnv(common.EnvDumpCount); n != "" {
c := 0
for _, ch := range n {
c = c*10 + int(ch-'0')

View File

@@ -7,6 +7,7 @@ import (
"math"
"os"
"path/filepath"
"ragflow/internal/common"
"sort"
"strconv"
"strings"
@@ -191,7 +192,7 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
}
// Also write CSV if BATCH_CSV env is set (backward compat).
if csvPath := os.Getenv("BATCH_CSV"); csvPath != "" {
if csvPath := common.GetEnv(common.EnvBatchCSV); csvPath != "" {
if err := WriteCSV(csvPath, diffs); err != nil {
log.Logf("CSV write error: %v", err)
} else {

View File

@@ -15,19 +15,19 @@ import (
// compare the noocr variant; PY_OCR_SUFFIX to override the Python variant.
func TestBatchCompareWithPython(t *testing.T) {
level := slog.LevelInfo
if os.Getenv("BATCH_LOG_LEVEL") == "debug" {
if common.GetEnv(common.EnvBatchLogLevel) == "debug" {
level = slog.LevelDebug
}
if os.Getenv("BATCH_LOG_LEVEL") == "warn" {
if common.GetEnv(common.EnvBatchLogLevel) == "warn" {
level = slog.LevelWarn
}
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
goVariant := "ocr"
if os.Getenv("BATCH_SKIP_OCR") == "1" {
if common.GetEnv(common.EnvBatchSkipOCR) == "1" {
goVariant = "noocr"
}
pyVariant := os.Getenv("PY_OCR_SUFFIX")
pyVariant := common.GetEnv(common.EnvPYOCRSuffix)
if pyVariant == "" {
pyVariant = goVariant
}

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"ragflow/internal/common"
"strconv"
"time"
)
@@ -26,12 +27,12 @@ func LoadConfig() Config {
pyVariant := "ocr"
td := filepath.Join("testdata")
return Config{
Count: envInt("BATCH_COUNT", 0),
Single: os.Getenv("BATCH_SINGLE"),
SkipOCR: os.Getenv("BATCH_SKIP_OCR") == "1",
CompareOnly: os.Getenv("BATCH_COMPARE_ONLY") == "1",
CompareFilter: os.Getenv("BATCH_COMPARE_FILTER"),
CSVOutput: envStr("BATCH_COMPARE_CSV", filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))),
Count: envInt(common.EnvBatchCount, 0),
Single: common.GetEnv(common.EnvBatchSingle),
SkipOCR: common.GetEnv(common.EnvBatchSkipOCR) == "1",
CompareOnly: common.GetEnv(common.EnvBatchCompareOnly) == "1",
CompareFilter: common.GetEnv(common.EnvBatchCompareFilter),
CSVOutput: envStr(common.EnvBatchCompareCSV, filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))),
GoTextDir: filepath.Join(td, "output", "go", goVariant, "text"),
PyTextDir: filepath.Join(td, "output", "py", pyVariant, "text"),
TablesDir: filepath.Join(td, "output", "go", goVariant, "tables"),
@@ -40,7 +41,7 @@ func LoadConfig() Config {
}
func envInt(key string, def int) int {
v := os.Getenv(key)
v := common.GetEnv(key)
if v == "" {
return def
}
@@ -52,7 +53,7 @@ func envInt(key string, def int) int {
}
func envStr(key, def string) string {
v := os.Getenv(key)
v := common.GetEnv(key)
if v == "" {
return def
}