zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-03 22:30:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): enable go test in CI pipeline (#15750)

Browse Source 
## What problem does this PR solve?

Go test files are never compiled in CI — only production binaries via
`go build`. This allowed a missing `"sort"` import in
`metadata_filter_test.go` to be merged without detection.

## Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)

## Changes

- Add `go test -count=1 ./internal/...` step after Go build in CI
workflow
- Fix missing `"sort"` import in `metadata_filter_test.go` (pre-existing
compile error)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Jack
2026-06-08 20:06:57 +08:00
committed by GitHub
parent c5d0060e0b
commit 338fdb65fb
32 changed files with 280 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/tokenizer/tokenizer.go
View File

@@ -19,6 +19,7 @@ package tokenizer
import (
"context"
"fmt"
"os"
"ragflow/internal/common"
"ragflow/internal/engine"
"runtime"
@@ -80,7 +81,11 @@ func Init(cfg *PoolConfig) error {
// Set default values
if cfg.DictPath == "" {
cfg.DictPath = "/usr/share/infinity/resource"
if env := os.Getenv("RAGFLOW_DICT_PATH"); env != "" {
cfg.DictPath = env
} else {
cfg.DictPath = "/usr/share/infinity/resource"
}
}
if cfg.MinSize <= 0 {
cfg.MinSize = runtime.NumCPU() * 2

16
internal/tokenizer/tokenizer_concurrent_test.go
View File

@@ -39,7 +39,7 @@ func init() {
func TestConcurrentTokenize(t *testing.T) {
// Use small pool to test expansion
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 2,
MaxSize: 10,
IdleTimeout: 5 * time.Second,
@@ -175,7 +175,7 @@ func TestConcurrentTokenize(t *testing.T) {
// TestConcurrentTokenizeWithPosition tests concurrent tokenization with position info
func TestConcurrentTokenizeWithPosition(t *testing.T) {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 2,
MaxSize: 8,
IdleTimeout: 3 * time.Second,
@@ -236,7 +236,7 @@ func TestConcurrentTokenizeWithPosition(t *testing.T) {
func TestPoolExhaustion(t *testing.T) {
// Very small pool to test exhaustion
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 1,
MaxSize: 2,
IdleTimeout: 10 * time.Second,
@@ -299,7 +299,7 @@ func TestPoolExhaustion(t *testing.T) {
// TestFineGrainedTokenizeConcurrent tests concurrent fine-grained tokenization
func TestFineGrainedTokenizeConcurrent(t *testing.T) {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 2,
MaxSize: 6,
IdleTimeout: 3 * time.Second,
@@ -346,7 +346,7 @@ func TestFineGrainedTokenizeConcurrent(t *testing.T) {
// TestTermFreqAndTagConcurrent tests concurrent term frequency and tag lookups
func TestTermFreqAndTagConcurrent(t *testing.T) {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 2,
MaxSize: 6,
IdleTimeout: 3 * time.Second,
@@ -392,7 +392,7 @@ func TestTermFreqAndTagConcurrent(t *testing.T) {
// BenchmarkTokenize benchmarks the tokenization performance
func BenchmarkTokenize(b *testing.B) {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: runtime.NumCPU() * 2,
MaxSize: runtime.NumCPU() * 4,
IdleTimeout: 5 * time.Minute,
@@ -428,7 +428,7 @@ func BenchmarkTokenize(b *testing.B) {
// BenchmarkTokenizeWithPosition benchmarks position-aware tokenization
func BenchmarkTokenizeWithPosition(b *testing.B) {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: runtime.NumCPU() * 2,
MaxSize: runtime.NumCPU() * 4,
IdleTimeout: 5 * time.Minute,
@@ -456,7 +456,7 @@ func BenchmarkTokenizeWithPosition(b *testing.B) {
// ExampleGetPoolStats demonstrates getting pool statistics
func ExampleGetPoolStats() {
cfg := &PoolConfig{
DictPath: "/usr/share/infinity/resource",
DictPath: "", // uses default or RAGFLOW_DICT_PATH env var
MinSize: 2,
MaxSize: 10,
IdleTimeout: 5 * time.Minute,