zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-31 13:03:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Port PR14140 and PR16881 to GO (#17102)

Browse Source 
### Summary

Port
https://github.com/infiniflow/ragflow/pull/14140/
https://github.com/infiniflow/ragflow/pull/16881
This commit is contained in:
qinling0210
2026-07-21 17:46:23 +08:00
committed by GitHub
parent 20b760f266
commit 0189ca3700
19 changed files with 570 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
internal/tokenizer/tokenizer.go
View File

@@ -73,6 +73,12 @@ type analyzerPool struct {
wg sync.WaitGroup
}
// defaultLanguage is applied to every analyzer instance on pool acquisition
// to clear any sticky language state left by a previous task. Ingestion
// callers should use the public API variants that accept an explicit
// language override.
const defaultLanguage = "English"
var (
globalPool *analyzerPool
poolOnce sync.Once
@@ -395,8 +401,9 @@ func Close() {
}
}
// withAnalyzer executes the given function with an exclusive analyzer instance
func withAnalyzer(fn func(*rag.Analyzer) error) error {
// withAnalyzer acquires an analyzer instance, applies the given language
// (with "" mapping to defaultLanguage), and executes fn.
func withAnalyzer(lang string, fn func(*rag.Analyzer) error) error {
if globalPool == nil {
return fmt.Errorf("tokenizer pool not initialized")
}
@@ -407,11 +414,16 @@ func withAnalyzer(fn func(*rag.Analyzer) error) error {
}
defer globalPool.release(instance)
if lang == "" {
lang = defaultLanguage
}
instance.analyzer.SetLanguage(lang)
return fn(instance.analyzer)
}
// withAnalyzerResult executes the given function with an exclusive analyzer instance and returns a result
func withAnalyzerResult[T any](fn func(*rag.Analyzer) (T, error)) (T, error) {
// withAnalyzerResult is the result-returning variant of withAnalyzer.
func withAnalyzerResult[T any](lang string, fn func(*rag.Analyzer) (T, error)) (T, error) {
var result T
if globalPool == nil {
return result, fmt.Errorf("tokenizer pool not initialized")
@@ -423,32 +435,63 @@ func withAnalyzerResult[T any](fn func(*rag.Analyzer) (T, error)) (T, error) {
}
defer globalPool.release(instance)
if lang == "" {
lang = defaultLanguage
}
instance.analyzer.SetLanguage(lang)
return fn(instance.analyzer)
}
// Tokenize tokenizes the text and returns a space-separated string of tokens
type Tokenizer struct {
lang string
}
// New returns a request-scoped tokenizer. Empty language falls back to English.
func New(lang string) Tokenizer {
return Tokenizer{lang: lang}
}
var defaultTokenizer = New("")
// Tokenize tokenizes the text and returns a space-separated string of tokens.
// Example: "hello world" -> "hello world"
//
// NOTE: For Infinity engine, returns input unchanged to match python's behavior
// NOTE: For Infinity engine, returns input unchanged to match python's behavior.
func Tokenize(text string) (string, error) {
return defaultTokenizer.Tokenize(text)
}
// Tokenize tokenizes the text using the tokenizer's request-scoped language.
func (t Tokenizer) Tokenize(text string) (string, error) {
if engineTypeProvider() == "infinity" {
return text, nil
}
return withAnalyzerResult(func(a *rag.Analyzer) (string, error) {
return withAnalyzerResult(t.lang, func(a *rag.Analyzer) (string, error) {
return a.Tokenize(text)
})
}
// TokenizeWithPosition tokenizes the text and returns a list of tokens with position information
// TokenizeWithPosition tokenizes the text and returns a list of tokens with position information.
func TokenizeWithPosition(text string) ([]rag.TokenWithPosition, error) {
return withAnalyzerResult(func(a *rag.Analyzer) ([]rag.TokenWithPosition, error) {
return defaultTokenizer.TokenizeWithPosition(text)
}
// TokenizeWithPosition tokenizes the text using the tokenizer's request-scoped language.
func (t Tokenizer) TokenizeWithPosition(text string) ([]rag.TokenWithPosition, error) {
return withAnalyzerResult(t.lang, func(a *rag.Analyzer) ([]rag.TokenWithPosition, error) {
return a.TokenizeWithPosition(text)
})
}
// Analyze analyzes the text and returns all tokens
// Analyze analyzes the text and returns all tokens.
func Analyze(text string) ([]rag.Token, error) {
return withAnalyzerResult(func(a *rag.Analyzer) ([]rag.Token, error) {
return defaultTokenizer.Analyze(text)
}
// Analyze analyzes the text using the tokenizer's request-scoped language.
func (t Tokenizer) Analyze(text string) ([]rag.Token, error) {
return withAnalyzerResult(t.lang, func(a *rag.Analyzer) ([]rag.Token, error) {
return a.Analyze(text)
})
}
@@ -462,16 +505,23 @@ func SetFineGrained(fineGrained bool) {
common.Debug("SetFineGrained is no-op in pool mode", zap.Bool("fine_grained", fineGrained))
}
// FineGrainedTokenize performs fine-grained tokenization on space-separated tokens
// FineGrainedTokenize performs fine-grained tokenization on space-separated
// tokens.
// Input: space-separated tokens (e.g., "hello world 测试")
// Output: space-separated fine-grained tokens (e.g., "hello world 测 试")
//
// NOTE: For Infinity engine, returns input unchanged to match python's behavior
// NOTE: For Infinity engine, returns input unchanged to match python's behavior.
func FineGrainedTokenize(tokens string) (string, error) {
return defaultTokenizer.FineGrainedTokenize(tokens)
}
// FineGrainedTokenize performs fine-grained tokenization using the tokenizer's
// request-scoped language.
func (t Tokenizer) FineGrainedTokenize(tokens string) (string, error) {
if engineTypeProvider() == "infinity" {
return tokens, nil
}
return withAnalyzerResult(func(a *rag.Analyzer) (string, error) {
return withAnalyzerResult(t.lang, func(a *rag.Analyzer) (string, error) {
return a.FineGrainedTokenize(tokens)
})
}
@@ -490,7 +540,7 @@ func IsInitialized() bool {
// GetTermFreq returns the frequency of a term (matching Python rag_tokenizer.freq)
// Returns: frequency value, or 0 if term not found
func GetTermFreq(term string) int32 {
result, _ := withAnalyzerResult(func(a *rag.Analyzer) (int32, error) {
result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (int32, error) {
return a.GetTermFreq(term), nil
})
return result
@@ -499,7 +549,7 @@ func GetTermFreq(term string) int32 {
// GetTermTag returns the POS tag of a term (matching Python rag_tokenizer.tag)
// Returns: POS tag string (e.g., "n", "v", "ns"), or empty string if term not found or no tag
func GetTermTag(term string) string {
result, _ := withAnalyzerResult(func(a *rag.Analyzer) (string, error) {
result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (string, error) {
return a.GetTermTag(term), nil
})
return result