mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-01 13:33:48 +08:00
90f46b0b4d
Ports doc-level auto-metadata extraction to Go and adds the knowledge_compiler component with scheduler/routing. Fixes Extractor metadata injection type assertion and enable_metadata default-on.
21 lines
557 B
Go
21 lines
557 B
Go
package common
|
|
|
|
// EstimateTokens is a lightweight, dependency-free token estimate used when no
|
|
// real Tokenizer is injected (≈4 chars/token, matching common LLM tokenizers'
|
|
// order of magnitude). Production wiring should inject a real Tokenizer.
|
|
func EstimateTokens(s string) int {
|
|
n := (len(s) + 3) / 4
|
|
if n == 0 {
|
|
return 0
|
|
}
|
|
return n
|
|
}
|
|
|
|
// numTokens returns the token count of s using tok when non-nil, else EstimateTokens.
|
|
func numTokens(tok Tokenizer, s string) int {
|
|
if tok != nil {
|
|
return tok.NumTokens(s)
|
|
}
|
|
return EstimateTokens(s)
|
|
}
|