zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-01 13:33:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(common,internal): round instead of truncate in normalize_overlapped_percent (closes #17418) (#17452)

Browse Source
This commit is contained in:
Harsh Kashyap
2026-07-31 09:21:21 +05:30
committed by GitHub
parent 5249de53da
commit 6cb43f74c4
2 changed files with 18 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/ingestion/component/chunker/token_test.go
View File

@@ -395,7 +395,11 @@ func TestNormalizeOverlappedPercent(t *testing.T) {
}{
{"zero", 0, 0},
{"fraction 0.1 -> 10", 0.1, 10},
{"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29},
{"fraction 0.3 -> 30", 0.3, 30},
{"fraction 0.5 -> 50", 0.5, 50},
{"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57},
{"fraction 0.93 -> 90 (clamp after round)", 0.93, 90},
{"fraction 0.95 -> 90 (clamp)", 0.95, 90},
{"percent 15", 15, 15},
{"int truncation 33.3 -> 33", 33.3, 33},
@@ -413,6 +417,8 @@ func TestNormalizeOverlappedPercent(t *testing.T) {
{"huge 1e300 -> 90", 1e300, 90},
{"huge -1e300 -> 0", -1e300, 0},
{"huge math.MaxFloat64 -> 90", math.MaxFloat64, 90},
{"huge math.MaxFloat64 +1 -> 90 (clamp pre-round)", math.MaxFloat64 + 1, 90},
{`numeric string fraction "0.29" -> 29`, "0.29", 29},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -491,6 +497,8 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) {
wantErr bool
}{
{"fraction 0.3 -> 30", 0.3, 30, false},
{"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29, false},
{"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57, false},
{"percent 0", 0, 0, false},
{"percent 30", 30, 30, false},
{"percent 90 (boundary)", 90, 90, false},

12
internal/ingestion/component/schema/chunker.go
View File

@@ -293,7 +293,12 @@ func clampOverlappedPercent(value float64) float64 {
if value > 90 {
value = 90
}
value = float64(int(value))
// Round rather than truncate so fraction inputs like 0.29 normalize
// to 29 (user expectation) rather than 28. Mirrors Python's
// common/float_utils.py:50-58 fix for #17418. We still clamp
// before rounding so out-of-int-range values (e.g. 1e300) land on
// 90, not an implementation-defined int.
value = math.Round(value)
return value
}
@@ -308,7 +313,10 @@ func (p *TokenChunkerParam) Validate() error {
// rather than silently clamped — eliminating the latent footgun where a
// fraction passed to a struct bypassed normalization.
if v := p.OverlappedPercent; 0 < v && v < 1 {
p.OverlappedPercent = float64(int(v * 100))
// Round rather than truncate so fraction inputs like 0.29 normalize
// to 29 (user expectation) rather than 28. Mirrors Python's
// common/float_utils.py:50-58 fix for #17418.
p.OverlappedPercent = math.Round(v * 100)
}
switch p.DelimiterMode {
case "token_size", "delimiter":