From 6cb43f74c4ecea801573b403df1d28ee28f791fb Mon Sep 17 00:00:00 2001 From: Harsh Kashyap Date: Fri, 31 Jul 2026 09:21:21 +0530 Subject: [PATCH] fix(common,internal): round instead of truncate in normalize_overlapped_percent (closes #17418) (#17452) --- internal/ingestion/component/chunker/token_test.go | 8 ++++++++ internal/ingestion/component/schema/chunker.go | 12 ++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go index bb0c432ff2..c3cdff1e25 100644 --- a/internal/ingestion/component/chunker/token_test.go +++ b/internal/ingestion/component/chunker/token_test.go @@ -395,7 +395,11 @@ func TestNormalizeOverlappedPercent(t *testing.T) { }{ {"zero", 0, 0}, {"fraction 0.1 -> 10", 0.1, 10}, + {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29}, + {"fraction 0.3 -> 30", 0.3, 30}, {"fraction 0.5 -> 50", 0.5, 50}, + {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57}, + {"fraction 0.93 -> 90 (clamp after round)", 0.93, 90}, {"fraction 0.95 -> 90 (clamp)", 0.95, 90}, {"percent 15", 15, 15}, {"int truncation 33.3 -> 33", 33.3, 33}, @@ -413,6 +417,8 @@ func TestNormalizeOverlappedPercent(t *testing.T) { {"huge 1e300 -> 90", 1e300, 90}, {"huge -1e300 -> 0", -1e300, 0}, {"huge math.MaxFloat64 -> 90", math.MaxFloat64, 90}, + {"huge math.MaxFloat64 +1 -> 90 (clamp pre-round)", math.MaxFloat64 + 1, 90}, + {`numeric string fraction "0.29" -> 29`, "0.29", 29}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -491,6 +497,8 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) { wantErr bool }{ {"fraction 0.3 -> 30", 0.3, 30, false}, + {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29, false}, + {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57, false}, {"percent 0", 0, 0, false}, {"percent 30", 30, 30, false}, {"percent 90 (boundary)", 90, 90, false}, diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go index 6e9453dea8..826fdceabd 100644 --- a/internal/ingestion/component/schema/chunker.go +++ b/internal/ingestion/component/schema/chunker.go @@ -293,7 +293,12 @@ func clampOverlappedPercent(value float64) float64 { if value > 90 { value = 90 } - value = float64(int(value)) + // Round rather than truncate so fraction inputs like 0.29 normalize + // to 29 (user expectation) rather than 28. Mirrors Python's + // common/float_utils.py:50-58 fix for #17418. We still clamp + // before rounding so out-of-int-range values (e.g. 1e300) land on + // 90, not an implementation-defined int. + value = math.Round(value) return value } @@ -308,7 +313,10 @@ func (p *TokenChunkerParam) Validate() error { // rather than silently clamped — eliminating the latent footgun where a // fraction passed to a struct bypassed normalization. if v := p.OverlappedPercent; 0 < v && v < 1 { - p.OverlappedPercent = float64(int(v * 100)) + // Round rather than truncate so fraction inputs like 0.29 normalize + // to 29 (user expectation) rather than 28. Mirrors Python's + // common/float_utils.py:50-58 fix for #17418. + p.OverlappedPercent = math.Round(v * 100) } switch p.DelimiterMode { case "token_size", "delimiter":