mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-09 21:04:49 +08:00
fix(harness): truncate text on rune boundary to keep UTF-8 valid (#16511)
### Summary `truncateText` in the `reduction` and `summarization` middlewares truncates with `s[:maxLen]`, which slices by byte. When `maxLen` lands inside a multi-byte character (common with CJK or other non-ASCII content flowing through the agent), the string is cut mid-rune and the tail byte(s) become invalid UTF-8. That broken text then goes into the reduced context / summary prompt. `TruncateToolResult` in the same `reduction` package already avoids this by slicing on a rune boundary and even notes it in a comment. This PR makes the two `truncateText` helpers do the same, so they stay consistent with the existing helper. Both functions keep their existing output shape (summarization still appends `...`). Added a small unit test in each package covering ASCII truncation and a CJK string, asserting the result stays valid UTF-8.
This commit is contained in:
@@ -165,5 +165,11 @@ func truncateText(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen]
|
||||
// Slice on a rune boundary; [:maxLen] indexes bytes and can split a
|
||||
// multi-byte UTF-8 character, same as TruncateToolResult guards against.
|
||||
runes := []rune(s)
|
||||
if maxLen > len(runes) {
|
||||
return s
|
||||
}
|
||||
return string(runes[:maxLen])
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package reduction
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestTruncateText(t *testing.T) {
|
||||
if got := truncateText("hello", 10); got != "hello" {
|
||||
t.Fatalf("short string changed: %q", got)
|
||||
}
|
||||
if got := truncateText("hello world", 5); got != "hello" {
|
||||
t.Fatalf("ascii truncation: got %q want %q", got, "hello")
|
||||
}
|
||||
|
||||
// A byte slice at maxLen would land in the middle of a multi-byte rune
|
||||
// and yield invalid UTF-8. Truncating on a rune boundary must not.
|
||||
got := truncateText("日本語のテキスト", 3)
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("result is not valid UTF-8: %q", got)
|
||||
}
|
||||
if got != "日本語" {
|
||||
t.Fatalf("got %q want %q", got, "日本語")
|
||||
}
|
||||
}
|
||||
@@ -275,7 +275,13 @@ func truncateText(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
// Slice on a rune boundary; [:maxLen] indexes bytes and can split a
|
||||
// multi-byte UTF-8 character, producing invalid output in the summary.
|
||||
runes := []rune(s)
|
||||
if maxLen > len(runes) {
|
||||
return s
|
||||
}
|
||||
return string(runes[:maxLen]) + "..."
|
||||
}
|
||||
|
||||
func buildSummaryPrompt[M core.MessageType](content string) M {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package summarization
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestTruncateText(t *testing.T) {
|
||||
if got := truncateText("hello", 10); got != "hello" {
|
||||
t.Fatalf("short string changed: %q", got)
|
||||
}
|
||||
if got := truncateText("hello world", 5); got != "hello..." {
|
||||
t.Fatalf("ascii truncation: got %q want %q", got, "hello...")
|
||||
}
|
||||
|
||||
// A byte slice at maxLen would split a multi-byte rune and produce
|
||||
// invalid UTF-8; truncating on a rune boundary must not.
|
||||
got := truncateText("日本語のテキスト", 3)
|
||||
if !utf8.ValidString(strings.TrimSuffix(got, "...")) {
|
||||
t.Fatalf("result is not valid UTF-8: %q", got)
|
||||
}
|
||||
if got != "日本語..." {
|
||||
t.Fatalf("got %q want %q", got, "日本語...")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user