fix(harness): truncate text on rune boundary to keep UTF-8 valid (#16511)

### Summary

`truncateText` in the `reduction` and `summarization` middlewares
truncates with `s[:maxLen]`, which slices by byte. When `maxLen` lands
inside a multi-byte character (common with CJK or other non-ASCII
content flowing through the agent), the string is cut mid-rune and the
tail byte(s) become invalid UTF-8. That broken text then goes into the
reduced context / summary prompt.

`TruncateToolResult` in the same `reduction` package already avoids this
by slicing on a rune boundary and even notes it in a comment. This PR
makes the two `truncateText` helpers do the same, so they stay
consistent with the existing helper.

Both functions keep their existing output shape (summarization still
appends `...`). Added a small unit test in each package covering ASCII
truncation and a CJK string, asserting the result stays valid UTF-8.
This commit is contained in:
ZF
2026-07-01 16:45:26 +08:00
committed by GitHub
parent d770217b25
commit 97a4c64cc8
4 changed files with 65 additions and 2 deletions

View File

@@ -165,5 +165,11 @@ func truncateText(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen]
// Slice on a rune boundary; [:maxLen] indexes bytes and can split a
// multi-byte UTF-8 character, same as TruncateToolResult guards against.
runes := []rune(s)
if maxLen > len(runes) {
return s
}
return string(runes[:maxLen])
}

View File

@@ -0,0 +1,25 @@
package reduction
import (
"testing"
"unicode/utf8"
)
func TestTruncateText(t *testing.T) {
if got := truncateText("hello", 10); got != "hello" {
t.Fatalf("short string changed: %q", got)
}
if got := truncateText("hello world", 5); got != "hello" {
t.Fatalf("ascii truncation: got %q want %q", got, "hello")
}
// A byte slice at maxLen would land in the middle of a multi-byte rune
// and yield invalid UTF-8. Truncating on a rune boundary must not.
got := truncateText("日本語のテキスト", 3)
if !utf8.ValidString(got) {
t.Fatalf("result is not valid UTF-8: %q", got)
}
if got != "日本語" {
t.Fatalf("got %q want %q", got, "日本語")
}
}

View File

@@ -275,7 +275,13 @@ func truncateText(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
// Slice on a rune boundary; [:maxLen] indexes bytes and can split a
// multi-byte UTF-8 character, producing invalid output in the summary.
runes := []rune(s)
if maxLen > len(runes) {
return s
}
return string(runes[:maxLen]) + "..."
}
func buildSummaryPrompt[M core.MessageType](content string) M {

View File

@@ -0,0 +1,26 @@
package summarization
import (
"strings"
"testing"
"unicode/utf8"
)
func TestTruncateText(t *testing.T) {
if got := truncateText("hello", 10); got != "hello" {
t.Fatalf("short string changed: %q", got)
}
if got := truncateText("hello world", 5); got != "hello..." {
t.Fatalf("ascii truncation: got %q want %q", got, "hello...")
}
// A byte slice at maxLen would split a multi-byte rune and produce
// invalid UTF-8; truncating on a rune boundary must not.
got := truncateText("日本語のテキスト", 3)
if !utf8.ValidString(strings.TrimSuffix(got, "...")) {
t.Fatalf("result is not valid UTF-8: %q", got)
}
if got != "日本語..." {
t.Fatalf("got %q want %q", got, "日本語...")
}
}