diff --git a/internal/harness/core/middlewares/reduction/reduction.go b/internal/harness/core/middlewares/reduction/reduction.go index b2ee8a9fa..073ad9d6c 100644 --- a/internal/harness/core/middlewares/reduction/reduction.go +++ b/internal/harness/core/middlewares/reduction/reduction.go @@ -165,5 +165,11 @@ func truncateText(s string, maxLen int) string { if len(s) <= maxLen { return s } - return s[:maxLen] + // Slice on a rune boundary; [:maxLen] indexes bytes and can split a + // multi-byte UTF-8 character, same as TruncateToolResult guards against. + runes := []rune(s) + if maxLen > len(runes) { + return s + } + return string(runes[:maxLen]) } diff --git a/internal/harness/core/middlewares/reduction/truncate_text_test.go b/internal/harness/core/middlewares/reduction/truncate_text_test.go new file mode 100644 index 000000000..d9a8f0e2b --- /dev/null +++ b/internal/harness/core/middlewares/reduction/truncate_text_test.go @@ -0,0 +1,25 @@ +package reduction + +import ( + "testing" + "unicode/utf8" +) + +func TestTruncateText(t *testing.T) { + if got := truncateText("hello", 10); got != "hello" { + t.Fatalf("short string changed: %q", got) + } + if got := truncateText("hello world", 5); got != "hello" { + t.Fatalf("ascii truncation: got %q want %q", got, "hello") + } + + // A byte slice at maxLen would land in the middle of a multi-byte rune + // and yield invalid UTF-8. Truncating on a rune boundary must not. + got := truncateText("日本語のテキスト", 3) + if !utf8.ValidString(got) { + t.Fatalf("result is not valid UTF-8: %q", got) + } + if got != "日本語" { + t.Fatalf("got %q want %q", got, "日本語") + } +} diff --git a/internal/harness/core/middlewares/summarization/summarization.go b/internal/harness/core/middlewares/summarization/summarization.go index dc81dbce3..a6afe386a 100644 --- a/internal/harness/core/middlewares/summarization/summarization.go +++ b/internal/harness/core/middlewares/summarization/summarization.go @@ -275,7 +275,13 @@ func truncateText(s string, maxLen int) string { if len(s) <= maxLen { return s } - return s[:maxLen] + "..." + // Slice on a rune boundary; [:maxLen] indexes bytes and can split a + // multi-byte UTF-8 character, producing invalid output in the summary. + runes := []rune(s) + if maxLen > len(runes) { + return s + } + return string(runes[:maxLen]) + "..." } func buildSummaryPrompt[M core.MessageType](content string) M { diff --git a/internal/harness/core/middlewares/summarization/truncate_text_test.go b/internal/harness/core/middlewares/summarization/truncate_text_test.go new file mode 100644 index 000000000..1dea7b15c --- /dev/null +++ b/internal/harness/core/middlewares/summarization/truncate_text_test.go @@ -0,0 +1,26 @@ +package summarization + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestTruncateText(t *testing.T) { + if got := truncateText("hello", 10); got != "hello" { + t.Fatalf("short string changed: %q", got) + } + if got := truncateText("hello world", 5); got != "hello..." { + t.Fatalf("ascii truncation: got %q want %q", got, "hello...") + } + + // A byte slice at maxLen would split a multi-byte rune and produce + // invalid UTF-8; truncating on a rune boundary must not. + got := truncateText("日本語のテキスト", 3) + if !utf8.ValidString(strings.TrimSuffix(got, "...")) { + t.Fatalf("result is not valid UTF-8: %q", got) + } + if got != "日本語..." { + t.Fatalf("got %q want %q", got, "日本語...") + } +}