fix: UTF-8-safe truncate (panic on Chinese mid-char)
Context shrink used &content[..200] which panics inside CJK codepoints.
This commit is contained in:
parent
d965bb4b4a
commit
4eb2a7c330
|
|
@ -124,6 +124,18 @@ fn tool_progress_line(name: &str, result_json: &str) -> String {
|
|||
format!("〔完成〕{name}")
|
||||
}
|
||||
|
||||
/// Largest byte index ≤ `max` that sits on a UTF-8 char boundary.
|
||||
fn floor_char_boundary(s: &str, max: usize) -> usize {
|
||||
if max >= s.len() {
|
||||
return s.len();
|
||||
}
|
||||
let mut end = max;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
end
|
||||
}
|
||||
|
||||
fn preview_progress(text: &str, max_chars: usize) -> String {
|
||||
let t = text.trim();
|
||||
if t.chars().count() <= max_chars {
|
||||
|
|
@ -217,7 +229,7 @@ pub fn truncate_messages(messages: &mut Vec<ChatMessage>, budget: usize) {
|
|||
for msg in messages.iter_mut() {
|
||||
if let Some(content) = msg.content.as_mut() {
|
||||
if content.len() > 200 {
|
||||
let keep = 200.min(content.len());
|
||||
let keep = floor_char_boundary(content, 200);
|
||||
let omitted = content.len().saturating_sub(keep);
|
||||
*content = format!("{}…\n[truncated {omitted} chars]", &content[..keep]);
|
||||
}
|
||||
|
|
@ -617,6 +629,17 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[test]
|
||||
fn floor_char_boundary_does_not_split_chinese() {
|
||||
let s2 = "abcdefghij宣告";
|
||||
// '告' is 3 bytes; index 11 lands inside it.
|
||||
let idx = 11;
|
||||
assert!(!s2.is_char_boundary(idx));
|
||||
let end = floor_char_boundary(s2, idx);
|
||||
assert!(s2.is_char_boundary(end));
|
||||
let _ = &s2[..end];
|
||||
}
|
||||
|
||||
fn truncate_keeps_system_and_shrinks_old_tools() {
|
||||
let mut msgs = vec![
|
||||
ChatMessage::system("sys"),
|
||||
|
|
|
|||
|
|
@ -382,14 +382,17 @@ async fn tool_write_file(ctx: &ToolContext, args: &Value) -> Result<Value> {
|
|||
|
||||
fn truncate_output(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!(
|
||||
"{}…\n[truncated {} bytes]",
|
||||
&s[..max],
|
||||
s.len().saturating_sub(max)
|
||||
)
|
||||
return s.to_string();
|
||||
}
|
||||
let mut end = max;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
format!(
|
||||
"{}…\n[truncated {} bytes]",
|
||||
&s[..end],
|
||||
s.len().saturating_sub(end)
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
Loading…
Reference in New Issue