fix: UTF-8-safe truncate (panic on Chinese mid-char)
Context shrink used &content[..200] which panics inside CJK codepoints.
This commit is contained in:
parent
d965bb4b4a
commit
4eb2a7c330
|
|
@ -124,6 +124,18 @@ fn tool_progress_line(name: &str, result_json: &str) -> String {
|
||||||
format!("〔完成〕{name}")
|
format!("〔完成〕{name}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Largest byte index ≤ `max` that sits on a UTF-8 char boundary.
|
||||||
|
fn floor_char_boundary(s: &str, max: usize) -> usize {
|
||||||
|
if max >= s.len() {
|
||||||
|
return s.len();
|
||||||
|
}
|
||||||
|
let mut end = max;
|
||||||
|
while end > 0 && !s.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
end
|
||||||
|
}
|
||||||
|
|
||||||
fn preview_progress(text: &str, max_chars: usize) -> String {
|
fn preview_progress(text: &str, max_chars: usize) -> String {
|
||||||
let t = text.trim();
|
let t = text.trim();
|
||||||
if t.chars().count() <= max_chars {
|
if t.chars().count() <= max_chars {
|
||||||
|
|
@ -217,7 +229,7 @@ pub fn truncate_messages(messages: &mut Vec<ChatMessage>, budget: usize) {
|
||||||
for msg in messages.iter_mut() {
|
for msg in messages.iter_mut() {
|
||||||
if let Some(content) = msg.content.as_mut() {
|
if let Some(content) = msg.content.as_mut() {
|
||||||
if content.len() > 200 {
|
if content.len() > 200 {
|
||||||
let keep = 200.min(content.len());
|
let keep = floor_char_boundary(content, 200);
|
||||||
let omitted = content.len().saturating_sub(keep);
|
let omitted = content.len().saturating_sub(keep);
|
||||||
*content = format!("{}…\n[truncated {omitted} chars]", &content[..keep]);
|
*content = format!("{}…\n[truncated {omitted} chars]", &content[..keep]);
|
||||||
}
|
}
|
||||||
|
|
@ -617,6 +629,17 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[test]
|
||||||
|
fn floor_char_boundary_does_not_split_chinese() {
|
||||||
|
let s2 = "abcdefghij宣告";
|
||||||
|
// '告' is 3 bytes; index 11 lands inside it.
|
||||||
|
let idx = 11;
|
||||||
|
assert!(!s2.is_char_boundary(idx));
|
||||||
|
let end = floor_char_boundary(s2, idx);
|
||||||
|
assert!(s2.is_char_boundary(end));
|
||||||
|
let _ = &s2[..end];
|
||||||
|
}
|
||||||
|
|
||||||
fn truncate_keeps_system_and_shrinks_old_tools() {
|
fn truncate_keeps_system_and_shrinks_old_tools() {
|
||||||
let mut msgs = vec![
|
let mut msgs = vec![
|
||||||
ChatMessage::system("sys"),
|
ChatMessage::system("sys"),
|
||||||
|
|
|
||||||
|
|
@ -382,14 +382,17 @@ async fn tool_write_file(ctx: &ToolContext, args: &Value) -> Result<Value> {
|
||||||
|
|
||||||
fn truncate_output(s: &str, max: usize) -> String {
|
fn truncate_output(s: &str, max: usize) -> String {
|
||||||
if s.len() <= max {
|
if s.len() <= max {
|
||||||
s.to_string()
|
return s.to_string();
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
"{}…\n[truncated {} bytes]",
|
|
||||||
&s[..max],
|
|
||||||
s.len().saturating_sub(max)
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
let mut end = max;
|
||||||
|
while end > 0 && !s.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
format!(
|
||||||
|
"{}…\n[truncated {} bytes]",
|
||||||
|
&s[..end],
|
||||||
|
s.len().saturating_sub(end)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue