// 語音輸入: hold a short recording from the microphone, turn it into text, put it in the message // box (the original: 聽寫 / transcribeAudio through its server). Here the recording uses macOS's // own AVAudioRecorder (through osascript, no extra install) and the text comes from an // OpenAI-compatible /audio/transcriptions endpoint with the person's own key. use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; /// Longest recording (estimated; the recorder script also stops itself at 10 minutes). pub const MAX_SECS: u64 = 600; /// 會議記錄 can run longer: two hours stays under the transcription service's 25 MB limit at this /// recorder's bitrate (estimated). pub const MEETING_MAX_SECS: u64 = 7_200; /// Transcription models tried in order. pub const MODELS: [&str; 2] = ["gpt-4o-mini-transcribe", "whisper-1"]; const RECORDER: &str = r#"ObjC.import('AVFoundation'); ObjC.import('Foundation'); function run(argv) { const url = $.NSURL.fileURLWithPath(argv[0]); const settings = $.NSDictionary.dictionaryWithObjectsForKeys( [$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)], [$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]); const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null); if (rec.isNil()) return 'fail'; if (!rec.record) return 'denied'; const fm = $.NSFileManager.defaultManager; for (let i = 0; i < 72000; i++) { if (fm.fileExistsAtPath(argv[1])) break; delay(0.1); } rec.stop; return 'ok'; }"#; /// 通話模式: listens until the person stops talking (AVAudioRecorder metering). /// Speech is louder than SPEECH_DB; SILENCE_TICKS × 0.1 s of quiet after speech ends a turn; /// nothing heard for WAIT_TICKS × 0.1 s gives up (all estimated). const SPEECH_DB: f64 = -38.0; const SILENCE_TICKS: u32 = 12; const WAIT_TICKS: u32 = 300; const LISTENER: &str = r#"ObjC.import('AVFoundation'); ObjC.import('Foundation'); function run(argv) { const url = $.NSURL.fileURLWithPath(argv[0]); const settings = $.NSDictionary.dictionaryWithObjectsForKeys( [$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)], [$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]); const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null); if (rec.isNil()) return 'fail'; rec.meteringEnabled = true; if (!rec.record) return 'denied'; const fm = $.NSFileManager.defaultManager; const speech = Number(argv[2]), quietMax = Number(argv[3]), waitMax = Number(argv[4]); let heard = false, quiet = 0; for (let i = 0; i < 6000; i++) { if (fm.fileExistsAtPath(argv[1])) break; delay(0.1); rec.updateMeters; const p = rec.averagePowerForChannel(0); if (p > speech) { heard = true; quiet = 0; } else if (heard) { quiet++; } if (heard && quiet >= quietMax) break; if (!heard && i >= waitMax) { rec.stop; return 'nothing'; } } rec.stop; return heard ? 'ok' : 'nothing'; }"#; /// The recorder: macOS's AVAudioRecorder script, or cpal (mic.rs) elsewhere. enum Rec { Script(Child), Mic(crate::mic::Recorder), } pub struct Recording { rec: Rec, pub file: PathBuf, stop: PathBuf, pub started: Instant, } pub fn start() -> Result { if !cfg!(target_os = "macos") { let file = std::env::temp_dir().join(format!("anybot-voice-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis())); let rec = crate::mic::start(file.clone(), None, MEETING_MAX_SECS); return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() }); } let base = std::env::temp_dir().join(format!("anybot-voice-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis())); let file = base.with_extension("m4a"); let stop = base.with_extension("stop"); let child = Command::new("osascript") .args(["-l", "JavaScript", "-e", RECORDER, "--"]) .arg(&file) .arg(&stop) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() .map_err(|e| format!("無法開始錄音:{e}"))?; Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() }) } /// Listen for one spoken turn: stops by itself when the person goes quiet. pub fn listen() -> Result { if !cfg!(target_os = "macos") { let file = std::env::temp_dir().join(format!("anybot-call-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis())); let rec = crate::mic::start(file.clone(), Some(crate::mic::Listen { speech_db: SPEECH_DB, quiet_ticks: SILENCE_TICKS, wait_ticks: WAIT_TICKS }), MAX_SECS); return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() }); } let base = std::env::temp_dir().join(format!("anybot-call-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis())); let (file, stop) = (base.with_extension("m4a"), base.with_extension("stop")); let child = Command::new("osascript") .args(["-l", "JavaScript", "-e", LISTENER, "--"]) .arg(&file) .arg(&stop) .args([SPEECH_DB.to_string(), SILENCE_TICKS.to_string(), WAIT_TICKS.to_string()]) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() .map_err(|e| format!("無法開始聽:{e}"))?; Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() }) } impl Recording { /// The listener stopped by itself (the person finished, or nobody spoke). pub fn done(&mut self) -> bool { match &mut self.rec { Rec::Script(c) => matches!(c.try_wait(), Ok(Some(_))), Rec::Mic(m) => m.done(), } } /// Stop and give back the audio file. pub fn finish(mut self) -> Result { let child = match &mut self.rec { Rec::Script(c) => c, Rec::Mic(m) => { return match m.finish() { Some("fail") => Err("找不到可以用的麥克風。".into()), Some("nothing") => Err("沒有聽到說話。".into()), None => Err("錄音沒有正常結束。".into()), Some(_) if std::fs::metadata(&self.file).map(|m| m.len() > 4000).unwrap_or(false) => Ok(self.file.clone()), Some(_) => Err("沒有錄到聲音。".into()), }; } }; let _ = std::fs::write(&self.stop, b""); let t = Instant::now(); let status = loop { if let Ok(Some(_)) = child.try_wait() { break true; } if t.elapsed() > Duration::from_secs(4) { let _ = child.kill(); break false; } std::thread::sleep(Duration::from_millis(50)); }; let _ = std::fs::remove_file(&self.stop); let mut said = String::new(); if let Some(mut o) = child.stdout.take() { use std::io::Read; let _ = o.read_to_string(&mut said); } match said.trim() { "denied" => { let _ = crate::computer::spawn_reaped(&mut Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone")); Err("macOS 還沒讓 AnyBot 使用麥克風。我已經打開「系統設定 → 隱私權與安全性 → 麥克風」:把 AnyBot(或「終端機」)打開,再試一次。".into()) } "fail" => Err("找不到可以用的麥克風。".into()), "nothing" => Err("沒有聽到說話。".into()), _ if !status => Err("錄音沒有正常結束。".into()), _ => { let ok = std::fs::metadata(&self.file).map(|m| m.len() > 1000).unwrap_or(false); if ok { Ok(self.file.clone()) } else { Err("沒有錄到聲音。".into()) } } } } pub fn cancel(mut self) { let _ = std::fs::write(&self.stop, b""); match &mut self.rec { Rec::Script(c) => { std::thread::sleep(Duration::from_millis(300)); let _ = c.kill(); } Rec::Mic(m) => { let _ = m.finish(); } } let _ = std::fs::remove_file(&self.stop); let _ = std::fs::remove_file(&self.file); } } /// multipart/form-data body for /audio/transcriptions. #[allow(dead_code)] pub fn multipart(model: &str, file_name: &str, audio: &[u8], boundary: &str) -> Vec { multipart_lang(model, file_name, audio, boundary, "") } /// Same, telling the service the spoken language (ISO 639-1, e.g. "zh"; empty = detect). pub fn multipart_lang(model: &str, file_name: &str, audio: &[u8], boundary: &str, lang: &str) -> Vec { let mut b = vec![]; let mut fields = vec![("model", model), ("response_format", "json")]; if !lang.is_empty() { fields.push(("language", lang)); } for (k, v) in fields { b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n").as_bytes()); } let kind = if file_name.ends_with(".wav") { "audio/wav" } else { "audio/mp4" }; b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{file_name}\"\r\nContent-Type: {kind}\r\n\r\n").as_bytes()); b.extend_from_slice(audio); b.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); b } /// Audio → text through `{base}/audio/transcriptions`. #[allow(dead_code)] pub fn transcribe(base_url: &str, api_key: &str, file: &std::path::Path) -> Result { transcribe_lang(base_url, api_key, file, "") } /// The spoken languages offered for voice chat (the original: 語言 Auto-detect…): (code, name). pub const SPOKEN: [(&str, &str); 6] = [("", "自動偵測"), ("zh", "中文"), ("en", "English"), ("ja", "日本語"), ("ko", "한국어"), ("es", "Español")]; pub fn transcribe_lang(base_url: &str, api_key: &str, file: &std::path::Path, lang: &str) -> Result { if api_key.trim().is_empty() { return Err("語音轉文字要用 OpenAI 的金鑰:請到「設定 → AI 供應商 → OpenAI」填入。".into()); } let audio = std::fs::read(file).map_err(|e| e.to_string())?; let boundary = format!("anybot{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)); let url = format!("{}/audio/transcriptions", base_url.trim_end_matches('/')); let agent: ureq::Agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(120))).http_status_as_error(false).build().into(); let mut last = String::new(); for model in MODELS { let name = if file.extension().and_then(|e| e.to_str()) == Some("wav") { "voice.wav" } else { "voice.m4a" }; let body = multipart_lang(model, name, &audio, &boundary, lang); let resp = agent .post(&url) .header("Authorization", &format!("Bearer {}", api_key.trim())) .header("Content-Type", &format!("multipart/form-data; boundary={boundary}")) .send(&body[..]); match resp { Ok(mut r) => { let status = r.status().as_u16(); let text = r.body_mut().read_to_string().unwrap_or_default(); if status == 200 { let v: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?; return Ok(v["text"].as_str().unwrap_or("").trim().to_string()); } last = format!("轉文字失敗(HTTP {status}):{}", text.chars().take(200).collect::()); // An unknown model: try the next one. if status != 404 && status != 400 { break; } } Err(e) => { last = format!("連不上語音轉文字服務:{e}"); break; } } } Err(last) } /// The voices installed on this Mac: (name, language code), Chinese and English first. pub fn voices() -> Vec<(String, String)> { static V: std::sync::OnceLock> = std::sync::OnceLock::new(); V.get_or_init(|| { if !cfg!(target_os = "macos") { let out = Command::new("espeak-ng").arg("--voices").output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default(); return parse_espeak_voices(&out); } let out = Command::new("say").args(["-v", "?"]).output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default(); parse_voices(&out) }) .clone() } pub fn parse_voices(out: &str) -> Vec<(String, String)> { let mut v: Vec<(String, String)> = out .lines() .filter_map(|l| { let head = l.split('#').next()?.trim_end(); let (name, lang) = head.rsplit_once(char::is_whitespace)?; let lang = lang.trim(); lang.contains('_').then(|| (name.trim().to_string(), lang.to_string())) }) .filter(|(n, _)| !n.is_empty()) .collect(); let rank = |l: &str| if l.starts_with("zh_TW") { 0 } else if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 }; v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0))); v } /// `espeak-ng --voices` (Linux): "Pty Language Age/Gender VoiceName File Other" → /// (language code to pass to -v, language as zh_CN / en_US). pub fn parse_espeak_voices(out: &str) -> Vec<(String, String)> { let mut v: Vec<(String, String)> = out .lines() .skip(1) .filter_map(|l| { let lang = l.split_whitespace().nth(1)?.to_string(); let norm = match lang.as_str() { "cmn" => "zh_CN".to_string(), "yue" => "zh_HK".to_string(), "cmn-latn-pinyin" => return None, l => { let mut parts = l.splitn(2, '-'); let a = parts.next().unwrap_or(""); match parts.next() { Some(b) => format!("{a}_{}", b.to_uppercase()), None => format!("{a}_{}", a.to_uppercase()), } } }; Some((lang, norm)) }) .collect(); let rank = |l: &str| if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 }; v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0))); // One entry per language (espeak-ng lists some twice, e.g. yue with and without Jyutping). v.dedup_by(|a, b| a.0 == b.0); v } // ---------- 語音聊天: reading answers aloud ---------- static SPEAKER: std::sync::Mutex> = std::sync::Mutex::new(None); /// Markdown → words worth saying. pub fn speakable(text: &str) -> String { let mut out = String::new(); let mut in_code = false; for line in text.lines() { let l = line.trim(); if l.starts_with("```") { in_code = !in_code; if in_code { out.push_str("(程式碼略過)\n"); } continue; } if in_code { continue; } let l = l.trim_start_matches(['#', '-', '*', '>', ' ']).replace("**", "").replace('`', ""); let l = l.split("](").next().unwrap_or(&l).replace('[', ""); if !l.is_empty() { out.push_str(&l); out.push('\n'); } } out.trim().to_string() } /// The voice for the text: Taiwanese Mandarin (美佳) when it has Chinese, else the system voice. pub fn voice_for(text: &str) -> Option<&'static str> { text.chars().any(|c| ('\u{4e00}'..='\u{9fff}').contains(&c)).then_some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" }) } /// Read an answer aloud (stops whatever was being read). #[allow(dead_code)] pub fn speak(text: &str) { speak_as(text, "", 1.0); } /// Words per minute of the system voices at 1x (`say`'s usual rate). pub const BASE_RATE: f32 = 175.0; /// The speeds offered (the original: 速度 1x…). pub const SPEEDS: [f32; 5] = [0.75, 1.0, 1.25, 1.5, 2.0]; /// Read aloud with a chosen voice ("" = pick by language) and speed (1.0 = normal). pub fn speak_as(text: &str, voice: &str, rate: f32) { stop_speaking(); let words = speakable(text); if words.is_empty() { return; } // Linux: espeak-ng takes the same -v (voice) and -s (words per minute, 175 by default). let mut c = Command::new(if cfg!(target_os = "macos") { "say" } else { "espeak-ng" }); if !voice.trim().is_empty() { c.args(["-v", voice.trim()]); } else if let Some(v) = voice_for(&words) { c.args(["-v", v]); } if (rate - 1.0).abs() > 0.01 { c.args([if cfg!(target_os = "macos") { "-r" } else { "-s" }, &format!("{}", (BASE_RATE * rate.clamp(0.5, 3.0)).round())]); } if let Ok(child) = c.arg(&words).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() { *SPEAKER.lock().unwrap() = Some(child); } } pub fn stop_speaking() { if let Some(mut c) = SPEAKER.lock().unwrap().take() { let _ = c.kill(); } } pub fn speaking() -> bool { let mut g = SPEAKER.lock().unwrap(); match g.as_mut().map(|c| c.try_wait()) { Some(Ok(None)) => true, _ => { *g = None; false } } } #[cfg(test)] mod tests { use super::*; use std::io::{Read, Write}; #[test] fn answers_are_cleaned_for_speaking() { let t = "## 行程\n- **上午**:赤崁樓\n```\nls -la\n```\n看 [地圖](https://x.y)"; assert_eq!(speakable(t), "行程\n上午:赤崁樓\n(程式碼略過)\n看 地圖"); assert_eq!(voice_for("你好"), Some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" })); let e = parse_espeak_voices("Pty Language Age/Gender VoiceName File Other Languages\n 5 cmn --/M Chinese_(Mandarin) sit/cmn (zh-cmn 5)(zh 5)\n 2 en-us --/M English_(America) gmw/en-US (en 3)\n"); assert_eq!(e[0], ("cmn".to_string(), "zh_CN".to_string())); assert_eq!(e[1], ("en-us".to_string(), "en_US".to_string())); let v = parse_voices("Samantha en_US # Hello\nMeijia zh_TW # 你好\nGrandma (Chinese (Taiwan)) zh_TW # 你好\n"); assert_eq!(v[0].1, "zh_TW"); assert!(v.iter().any(|x| x.0 == "Grandma (Chinese (Taiwan))")); assert_eq!(v.last().unwrap().0, "Samantha"); let body = String::from_utf8_lossy(&multipart_lang("m", "a.m4a", b"x", "B", "zh")).to_string(); assert!(body.contains("name=\"language\"\r\n\r\nzh")); assert_eq!(voice_for("hello"), None); } #[test] fn transcription_request_is_well_formed() { let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = server.local_addr().unwrap().port(); let seen = std::thread::spawn(move || { let (mut s, _) = server.accept().unwrap(); s.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); let mut buf = vec![0u8; 65536]; let mut got = vec![]; while let Ok(n) = s.read(&mut buf) { if n == 0 { break; } got.extend_from_slice(&buf[..n]); if got.windows(4).any(|w| w == b"--\r\n") { break; } } let reply = "{\"text\":\"你好,幫我排行程\"}"; let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes()); String::from_utf8_lossy(&got).to_string() }); let f = std::env::temp_dir().join(format!("anybot-voice-test-{}.m4a", std::process::id())); std::fs::write(&f, vec![7u8; 2048]).unwrap(); let text = transcribe(&format!("http://127.0.0.1:{port}/v1"), "sk-test", &f).unwrap(); let req = seen.join().unwrap(); let _ = std::fs::remove_file(&f); assert_eq!(text, "你好,幫我排行程"); assert!(req.starts_with("POST /v1/audio/transcriptions")); assert!(req.contains("Bearer sk-test") && req.contains("name=\"model\"\r\n\r\ngpt-4o-mini-transcribe") && req.contains("filename=\"voice.m4a\"")); assert!(transcribe("http://127.0.0.1:1", "", &std::path::PathBuf::from("/x")).unwrap_err().contains("OpenAI")); } }