AnyBot/src/voice.rs

458 lines
20 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// 語音輸入: hold a short recording from the microphone, turn it into text, put it in the message
// box (the original: 聽寫 / transcribeAudio through its server). Here the recording uses macOS's
// own AVAudioRecorder (through osascript, no extra install) and the text comes from an
// OpenAI-compatible /audio/transcriptions endpoint with the person's own key.
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
/// Longest recording (estimated; the recorder script also stops itself at 10 minutes).
pub const MAX_SECS: u64 = 600;
/// 會議記錄 can run longer: two hours stays under the transcription service's 25 MB limit at this
/// recorder's bitrate (estimated).
pub const MEETING_MAX_SECS: u64 = 7_200;
/// Transcription models tried in order.
pub const MODELS: [&str; 2] = ["gpt-4o-mini-transcribe", "whisper-1"];
const RECORDER: &str = r#"ObjC.import('AVFoundation');
ObjC.import('Foundation');
function run(argv) {
const url = $.NSURL.fileURLWithPath(argv[0]);
const settings = $.NSDictionary.dictionaryWithObjectsForKeys(
[$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)],
[$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]);
const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null);
if (rec.isNil()) return 'fail';
if (!rec.record) return 'denied';
const fm = $.NSFileManager.defaultManager;
for (let i = 0; i < 72000; i++) { if (fm.fileExistsAtPath(argv[1])) break; delay(0.1); }
rec.stop;
return 'ok';
}"#;
/// 通話模式: listens until the person stops talking (AVAudioRecorder metering).
/// Speech is louder than SPEECH_DB; SILENCE_TICKS × 0.1 s of quiet after speech ends a turn;
/// nothing heard for WAIT_TICKS × 0.1 s gives up (all estimated).
const SPEECH_DB: f64 = -38.0;
const SILENCE_TICKS: u32 = 12;
const WAIT_TICKS: u32 = 300;
const LISTENER: &str = r#"ObjC.import('AVFoundation');
ObjC.import('Foundation');
function run(argv) {
const url = $.NSURL.fileURLWithPath(argv[0]);
const settings = $.NSDictionary.dictionaryWithObjectsForKeys(
[$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)],
[$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]);
const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null);
if (rec.isNil()) return 'fail';
rec.meteringEnabled = true;
if (!rec.record) return 'denied';
const fm = $.NSFileManager.defaultManager;
const speech = Number(argv[2]), quietMax = Number(argv[3]), waitMax = Number(argv[4]);
let heard = false, quiet = 0;
for (let i = 0; i < 6000; i++) {
if (fm.fileExistsAtPath(argv[1])) break;
delay(0.1);
rec.updateMeters;
const p = rec.averagePowerForChannel(0);
if (p > speech) { heard = true; quiet = 0; } else if (heard) { quiet++; }
if (heard && quiet >= quietMax) break;
if (!heard && i >= waitMax) { rec.stop; return 'nothing'; }
}
rec.stop;
return heard ? 'ok' : 'nothing';
}"#;
/// The recorder: macOS's AVAudioRecorder script, or cpal (mic.rs) elsewhere.
enum Rec {
Script(Child),
Mic(crate::mic::Recorder),
}
pub struct Recording {
rec: Rec,
pub file: PathBuf,
stop: PathBuf,
pub started: Instant,
}
pub fn start() -> Result<Recording, String> {
if !cfg!(target_os = "macos") {
let file = std::env::temp_dir().join(format!("anybot-voice-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis()));
let rec = crate::mic::start(file.clone(), None, MEETING_MAX_SECS);
return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() });
}
let base = std::env::temp_dir().join(format!("anybot-voice-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis()));
let file = base.with_extension("m4a");
let stop = base.with_extension("stop");
let child = Command::new("osascript")
.args(["-l", "JavaScript", "-e", RECORDER, "--"])
.arg(&file)
.arg(&stop)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("無法開始錄音:{e}"))?;
Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() })
}
/// Listen for one spoken turn: stops by itself when the person goes quiet.
pub fn listen() -> Result<Recording, String> {
if !cfg!(target_os = "macos") {
let file = std::env::temp_dir().join(format!("anybot-call-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis()));
let rec = crate::mic::start(file.clone(), Some(crate::mic::Listen { speech_db: SPEECH_DB, quiet_ticks: SILENCE_TICKS, wait_ticks: WAIT_TICKS }), MAX_SECS);
return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() });
}
let base = std::env::temp_dir().join(format!("anybot-call-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis()));
let (file, stop) = (base.with_extension("m4a"), base.with_extension("stop"));
let child = Command::new("osascript")
.args(["-l", "JavaScript", "-e", LISTENER, "--"])
.arg(&file)
.arg(&stop)
.args([SPEECH_DB.to_string(), SILENCE_TICKS.to_string(), WAIT_TICKS.to_string()])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("無法開始聽:{e}"))?;
Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() })
}
impl Recording {
/// The listener stopped by itself (the person finished, or nobody spoke).
pub fn done(&mut self) -> bool {
match &mut self.rec {
Rec::Script(c) => matches!(c.try_wait(), Ok(Some(_))),
Rec::Mic(m) => m.done(),
}
}
/// Stop and give back the audio file.
pub fn finish(mut self) -> Result<PathBuf, String> {
let child = match &mut self.rec {
Rec::Script(c) => c,
Rec::Mic(m) => {
return match m.finish() {
Some("fail") => Err("找不到可以用的麥克風。".into()),
Some("nothing") => Err("沒有聽到說話。".into()),
None => Err("錄音沒有正常結束。".into()),
Some(_) if std::fs::metadata(&self.file).map(|m| m.len() > 4000).unwrap_or(false) => Ok(self.file.clone()),
Some(_) => Err("沒有錄到聲音。".into()),
};
}
};
let _ = std::fs::write(&self.stop, b"");
let t = Instant::now();
let status = loop {
if let Ok(Some(_)) = child.try_wait() {
break true;
}
if t.elapsed() > Duration::from_secs(4) {
let _ = child.kill();
break false;
}
std::thread::sleep(Duration::from_millis(50));
};
let _ = std::fs::remove_file(&self.stop);
let mut said = String::new();
if let Some(mut o) = child.stdout.take() {
use std::io::Read;
let _ = o.read_to_string(&mut said);
}
match said.trim() {
"denied" => {
let _ = crate::computer::spawn_reaped(&mut Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone"));
Err("macOS 還沒讓 AnyBot 使用麥克風。我已經打開「系統設定 → 隱私權與安全性 → 麥克風」:把 AnyBot(或「終端機」)打開,再試一次。".into())
}
"fail" => Err("找不到可以用的麥克風。".into()),
"nothing" => Err("沒有聽到說話。".into()),
_ if !status => Err("錄音沒有正常結束。".into()),
_ => {
let ok = std::fs::metadata(&self.file).map(|m| m.len() > 1000).unwrap_or(false);
if ok {
Ok(self.file.clone())
} else {
Err("沒有錄到聲音。".into())
}
}
}
}
pub fn cancel(mut self) {
let _ = std::fs::write(&self.stop, b"");
match &mut self.rec {
Rec::Script(c) => {
std::thread::sleep(Duration::from_millis(300));
let _ = c.kill();
}
Rec::Mic(m) => {
let _ = m.finish();
}
}
let _ = std::fs::remove_file(&self.stop);
let _ = std::fs::remove_file(&self.file);
}
}
/// Same, telling the service the spoken language (ISO 639-1, e.g. "zh"; empty = detect).
pub fn multipart_lang(model: &str, file_name: &str, audio: &[u8], boundary: &str, lang: &str) -> Vec<u8> {
let mut b = vec![];
let mut fields = vec![("model", model), ("response_format", "json")];
if !lang.is_empty() {
fields.push(("language", lang));
}
for (k, v) in fields {
b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n").as_bytes());
}
let kind = if file_name.ends_with(".wav") { "audio/wav" } else { "audio/mp4" };
b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{file_name}\"\r\nContent-Type: {kind}\r\n\r\n").as_bytes());
b.extend_from_slice(audio);
b.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
b
}
/// The spoken languages offered for voice chat (the original: 語言 Auto-detect…): (code, name).
pub const SPOKEN: [(&str, &str); 6] = [("", "自動偵測"), ("zh", "中文"), ("en", "English"), ("ja", "日本語"), ("ko", "한국어"), ("es", "Español")];
pub fn transcribe_lang(base_url: &str, api_key: &str, file: &std::path::Path, lang: &str) -> Result<String, String> {
if api_key.trim().is_empty() {
return Err("語音轉文字要用 OpenAI 的金鑰:請到「設定 → AI 供應商 → OpenAI」填入。".into());
}
let audio = std::fs::read(file).map_err(|e| e.to_string())?;
let boundary = format!("anybot{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0));
let url = format!("{}/audio/transcriptions", base_url.trim_end_matches('/'));
let agent: ureq::Agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(120))).http_status_as_error(false).build().into();
let mut last = String::new();
for model in MODELS {
let name = if file.extension().and_then(|e| e.to_str()) == Some("wav") { "voice.wav" } else { "voice.m4a" };
let body = multipart_lang(model, name, &audio, &boundary, lang);
let resp = agent
.post(&url)
.header("Authorization", &format!("Bearer {}", api_key.trim()))
.header("Content-Type", &format!("multipart/form-data; boundary={boundary}"))
.send(&body[..]);
match resp {
Ok(mut r) => {
let status = r.status().as_u16();
let text = r.body_mut().read_to_string().unwrap_or_default();
if status == 200 {
let v: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
return Ok(v["text"].as_str().unwrap_or("").trim().to_string());
}
last = format!("轉文字失敗(HTTP {status}):{}", text.chars().take(200).collect::<String>());
// An unknown model: try the next one.
if status != 404 && status != 400 {
break;
}
}
Err(e) => {
last = format!("連不上語音轉文字服務:{e}");
break;
}
}
}
Err(last)
}
/// The voices installed on this Mac: (name, language code), Chinese and English first.
pub fn voices() -> Vec<(String, String)> {
static V: std::sync::OnceLock<Vec<(String, String)>> = std::sync::OnceLock::new();
V.get_or_init(|| {
if !cfg!(target_os = "macos") {
let out = Command::new("espeak-ng").arg("--voices").output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
return parse_espeak_voices(&out);
}
let out = Command::new("say").args(["-v", "?"]).output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
parse_voices(&out)
})
.clone()
}
pub fn parse_voices(out: &str) -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = out
.lines()
.filter_map(|l| {
let head = l.split('#').next()?.trim_end();
let (name, lang) = head.rsplit_once(char::is_whitespace)?;
let lang = lang.trim();
lang.contains('_').then(|| (name.trim().to_string(), lang.to_string()))
})
.filter(|(n, _)| !n.is_empty())
.collect();
let rank = |l: &str| if l.starts_with("zh_TW") { 0 } else if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 };
v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0)));
v
}
/// `espeak-ng --voices` (Linux): "Pty Language Age/Gender VoiceName File Other" →
/// (language code to pass to -v, language as zh_CN / en_US).
pub fn parse_espeak_voices(out: &str) -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = out
.lines()
.skip(1)
.filter_map(|l| {
let lang = l.split_whitespace().nth(1)?.to_string();
let norm = match lang.as_str() {
"cmn" => "zh_CN".to_string(),
"yue" => "zh_HK".to_string(),
"cmn-latn-pinyin" => return None,
l => {
let mut parts = l.splitn(2, '-');
let a = parts.next().unwrap_or("");
match parts.next() {
Some(b) => format!("{a}_{}", b.to_uppercase()),
None => format!("{a}_{}", a.to_uppercase()),
}
}
};
Some((lang, norm))
})
.collect();
let rank = |l: &str| if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 };
v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0)));
// One entry per language (espeak-ng lists some twice, e.g. yue with and without Jyutping).
v.dedup_by(|a, b| a.0 == b.0);
v
}
// ---------- 語音聊天: reading answers aloud ----------
static SPEAKER: std::sync::Mutex<Option<Child>> = std::sync::Mutex::new(None);
/// Markdown → words worth saying.
pub fn speakable(text: &str) -> String {
let mut out = String::new();
let mut in_code = false;
for line in text.lines() {
let l = line.trim();
if l.starts_with("```") {
in_code = !in_code;
if in_code {
out.push_str("(程式碼略過)\n");
}
continue;
}
if in_code {
continue;
}
let l = l.trim_start_matches(['#', '-', '*', '>', ' ']).replace("**", "").replace('`', "");
let l = l.split("](").next().unwrap_or(&l).replace('[', "");
if !l.is_empty() {
out.push_str(&l);
out.push('\n');
}
}
out.trim().to_string()
}
/// The voice for the text: Taiwanese Mandarin (美佳) when it has Chinese, else the system voice.
pub fn voice_for(text: &str) -> Option<&'static str> {
text.chars().any(|c| ('\u{4e00}'..='\u{9fff}').contains(&c)).then_some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" })
}
/// Words per minute of the system voices at 1x (`say`'s usual rate).
pub const BASE_RATE: f32 = 175.0;
/// The speeds offered (the original: 速度 1x…).
pub const SPEEDS: [f32; 5] = [0.75, 1.0, 1.25, 1.5, 2.0];
/// Read aloud with a chosen voice ("" = pick by language) and speed (1.0 = normal).
pub fn speak_as(text: &str, voice: &str, rate: f32) {
stop_speaking();
let words = speakable(text);
if words.is_empty() {
return;
}
// Linux: espeak-ng takes the same -v (voice) and -s (words per minute, 175 by default).
let mut c = Command::new(if cfg!(target_os = "macos") { "say" } else { "espeak-ng" });
if !voice.trim().is_empty() {
c.args(["-v", voice.trim()]);
} else if let Some(v) = voice_for(&words) {
c.args(["-v", v]);
}
if (rate - 1.0).abs() > 0.01 {
c.args([if cfg!(target_os = "macos") { "-r" } else { "-s" }, &format!("{}", (BASE_RATE * rate.clamp(0.5, 3.0)).round())]);
}
if let Ok(child) = c.arg(&words).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() {
*SPEAKER.lock().unwrap() = Some(child);
}
}
pub fn stop_speaking() {
if let Some(mut c) = SPEAKER.lock().unwrap().take() {
let _ = c.kill();
}
}
pub fn speaking() -> bool {
let mut g = SPEAKER.lock().unwrap();
match g.as_mut().map(|c| c.try_wait()) {
Some(Ok(None)) => true,
_ => {
*g = None;
false
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
#[test]
fn answers_are_cleaned_for_speaking() {
let t = "## 行程\n- **上午**:赤崁樓\n```\nls -la\n```\n看 [地圖](https://x.y)";
assert_eq!(speakable(t), "行程\n上午:赤崁樓\n(程式碼略過)\n看 地圖");
assert_eq!(voice_for("你好"), Some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" }));
let e = parse_espeak_voices("Pty Language Age/Gender VoiceName File Other Languages\n 5 cmn --/M Chinese_(Mandarin) sit/cmn (zh-cmn 5)(zh 5)\n 2 en-us --/M English_(America) gmw/en-US (en 3)\n");
assert_eq!(e[0], ("cmn".to_string(), "zh_CN".to_string()));
assert_eq!(e[1], ("en-us".to_string(), "en_US".to_string()));
let v = parse_voices("Samantha en_US # Hello\nMeijia zh_TW # 你好\nGrandma (Chinese (Taiwan)) zh_TW # 你好\n");
assert_eq!(v[0].1, "zh_TW");
assert!(v.iter().any(|x| x.0 == "Grandma (Chinese (Taiwan))"));
assert_eq!(v.last().unwrap().0, "Samantha");
let body = String::from_utf8_lossy(&multipart_lang("m", "a.m4a", b"x", "B", "zh")).to_string();
assert!(body.contains("name=\"language\"\r\n\r\nzh"));
assert_eq!(voice_for("hello"), None);
}
#[test]
fn transcription_request_is_well_formed() {
let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = server.local_addr().unwrap().port();
let seen = std::thread::spawn(move || {
let (mut s, _) = server.accept().unwrap();
s.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
let mut buf = vec![0u8; 65536];
let mut got = vec![];
while let Ok(n) = s.read(&mut buf) {
if n == 0 {
break;
}
got.extend_from_slice(&buf[..n]);
if got.windows(4).any(|w| w == b"--\r\n") {
break;
}
}
let reply = "{\"text\":\"你好,幫我排行程\"}";
let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes());
String::from_utf8_lossy(&got).to_string()
});
let f = std::env::temp_dir().join(format!("anybot-voice-test-{}.m4a", std::process::id()));
std::fs::write(&f, vec![7u8; 2048]).unwrap();
let text = transcribe_lang(&format!("http://127.0.0.1:{port}/v1"), "sk-test", &f, "").unwrap();
let req = seen.join().unwrap();
let _ = std::fs::remove_file(&f);
assert_eq!(text, "你好,幫我排行程");
assert!(req.starts_with("POST /v1/audio/transcriptions"));
assert!(req.contains("Bearer sk-test") && req.contains("name=\"model\"\r\n\r\ngpt-4o-mini-transcribe") && req.contains("filename=\"voice.m4a\""));
assert!(transcribe_lang("http://127.0.0.1:1", "", &std::path::PathBuf::from("/x"), "").unwrap_err().contains("OpenAI"));
}
}