AnyBot/src/voice.rs

// 語音輸入: hold a short recording from the microphone, turn it into text, put it in the message
// box (the original: 聽寫 / transcribeAudio through its server). Here the recording uses macOS's
// own AVAudioRecorder (through osascript, no extra install) and the text comes from an
// OpenAI-compatible /audio/transcriptions endpoint with the person's own key.

use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

/// Longest recording (estimated; the recorder script also stops itself at 10 minutes).
pub const MAX_SECS: u64 = 600;
/// 會議記錄 can run longer: two hours stays under the transcription service's 25 MB limit at this
/// recorder's bitrate (estimated).
pub const MEETING_MAX_SECS: u64 = 7_200;
/// Transcription models tried in order.
pub const MODELS: [&str; 2] = ["gpt-4o-mini-transcribe", "whisper-1"];

const RECORDER: &str = r#"ObjC.import('AVFoundation');
ObjC.import('Foundation');
function run(argv) {
  const url = $.NSURL.fileURLWithPath(argv[0]);
  const settings = $.NSDictionary.dictionaryWithObjectsForKeys(
    [$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)],
    [$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]);
  const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null);
  if (rec.isNil()) return 'fail';
  if (!rec.record) return 'denied';
  const fm = $.NSFileManager.defaultManager;
  for (let i = 0; i < 72000; i++) { if (fm.fileExistsAtPath(argv[1])) break; delay(0.1); }
  rec.stop;
  return 'ok';
}"#;

/// 通話模式: listens until the person stops talking (AVAudioRecorder metering).
/// Speech is louder than SPEECH_DB; SILENCE_TICKS × 0.1 s of quiet after speech ends a turn;
/// nothing heard for WAIT_TICKS × 0.1 s gives up (all estimated).
const SPEECH_DB: f64 = -38.0;
const SILENCE_TICKS: u32 = 12;
const WAIT_TICKS: u32 = 300;
const LISTENER: &str = r#"ObjC.import('AVFoundation');
ObjC.import('Foundation');
function run(argv) {
  const url = $.NSURL.fileURLWithPath(argv[0]);
  const settings = $.NSDictionary.dictionaryWithObjectsForKeys(
    [$.NSNumber.numberWithInt(1633772320), $.NSNumber.numberWithDouble(16000.0), $.NSNumber.numberWithInt(1)],
    [$('AVFormatIDKey'), $('AVSampleRateKey'), $('AVNumberOfChannelsKey')]);
  const rec = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, null);
  if (rec.isNil()) return 'fail';
  rec.meteringEnabled = true;
  if (!rec.record) return 'denied';
  const fm = $.NSFileManager.defaultManager;
  const speech = Number(argv[2]), quietMax = Number(argv[3]), waitMax = Number(argv[4]);
  let heard = false, quiet = 0;
  for (let i = 0; i < 6000; i++) {
    if (fm.fileExistsAtPath(argv[1])) break;
    delay(0.1);
    rec.updateMeters;
    const p = rec.averagePowerForChannel(0);
    if (p > speech) { heard = true; quiet = 0; } else if (heard) { quiet++; }
    if (heard && quiet >= quietMax) break;
    if (!heard && i >= waitMax) { rec.stop; return 'nothing'; }
  }
  rec.stop;
  return heard ? 'ok' : 'nothing';
}"#;

/// The recorder: macOS's AVAudioRecorder script, or cpal (mic.rs) elsewhere.
enum Rec {
    Script(Child),
    Mic(crate::mic::Recorder),
}

pub struct Recording {
    rec: Rec,
    pub file: PathBuf,
    stop: PathBuf,
    pub started: Instant,
}

pub fn start() -> Result<Recording, String> {
    if !cfg!(target_os = "macos") {
        let file = std::env::temp_dir().join(format!("anybot-voice-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis()));
        let rec = crate::mic::start(file.clone(), None, MEETING_MAX_SECS);
        return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() });
    }
    let base = std::env::temp_dir().join(format!("anybot-voice-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis()));
    let file = base.with_extension("m4a");
    let stop = base.with_extension("stop");
    let child = Command::new("osascript")
        .args(["-l", "JavaScript", "-e", RECORDER, "--"])
        .arg(&file)
        .arg(&stop)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .spawn()
        .map_err(|e| format!("無法開始錄音：{e}"))?;
    Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() })
}

/// Listen for one spoken turn: stops by itself when the person goes quiet.
pub fn listen() -> Result<Recording, String> {
    if !cfg!(target_os = "macos") {
        let file = std::env::temp_dir().join(format!("anybot-call-{}-{}.wav", std::process::id(), chrono::Utc::now().timestamp_millis()));
        let rec = crate::mic::start(file.clone(), Some(crate::mic::Listen { speech_db: SPEECH_DB, quiet_ticks: SILENCE_TICKS, wait_ticks: WAIT_TICKS }), MAX_SECS);
        return Ok(Recording { rec: Rec::Mic(rec), stop: file.with_extension("stop"), file, started: Instant::now() });
    }
    let base = std::env::temp_dir().join(format!("anybot-call-{}-{}", std::process::id(), chrono::Utc::now().timestamp_millis()));
    let (file, stop) = (base.with_extension("m4a"), base.with_extension("stop"));
    let child = Command::new("osascript")
        .args(["-l", "JavaScript", "-e", LISTENER, "--"])
        .arg(&file)
        .arg(&stop)
        .args([SPEECH_DB.to_string(), SILENCE_TICKS.to_string(), WAIT_TICKS.to_string()])
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .spawn()
        .map_err(|e| format!("無法開始聽：{e}"))?;
    Ok(Recording { rec: Rec::Script(child), file, stop, started: Instant::now() })
}

impl Recording {
    /// The listener stopped by itself (the person finished, or nobody spoke).
    pub fn done(&mut self) -> bool {
        match &mut self.rec {
            Rec::Script(c) => matches!(c.try_wait(), Ok(Some(_))),
            Rec::Mic(m) => m.done(),
        }
    }

    /// Stop and give back the audio file.
    pub fn finish(mut self) -> Result<PathBuf, String> {
        let child = match &mut self.rec {
            Rec::Script(c) => c,
            Rec::Mic(m) => {
                return match m.finish() {
                    Some("fail") => Err("找不到可以用的麥克風。".into()),
                    Some("nothing") => Err("沒有聽到說話。".into()),
                    None => Err("錄音沒有正常結束。".into()),
                    Some(_) if std::fs::metadata(&self.file).map(|m| m.len() > 4000).unwrap_or(false) => Ok(self.file.clone()),
                    Some(_) => Err("沒有錄到聲音。".into()),
                };
            }
        };
        let _ = std::fs::write(&self.stop, b"");
        let t = Instant::now();
        let status = loop {
            if let Ok(Some(_)) = child.try_wait() {
                break true;
            }
            if t.elapsed() > Duration::from_secs(4) {
                let _ = child.kill();
                break false;
            }
            std::thread::sleep(Duration::from_millis(50));
        };
        let _ = std::fs::remove_file(&self.stop);
        let mut said = String::new();
        if let Some(mut o) = child.stdout.take() {
            use std::io::Read;
            let _ = o.read_to_string(&mut said);
        }
        match said.trim() {
            "denied" => {
                let _ = Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone").spawn();
                Err("macOS 還沒讓 AnyBot 使用麥克風。我已經打開「系統設定 → 隱私權與安全性 → 麥克風」：把 AnyBot（或「終端機」）打開，再試一次。".into())
            }
            "fail" => Err("找不到可以用的麥克風。".into()),
            "nothing" => Err("沒有聽到說話。".into()),
            _ if !status => Err("錄音沒有正常結束。".into()),
            _ => {
                let ok = std::fs::metadata(&self.file).map(|m| m.len() > 1000).unwrap_or(false);
                if ok {
                    Ok(self.file.clone())
                } else {
                    Err("沒有錄到聲音。".into())
                }
            }
        }
    }

    pub fn cancel(mut self) {
        let _ = std::fs::write(&self.stop, b"");
        match &mut self.rec {
            Rec::Script(c) => {
                std::thread::sleep(Duration::from_millis(300));
                let _ = c.kill();
            }
            Rec::Mic(m) => {
                let _ = m.finish();
            }
        }
        let _ = std::fs::remove_file(&self.stop);
        let _ = std::fs::remove_file(&self.file);
    }
}

/// multipart/form-data body for /audio/transcriptions.
#[allow(dead_code)]
pub fn multipart(model: &str, file_name: &str, audio: &[u8], boundary: &str) -> Vec<u8> {
    multipart_lang(model, file_name, audio, boundary, "")
}

/// Same, telling the service the spoken language (ISO 639-1, e.g. "zh"; empty = detect).
pub fn multipart_lang(model: &str, file_name: &str, audio: &[u8], boundary: &str, lang: &str) -> Vec<u8> {
    let mut b = vec![];
    let mut fields = vec![("model", model), ("response_format", "json")];
    if !lang.is_empty() {
        fields.push(("language", lang));
    }
    for (k, v) in fields {
        b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n").as_bytes());
    }
    let kind = if file_name.ends_with(".wav") { "audio/wav" } else { "audio/mp4" };
    b.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{file_name}\"\r\nContent-Type: {kind}\r\n\r\n").as_bytes());
    b.extend_from_slice(audio);
    b.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
    b
}

/// Audio → text through `{base}/audio/transcriptions`.
#[allow(dead_code)]
pub fn transcribe(base_url: &str, api_key: &str, file: &std::path::Path) -> Result<String, String> {
    transcribe_lang(base_url, api_key, file, "")
}

/// The spoken languages offered for voice chat (the original: 語言 Auto-detect…): (code, name).
pub const SPOKEN: [(&str, &str); 6] = [("", "自動偵測"), ("zh", "中文"), ("en", "English"), ("ja", "日本語"), ("ko", "한국어"), ("es", "Español")];

pub fn transcribe_lang(base_url: &str, api_key: &str, file: &std::path::Path, lang: &str) -> Result<String, String> {
    if api_key.trim().is_empty() {
        return Err("語音轉文字要用 OpenAI 的金鑰：請到「設定 → AI 供應商 → OpenAI」填入。".into());
    }
    let audio = std::fs::read(file).map_err(|e| e.to_string())?;
    let boundary = format!("anybot{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0));
    let url = format!("{}/audio/transcriptions", base_url.trim_end_matches('/'));
    let agent: ureq::Agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(120))).http_status_as_error(false).build().into();
    let mut last = String::new();
    for model in MODELS {
        let name = if file.extension().and_then(|e| e.to_str()) == Some("wav") { "voice.wav" } else { "voice.m4a" };
        let body = multipart_lang(model, name, &audio, &boundary, lang);
        let resp = agent
            .post(&url)
            .header("Authorization", &format!("Bearer {}", api_key.trim()))
            .header("Content-Type", &format!("multipart/form-data; boundary={boundary}"))
            .send(&body[..]);
        match resp {
            Ok(mut r) => {
                let status = r.status().as_u16();
                let text = r.body_mut().read_to_string().unwrap_or_default();
                if status == 200 {
                    let v: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
                    return Ok(v["text"].as_str().unwrap_or("").trim().to_string());
                }
                last = format!("轉文字失敗（HTTP {status}）：{}", text.chars().take(200).collect::<String>());
                // An unknown model: try the next one.
                if status != 404 && status != 400 {
                    break;
                }
            }
            Err(e) => {
                last = format!("連不上語音轉文字服務：{e}");
                break;
            }
        }
    }
    Err(last)
}

/// The voices installed on this Mac: (name, language code), Chinese and English first.
pub fn voices() -> Vec<(String, String)> {
    static V: std::sync::OnceLock<Vec<(String, String)>> = std::sync::OnceLock::new();
    V.get_or_init(|| {
        if !cfg!(target_os = "macos") {
            let out = Command::new("espeak-ng").arg("--voices").output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
            return parse_espeak_voices(&out);
        }
        let out = Command::new("say").args(["-v", "?"]).output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
        parse_voices(&out)
    })
    .clone()
}

pub fn parse_voices(out: &str) -> Vec<(String, String)> {
    let mut v: Vec<(String, String)> = out
        .lines()
        .filter_map(|l| {
            let head = l.split('#').next()?.trim_end();
            let (name, lang) = head.rsplit_once(char::is_whitespace)?;
            let lang = lang.trim();
            lang.contains('_').then(|| (name.trim().to_string(), lang.to_string()))
        })
        .filter(|(n, _)| !n.is_empty())
        .collect();
    let rank = |l: &str| if l.starts_with("zh_TW") { 0 } else if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 };
    v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0)));
    v
}

/// `espeak-ng --voices` (Linux): "Pty Language Age/Gender VoiceName File Other" →
/// (language code to pass to -v, language as zh_CN / en_US).
pub fn parse_espeak_voices(out: &str) -> Vec<(String, String)> {
    let mut v: Vec<(String, String)> = out
        .lines()
        .skip(1)
        .filter_map(|l| {
            let lang = l.split_whitespace().nth(1)?.to_string();
            let norm = match lang.as_str() {
                "cmn" => "zh_CN".to_string(),
                "yue" => "zh_HK".to_string(),
                "cmn-latn-pinyin" => return None,
                l => {
                    let mut parts = l.splitn(2, '-');
                    let a = parts.next().unwrap_or("");
                    match parts.next() {
                        Some(b) => format!("{a}_{}", b.to_uppercase()),
                        None => format!("{a}_{}", a.to_uppercase()),
                    }
                }
            };
            Some((lang, norm))
        })
        .collect();
    let rank = |l: &str| if l.starts_with("zh") { 1 } else if l.starts_with("en") { 2 } else if l.starts_with("ja") { 3 } else { 4 };
    v.sort_by(|a, b| rank(&a.1).cmp(&rank(&b.1)).then(a.0.cmp(&b.0)));
    // One entry per language (espeak-ng lists some twice, e.g. yue with and without Jyutping).
    v.dedup_by(|a, b| a.0 == b.0);
    v
}

// ---------- 語音聊天: reading answers aloud ----------

static SPEAKER: std::sync::Mutex<Option<Child>> = std::sync::Mutex::new(None);

/// Markdown → words worth saying.
pub fn speakable(text: &str) -> String {
    let mut out = String::new();
    let mut in_code = false;
    for line in text.lines() {
        let l = line.trim();
        if l.starts_with("```") {
            in_code = !in_code;
            if in_code {
                out.push_str("（程式碼略過）\n");
            }
            continue;
        }
        if in_code {
            continue;
        }
        let l = l.trim_start_matches(['#', '-', '*', '>', ' ']).replace("**", "").replace('`', "");
        let l = l.split("](").next().unwrap_or(&l).replace('[', "");
        if !l.is_empty() {
            out.push_str(&l);
            out.push('\n');
        }
    }
    out.trim().to_string()
}

/// The voice for the text: Taiwanese Mandarin (美佳) when it has Chinese, else the system voice.
pub fn voice_for(text: &str) -> Option<&'static str> {
    text.chars().any(|c| ('\u{4e00}'..='\u{9fff}').contains(&c)).then_some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" })
}

/// Read an answer aloud (stops whatever was being read).
#[allow(dead_code)]
pub fn speak(text: &str) {
    speak_as(text, "", 1.0);
}

/// Words per minute of the system voices at 1x (`say`'s usual rate).
pub const BASE_RATE: f32 = 175.0;

/// The speeds offered (the original: 速度 1x…).
pub const SPEEDS: [f32; 5] = [0.75, 1.0, 1.25, 1.5, 2.0];

/// Read aloud with a chosen voice ("" = pick by language) and speed (1.0 = normal).
pub fn speak_as(text: &str, voice: &str, rate: f32) {
    stop_speaking();
    let words = speakable(text);
    if words.is_empty() {
        return;
    }
    // Linux: espeak-ng takes the same -v (voice) and -s (words per minute, 175 by default).
    let mut c = Command::new(if cfg!(target_os = "macos") { "say" } else { "espeak-ng" });
    if !voice.trim().is_empty() {
        c.args(["-v", voice.trim()]);
    } else if let Some(v) = voice_for(&words) {
        c.args(["-v", v]);
    }
    if (rate - 1.0).abs() > 0.01 {
        c.args([if cfg!(target_os = "macos") { "-r" } else { "-s" }, &format!("{}", (BASE_RATE * rate.clamp(0.5, 3.0)).round())]);
    }
    if let Ok(child) = c.arg(&words).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() {
        *SPEAKER.lock().unwrap() = Some(child);
    }
}

pub fn stop_speaking() {
    if let Some(mut c) = SPEAKER.lock().unwrap().take() {
        let _ = c.kill();
    }
}

pub fn speaking() -> bool {
    let mut g = SPEAKER.lock().unwrap();
    match g.as_mut().map(|c| c.try_wait()) {
        Some(Ok(None)) => true,
        _ => {
            *g = None;
            false
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::{Read, Write};

    #[test]
    fn answers_are_cleaned_for_speaking() {
        let t = "## 行程\n- **上午**：赤崁樓\n```\nls -la\n```\n看 [地圖](https://x.y)";
        assert_eq!(speakable(t), "行程\n上午：赤崁樓\n（程式碼略過）\n看 地圖");
        assert_eq!(voice_for("你好"), Some(if cfg!(target_os = "macos") { "Meijia" } else { "cmn" }));
        let e = parse_espeak_voices("Pty Language       Age/Gender VoiceName          File                 Other Languages\n 5  cmn              --/M      Chinese_(Mandarin) sit/cmn              (zh-cmn 5)(zh 5)\n 2  en-us            --/M      English_(America)  gmw/en-US            (en 3)\n");
        assert_eq!(e[0], ("cmn".to_string(), "zh_CN".to_string()));
        assert_eq!(e[1], ("en-us".to_string(), "en_US".to_string()));
        let v = parse_voices("Samantha            en_US    # Hello\nMeijia              zh_TW    # 你好\nGrandma (Chinese (Taiwan)) zh_TW    # 你好\n");
        assert_eq!(v[0].1, "zh_TW");
        assert!(v.iter().any(|x| x.0 == "Grandma (Chinese (Taiwan))"));
        assert_eq!(v.last().unwrap().0, "Samantha");
        let body = String::from_utf8_lossy(&multipart_lang("m", "a.m4a", b"x", "B", "zh")).to_string();
        assert!(body.contains("name=\"language\"\r\n\r\nzh"));
        assert_eq!(voice_for("hello"), None);
    }

    #[test]
    fn transcription_request_is_well_formed() {
        let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let port = server.local_addr().unwrap().port();
        let seen = std::thread::spawn(move || {
            let (mut s, _) = server.accept().unwrap();
            s.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
            let mut buf = vec![0u8; 65536];
            let mut got = vec![];
            while let Ok(n) = s.read(&mut buf) {
                if n == 0 {
                    break;
                }
                got.extend_from_slice(&buf[..n]);
                if got.windows(4).any(|w| w == b"--\r\n") {
                    break;
                }
            }
            let reply = "{\"text\":\"你好，幫我排行程\"}";
            let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes());
            String::from_utf8_lossy(&got).to_string()
        });
        let f = std::env::temp_dir().join(format!("anybot-voice-test-{}.m4a", std::process::id()));
        std::fs::write(&f, vec![7u8; 2048]).unwrap();
        let text = transcribe(&format!("http://127.0.0.1:{port}/v1"), "sk-test", &f).unwrap();
        let req = seen.join().unwrap();
        let _ = std::fs::remove_file(&f);
        assert_eq!(text, "你好，幫我排行程");
        assert!(req.starts_with("POST /v1/audio/transcriptions"));
        assert!(req.contains("Bearer sk-test") && req.contains("name=\"model\"\r\n\r\ngpt-4o-mini-transcribe") && req.contains("filename=\"voice.m4a\""));
        assert!(transcribe("http://127.0.0.1:1", "", &std::path::PathBuf::from("/x")).unwrap_err().contains("OpenAI"));
    }
}