411 lines
18 KiB
Rust
411 lines
18 KiB
Rust
// Attachments in the window: the chips above the message box, the ＋ button, dropping files on
|
||
// the window, pasting a picture with ⌘V, and the thumbnails in sent messages.
|
||
|
||
use super::*;
|
||
use crate::attach::{self, Attachment};
|
||
|
||
#[derive(Default)]
|
||
pub struct AttachUi {
|
||
pub pending: HashMap<u64, Vec<Attachment>>,
|
||
picked: Option<Receiver<(u64, Vec<PathBuf>)>>,
|
||
pub(super) thumbs: HashMap<String, Option<egui::TextureHandle>>,
|
||
pub error: Option<String>,
|
||
voice: Option<(u64, crate::voice::Recording)>,
|
||
voice_rx: Option<Receiver<(u64, Result<String, String>)>>,
|
||
/// The recording is a meeting (longer; becomes a transcript to summarize).
|
||
pub meeting: bool,
|
||
/// 通話模式: the bot being talked to, and where the call is.
|
||
pub call: Option<(u64, CallPhase)>,
|
||
call_rec: Option<crate::voice::Recording>,
|
||
call_rx: Option<Receiver<Result<String, String>>>,
|
||
}
|
||
|
||
#[derive(Clone, Copy, PartialEq, Debug)]
|
||
pub enum CallPhase {
|
||
Listening,
|
||
Transcribing,
|
||
Thinking,
|
||
Speaking,
|
||
}
|
||
|
||
impl CallPhase {
|
||
pub fn label(self) -> &'static str {
|
||
match self {
|
||
CallPhase::Listening => "正在聽…",
|
||
CallPhase::Transcribing => "聽懂中…",
|
||
CallPhase::Thinking => "想一下…",
|
||
CallPhase::Speaking => "正在說…",
|
||
}
|
||
}
|
||
}
|
||
|
||
impl App {
|
||
fn add_attachment_paths(&mut self, bot_id: u64, paths: Vec<PathBuf>) {
|
||
for path in paths {
|
||
match attach::store(bot_id, &path) {
|
||
Ok(a) => self.att.pending.entry(bot_id).or_default().push(a),
|
||
Err(e) => self.att.error = Some(e),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Files dropped on the window, the picker's answer, and ⌘V with a picture on the clipboard.
|
||
pub(super) fn attachments_poll(&mut self, ctx: &egui::Context, bot_id: u64, composer_focused: bool) {
|
||
let dropped: Vec<PathBuf> = ctx.input(|i| i.raw.dropped_files.iter().map(|f| f.path().to_path_buf()).filter(|p| !p.as_os_str().is_empty()).collect());
|
||
if !dropped.is_empty() {
|
||
self.add_attachment_paths(bot_id, dropped);
|
||
}
|
||
if let Some(rx) = &self.att.picked {
|
||
if let Ok((id, paths)) = rx.try_recv() {
|
||
self.att.picked = None;
|
||
self.add_attachment_paths(id, paths);
|
||
}
|
||
}
|
||
// ⌘V with no text on the clipboard: maybe a picture. The window answers the paste keys
|
||
// itself and hands over the picture (no V key arrives for ⌘V); a V key with ⌘ that does
|
||
// arrive (⌘⇧V) looks at the clipboard as before.
|
||
let pasted_text = ctx.input(|i| i.events.iter().any(|e| matches!(e, egui::Event::Paste(t) if !t.is_empty())));
|
||
let pasted_image = ctx.input(|i| i.events.iter().find_map(|e| if let egui::Event::PasteImage(img) = e { Some(img.clone()) } else { None }));
|
||
let cmd_v = ctx.input(|i| i.events.iter().any(|e| matches!(e, egui::Event::Key { key: Key::V, pressed: true, modifiers, .. } if modifiers.command)));
|
||
if !composer_focused || self.demo {
|
||
return;
|
||
}
|
||
if let Some(img) = pasted_image {
|
||
let rgba: Vec<u8> = img.pixels.iter().flat_map(|c| c.to_srgba_unmultiplied()).collect();
|
||
match attach::store_pasted(bot_id, img.size, rgba) {
|
||
Ok(a) => self.att.pending.entry(bot_id).or_default().push(a),
|
||
Err(e) => self.att.error = Some(e),
|
||
}
|
||
} else if cmd_v && !pasted_text {
|
||
if let Some(a) = attach::paste_image(bot_id) {
|
||
self.att.pending.entry(bot_id).or_default().push(a);
|
||
}
|
||
}
|
||
}
|
||
|
||
pub(super) fn pick_attachments(&mut self, ctx: &egui::Context, bot_id: u64) {
|
||
let (tx, rx) = channel();
|
||
self.att.picked = Some(rx);
|
||
let c = ctx.clone();
|
||
std::thread::spawn(move || {
|
||
let paths = attach::pick_files();
|
||
let _ = tx.send((bot_id, paths));
|
||
c.request_repaint();
|
||
});
|
||
}
|
||
|
||
/// "Drop to attach" while files hover over the window.
|
||
pub(super) fn drop_overlay(&self, ctx: &egui::Context) {
|
||
if !ctx.input(|i| !i.raw.hovered_files.is_empty()) {
|
||
return;
|
||
}
|
||
let p = self.pal;
|
||
let screen = ctx.content_rect();
|
||
let painter = ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, egui::Id::unique("drop-overlay")));
|
||
painter.rect_filled(screen, 0, p.pane.gamma_multiply(0.85));
|
||
let inner = screen.shrink(24.0);
|
||
painter.rect_stroke(inner, 18, Stroke::new(2.0, p.accent), egui::StrokeKind::Inside);
|
||
painter.text(inner.center(), egui::Align2::CENTER_CENTER, "放開以附加檔案", egui::FontId::proportional(FONT_XL), p.accent);
|
||
}
|
||
|
||
fn thumb(&mut self, ctx: &egui::Context, path: &str) -> Option<egui::TextureHandle> {
|
||
if let Some(t) = self.att.thumbs.get(path) {
|
||
return t.clone();
|
||
}
|
||
let t = image::open(path).ok().map(|img| {
|
||
let img = img.thumbnail(240, 240).to_rgba8();
|
||
let size = [img.width() as usize, img.height() as usize];
|
||
ctx.load_texture(format!("thumb-{path}"), egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw()), egui::TextureOptions::LINEAR)
|
||
});
|
||
self.att.thumbs.insert(path.to_string(), t.clone());
|
||
t
|
||
}
|
||
|
||
/// One attachment: a picture thumbnail, or a file chip. Returns true when its × was clicked.
|
||
pub(super) fn attachment_chip(&mut self, ui: &mut egui::Ui, a: &Attachment, removable: bool) -> bool {
|
||
let p = self.pal;
|
||
let ctx = ui.ctx().clone();
|
||
let mut remove = false;
|
||
let thumb = if a.image { self.thumb(&ctx, &a.path) } else { None };
|
||
let frame = Frame::new().fill(p.hover).corner_radius(10).inner_margin(if thumb.is_some() { Margin::same(3) } else { Margin::symmetric(10, 6) });
|
||
let r = frame
|
||
.show(ui, |ui| {
|
||
ui.horizontal(|ui| {
|
||
if let Some(t) = &thumb {
|
||
let s = t.size_vec2();
|
||
let k = (64.0 / s.y).min(120.0 / s.x);
|
||
ui.add(egui::Image::new((t.id(), s * k)).corner_radius(8));
|
||
} else {
|
||
let (rect, _) = ui.allocate_exact_size(vec2(14.0, 16.0), egui::Sense::hover());
|
||
let r = rect.shrink2(vec2(1.5, 1.0));
|
||
ui.painter().rect_stroke(r, 2.0, Stroke::new(1.3, p.text2), egui::StrokeKind::Middle);
|
||
ui.painter().line_segment([r.left_top() + vec2(3.0, 5.0), r.right_top() + vec2(-3.0, 5.0)], Stroke::new(1.0, p.text3));
|
||
ui.painter().line_segment([r.left_top() + vec2(3.0, 8.5), r.right_top() + vec2(-3.0, 8.5)], Stroke::new(1.0, p.text3));
|
||
ui.vertical(|ui| {
|
||
ui.spacing_mut().item_spacing.y = 0.0;
|
||
let name: String = a.name.chars().take(28).collect();
|
||
ui.label(RichText::new(name).size(FONT_XS).color(p.text));
|
||
ui.label(RichText::new(attach::human(a.bytes)).size(10.5).color(p.text3));
|
||
});
|
||
}
|
||
if removable && ui.add(egui::Button::new(RichText::new("×").size(13.0).color(p.text3)).frame(false)).on_hover_text(t("移除")).clicked() {
|
||
remove = true;
|
||
}
|
||
});
|
||
})
|
||
.response;
|
||
if !removable && r.interact(egui::Sense::click()).on_hover_text(t("打開")).clicked() {
|
||
crate::preview::open_with_system(&a.path);
|
||
}
|
||
remove
|
||
}
|
||
|
||
/// The chips above the message box.
|
||
pub(super) fn pending_chips(&mut self, ui: &mut egui::Ui, bot_id: u64) {
|
||
let list = self.att.pending.get(&bot_id).cloned().unwrap_or_default();
|
||
if list.is_empty() && self.att.error.is_none() {
|
||
return;
|
||
}
|
||
let mut remove = None;
|
||
ui.horizontal_wrapped(|ui| {
|
||
for (i, a) in list.iter().enumerate() {
|
||
if self.attachment_chip(ui, a, true) {
|
||
remove = Some(i);
|
||
}
|
||
}
|
||
});
|
||
if let Some(i) = remove {
|
||
if let Some(v) = self.att.pending.get_mut(&bot_id) {
|
||
let a = v.remove(i);
|
||
let _ = std::fs::remove_file(&a.path);
|
||
}
|
||
}
|
||
if let Some(e) = self.att.error.clone() {
|
||
ui.horizontal(|ui| {
|
||
ui.label(RichText::new(e).size(FONT_XS).color(self.pal.danger));
|
||
if ui.add(egui::Button::new(RichText::new(t("知道了")).size(FONT_XS)).frame(false)).clicked() {
|
||
self.att.error = None;
|
||
}
|
||
});
|
||
}
|
||
ui.add_space(4.0);
|
||
}
|
||
|
||
/// Take this bot's pending attachments for a message (copying them onto its own computer when
|
||
/// it's running).
|
||
pub(super) fn take_attachments(&mut self, bot_id: u64) -> Vec<Attachment> {
|
||
let mut list = self.att.pending.remove(&bot_id).unwrap_or_default();
|
||
for a in list.iter_mut() {
|
||
if let Ok(dst) = crate::cloudbox::copy_in(&a.path, bot_id, &a.name) {
|
||
a.on_box = dst;
|
||
}
|
||
}
|
||
list
|
||
}
|
||
|
||
pub(super) fn has_pending_attachments(&self, bot_id: u64) -> bool {
|
||
self.att.pending.get(&bot_id).map(|v| !v.is_empty()).unwrap_or(false)
|
||
}
|
||
}
|
||
|
||
// ---------- 語音輸入 ----------
|
||
|
||
impl App {
|
||
pub(super) fn voice_recording(&self, bot_id: u64) -> Option<Instant> {
|
||
self.att.voice.as_ref().filter(|(id, _)| *id == bot_id).map(|(_, r)| r.started)
|
||
}
|
||
|
||
pub(super) fn voice_busy(&self) -> bool {
|
||
self.att.voice_rx.is_some()
|
||
}
|
||
|
||
/// The mic button: start, or stop and turn the recording into text.
|
||
pub(super) fn voice_toggle(&mut self, ctx: &egui::Context, bot_id: u64) {
|
||
if self.voice_stop(ctx) {
|
||
return;
|
||
}
|
||
crate::voice::stop_speaking();
|
||
match crate::voice::start() {
|
||
Ok(r) => self.att.voice = Some((bot_id, r)),
|
||
Err(e) => self.att.error = Some(e),
|
||
}
|
||
}
|
||
|
||
/// Stop the recording that is running and turn it into text (false: none was running).
|
||
fn voice_stop(&mut self, ctx: &egui::Context) -> bool {
|
||
let Some((id, rec)) = self.att.voice.take() else { return false };
|
||
let lang = self.state.bot(id).map(|b| b.voice_lang.clone()).unwrap_or_default();
|
||
let ps = self.state.provider(ProviderKind::OpenAi);
|
||
let base = providers::resolve_base(ProviderKind::OpenAi, &ps.base_url);
|
||
let key = providers::resolve_key(ProviderKind::OpenAi, &ps.api_key);
|
||
let (tx, rx) = channel();
|
||
self.att.voice_rx = Some(rx);
|
||
let c = ctx.clone();
|
||
std::thread::spawn(move || {
|
||
let r = rec.finish().and_then(|f| {
|
||
let t = crate::voice::transcribe_lang(&base, &key, &f, &lang);
|
||
let _ = std::fs::remove_file(&f);
|
||
t
|
||
});
|
||
let _ = tx.send((id, r));
|
||
c.request_repaint();
|
||
});
|
||
true
|
||
}
|
||
|
||
/// 會議記錄: a long recording that becomes a transcript for the bot to summarize.
|
||
pub(super) fn start_meeting(&mut self, ctx: &egui::Context, bot_id: u64) {
|
||
if self.att.voice.is_some() {
|
||
return;
|
||
}
|
||
self.att.meeting = true;
|
||
self.voice_toggle(ctx, bot_id);
|
||
if self.att.voice.is_none() {
|
||
self.att.meeting = false;
|
||
}
|
||
}
|
||
|
||
pub(super) fn meeting_on(&self) -> bool {
|
||
self.att.meeting && self.att.voice.is_some()
|
||
}
|
||
|
||
/// Put the text where the person was typing.
|
||
pub(super) fn voice_poll(&mut self, ctx: &egui::Context) {
|
||
let max = if self.att.meeting { crate::voice::MEETING_MAX_SECS } else { crate::voice::MAX_SECS };
|
||
if self.att.voice.as_ref().is_some_and(|(_, r)| r.started.elapsed().as_secs() >= max) {
|
||
// What was recorded up to the limit is kept and turned into text like any
|
||
// recording (it was thrown away: two hours of a meeting, gone at the last second).
|
||
self.voice_stop(ctx);
|
||
self.att.error = Some(format!("錄音最長 {}，已經停止；錄到的部分正在轉成文字。", crate::voice::length_words(max)));
|
||
}
|
||
let Some(rx) = &self.att.voice_rx else { return };
|
||
if let Ok((id, r)) = rx.try_recv() {
|
||
self.att.voice_rx = None;
|
||
let meeting = std::mem::take(&mut self.att.meeting);
|
||
match r {
|
||
Ok(t) if t.is_empty() => self.att.error = Some("沒有聽到說話。".into()),
|
||
Ok(t) if meeting => {
|
||
// The transcript goes with the message as a file; the bot is asked to sum it up.
|
||
let dir = std::env::temp_dir().join("anybot-meetings");
|
||
let _ = std::fs::create_dir_all(&dir);
|
||
let file = dir.join(format!("會議記錄 {}.txt", chrono::Local::now().format("%Y-%m-%d %H%M")));
|
||
match std::fs::write(&file, &t).map_err(|e| e.to_string()).and_then(|_| crate::attach::store(id, &file)) {
|
||
Ok(a) => {
|
||
self.att.pending.entry(id).or_default().push(a);
|
||
self.drafts.insert(id, "這是剛才會議的文字記錄，請整理：重點、做了哪些決定、待辦事項（誰、做什麼、什麼時候）。".into());
|
||
}
|
||
Err(e) => self.att.error = Some(format!("存不了會議記錄：{e}")),
|
||
}
|
||
}
|
||
Ok(t) => {
|
||
let d = self.drafts.entry(id).or_default();
|
||
if !d.is_empty() && !d.ends_with([' ', '\n']) {
|
||
d.push(' ');
|
||
}
|
||
d.push_str(&t);
|
||
}
|
||
Err(e) => self.att.error = Some(e),
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------- 通話模式 ----------
|
||
|
||
impl App {
|
||
pub(super) fn call_start(&mut self, bot_id: u64) {
|
||
crate::voice::stop_speaking();
|
||
self.att.call = Some((bot_id, CallPhase::Listening));
|
||
self.att.call_rec = None;
|
||
self.att.call_rx = None;
|
||
}
|
||
|
||
pub(super) fn call_end(&mut self) {
|
||
if let Some(r) = self.att.call_rec.take() {
|
||
r.cancel();
|
||
}
|
||
crate::voice::stop_speaking();
|
||
self.att.call = None;
|
||
self.att.call_rx = None;
|
||
}
|
||
|
||
pub(super) fn call_state(&self) -> Option<(u64, CallPhase)> {
|
||
self.att.call
|
||
}
|
||
|
||
/// Listen → understand → the bot answers → it's read aloud → listen again.
|
||
pub(super) fn call_poll(&mut self, ctx: &egui::Context) {
|
||
let Some((bot_id, phase)) = self.att.call else { return };
|
||
ctx.request_repaint_after(std::time::Duration::from_millis(150));
|
||
match phase {
|
||
CallPhase::Listening => {
|
||
if self.att.call_rec.is_none() {
|
||
match crate::voice::listen() {
|
||
Ok(r) => self.att.call_rec = Some(r),
|
||
Err(e) => {
|
||
self.att.error = Some(e);
|
||
self.call_end();
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
if self.att.call_rec.as_mut().map(|r| r.done()).unwrap_or(false) {
|
||
let rec = self.att.call_rec.take().unwrap();
|
||
let lang = self.state.bot(bot_id).map(|b| b.voice_lang.clone()).unwrap_or_default();
|
||
let ps = self.state.provider(ProviderKind::OpenAi);
|
||
let base = providers::resolve_base(ProviderKind::OpenAi, &ps.base_url);
|
||
let key = providers::resolve_key(ProviderKind::OpenAi, &ps.api_key);
|
||
let (tx, rx) = channel();
|
||
self.att.call_rx = Some(rx);
|
||
let c = ctx.clone();
|
||
std::thread::spawn(move || {
|
||
let r = rec.finish().and_then(|f| {
|
||
let t = crate::voice::transcribe_lang(&base, &key, &f, &lang);
|
||
let _ = std::fs::remove_file(&f);
|
||
t
|
||
});
|
||
let _ = tx.send(r);
|
||
c.request_repaint();
|
||
});
|
||
self.att.call = Some((bot_id, CallPhase::Transcribing));
|
||
}
|
||
}
|
||
CallPhase::Transcribing => {
|
||
let Some(rx) = &self.att.call_rx else { return };
|
||
if let Ok(r) = rx.try_recv() {
|
||
self.att.call_rx = None;
|
||
match r {
|
||
Ok(t) if !t.trim().is_empty() => {
|
||
self.drafts.insert(bot_id, t);
|
||
self.send(ctx, bot_id);
|
||
self.att.call = Some((bot_id, CallPhase::Thinking));
|
||
}
|
||
// Nothing said: keep listening; a real error ends the call.
|
||
Err(e) if !e.contains("沒有聽到") => {
|
||
self.att.error = Some(e);
|
||
self.call_end();
|
||
}
|
||
_ => self.att.call = Some((bot_id, CallPhase::Listening)),
|
||
}
|
||
}
|
||
}
|
||
CallPhase::Thinking => {
|
||
if !self.streams.contains_key(&bot_id) && !self.approvals.contains_key(&bot_id) {
|
||
let answer = self.state.bot(bot_id).and_then(|b| b.messages.iter().rev().find(|m| m.role == Role::Assistant && !m.text.trim().is_empty())).map(|m| m.text.clone());
|
||
// Answers are read here, not by the 朗讀 switch (no double reading).
|
||
if let Some(a) = answer {
|
||
let (v, r) = self.state.bot(bot_id).map(|b| (b.voice.clone(), b.voice_rate)).unwrap_or_default();
|
||
crate::voice::speak_as(&a, &v, if r > 0.0 { r } else { 1.0 });
|
||
}
|
||
self.att.call = Some((bot_id, CallPhase::Speaking));
|
||
}
|
||
}
|
||
CallPhase::Speaking => {
|
||
if !crate::voice::speaking() {
|
||
self.att.call = Some((bot_id, CallPhase::Listening));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|