add voice
This commit is contained in:
parent
0e02c2e148
commit
f8d00df017
13
.env.example
13
.env.example
|
|
@ -17,7 +17,20 @@ LAZYBOY_SECURE_COOKIE=false
|
|||
LAZYBOY_COMPUTER_MEMORY_MB=2048
|
||||
LAZYBOY_COMPUTER_CPUS=2
|
||||
LAZYBOY_COMPUTER_PIDS=2048
|
||||
# Only affects the Agent desktop container. Disabled by default.
|
||||
LAZYBOY_COMPUTER_SUDO=false
|
||||
# Linux only (optional): point this at the host's LXCFS root to make htop/free
|
||||
# report the per-Agent cgroup quota. Leave the default empty directory on macOS.
|
||||
LAZYBOY_LXCFS_ROOT=./data/lxcfs
|
||||
LAZYBOY_MEMORY_ENABLED=true
|
||||
LAZYBOY_MEMORY_MODEL_CACHE=./data/fastembed
|
||||
LAZYBOY_MEMORY_TOP_K=8
|
||||
LAZYBOY_MEMORY_BYTE_BUDGET=6000
|
||||
|
||||
# Hourly cleanup of diagnostics; conversations and current memories are retained.
|
||||
LAZYBOY_EVENT_RETENTION_DAYS=30
|
||||
LAZYBOY_CHECKPOINT_RETENTION_DAYS=7
|
||||
LAZYBOY_RUN_RETENTION_DAYS=90
|
||||
LAZYBOY_RECORDING_RETENTION_DAYS=30
|
||||
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90
|
||||
LAZYBOY_DB_WARN_MB=1024
|
||||
|
|
|
|||
|
|
@ -2251,11 +2251,20 @@ dependencies = [
|
|||
name = "lazyboy-harness"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"futures-util",
|
||||
"http",
|
||||
"lazyboy-contracts",
|
||||
"reqwest 0.12.28",
|
||||
"rig-core",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.26.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4741,7 +4750,11 @@ checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084"
|
|||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite 0.26.2",
|
||||
]
|
||||
|
||||
|
|
@ -4953,6 +4966,8 @@ dependencies = [
|
|||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.5",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1",
|
||||
"thiserror",
|
||||
"utf-8",
|
||||
|
|
|
|||
|
|
@ -41,7 +41,8 @@ tracing = "0.1"
|
|||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
||||
tokio-tungstenite = { version = "0.26", features = ["connect"] }
|
||||
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
|
||||
tokio-tungstenite = { version = "0.26", features = ["connect", "rustls-tls-native-roots"] }
|
||||
futures-util = "0.3"
|
||||
bollard = "0.18"
|
||||
hmac = "0.12"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ LazyBoy 是本機系統，不是雲端沙盒。機器要跑四件事：**Postgre
|
|||
| GPU | 不需要 | 模型走網路 API，畫面是 CPU 上的 Xvfb |
|
||||
| 網路 | 第一次建映像、拉套件需要 | 之後離線也能開 UI；聊天要模型金鑰能連外 |
|
||||
|
||||
每個 Agent 桌面都會套用 Docker 的 CPU、記憶體與 PID 上限，預設是 2 CPU、2 GB、2048 個 PID。可在 `.env` 調整 `LAZYBOY_COMPUTER_CPUS`、`LAZYBOY_COMPUTER_MEMORY_MB`、`LAZYBOY_COMPUTER_PIDS`。Linux 若安裝 LXCFS，將 `LAZYBOY_LXCFS_ROOT` 指到它的 `/var/lib/lxcfs`，容器內的 `htop`、`free` 等也會顯示 cgroup 配額；macOS 仍會套用配額，但 Docker Desktop 不提供這個 `/proc` 虛擬化。桌面容器不掛主機 Docker socket，也不使用 `privileged`；需要一般管理命令時才在 `.env` 設定 `LAZYBOY_COMPUTER_SUDO=true`，權限只在該 Agent 容器內生效。
|
||||
|
||||
預設一個 Team 電腦容器可同時掛最多 **8** 個螢幕（`TEAM_SCREEN_LIMIT`）。再開私人電腦就是再一個容器、再 2 GB。分頁開著時心跳會讓桌面保持熱機；關掉分頁約 10 分鐘後凍結（記憶體還在），約 6 小時後才真正停機。
|
||||
|
||||
---
|
||||
|
|
@ -111,6 +113,9 @@ Team：工作區共用一個家目錄，每個 bot 有自己的 `DISPLAY`（`:1`
|
|||
**教技能**
|
||||
你示範，容器內 CDP 錄「點了哪個控制項、填了什麼、去了哪一頁」，再抽幾個關鍵畫面。停下來後模型整理成意圖級 playbook，之後用普通工具在**當下畫面**找控制項，不是重播座標。密碼欄不錄。技能可匯出 JSON。
|
||||
|
||||
**Slash 指令與長目標**
|
||||
在 `data/skills/<name>/SKILL.md` 放工作區共用的唯讀技能，就能在輸入框打 `/name 參數` 執行；輸入 `/` 會顯示可用技能。`/goal` 是 harness 的持續執行模式，和錄製示範產生的 playbook 分開。選單支援方向鍵、Enter／Tab 選取、Esc 關閉。`/goal 目標` 會先規劃、執行並檢查結果，直到模型回報已驗證完成；只有需要登入、驗證碼、接管畫面或缺少必要資訊時才會停下來請你處理。缺少必要資訊時會記為等待輸入，補充訊息後接續原目標；停止按鈕可以取消。一般訊息仍維持 40 回合上限，教學技能 80 回合。
|
||||
|
||||
**記憶**
|
||||
`pgvector` + MiniLM（384 維）。只有你叫它記住、或它呼叫 `remember` 的內容會進長期記憶。密碼與 token 會被拒。清除對話不會清記憶。
|
||||
|
||||
|
|
@ -148,7 +153,7 @@ Compose 裡 supervisor **不**對主機開埠。API 在容器網路連 `supervis
|
|||
|
||||
同一 bot 已有進行中的工作時，新訊息會排隊（`queuedBehindActive`）。人正在接管時，後面的話只排隊，思考轉圈不會假裝它還在動。問候路徑會把工具表清空，從源頭避免「哈囉」去開電腦。
|
||||
|
||||
`execute_run` 每一輪：續租約 → 寫步驟文字 → 問模型 → 沒有工具就結束（技能沒過會再把畫面塞回去）→ 有工具且需要沙盒才 boot → 畫面沒變就不重複塞圖。回合上限：聊天 4、一般 40、技能 80。
|
||||
`execute_run` 每一輪：續租約 → 寫步驟文字 → 問模型 → 沒有工具就結束（技能沒過會再把畫面塞回去）→ 有工具且需要沙盒才 boot → 畫面沒變就不重複塞圖。回合上限：聊天 4、一般 40、教學技能 80；`/goal` 會持續到完成或明確需要人介入。目標執行期間，同一對話送進來的新訊息會作為下一輪的補充指示。
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -242,6 +247,7 @@ LazyBoy/
|
|||
│ ├── computer.rs boot / 凍結 / 心跳 / 螢幕租約
|
||||
│ ├── sessions.rs 對話 CRUD、送訊息、SSE
|
||||
│ ├── skills.rs 示範錄製與蒸馏
|
||||
│ ├── file_skills.rs 讀取 data/skills 下的唯讀 SKILL.md
|
||||
│ ├── schedules.rs cron
|
||||
│ ├── vault.rs 登入保險箱
|
||||
│ ├── memory.rs pgvector 記憶
|
||||
|
|
@ -257,6 +263,7 @@ LazyBoy/
|
|||
│ └── lazyboy-screen Team 額外 DISPLAY
|
||||
├── migrations/ sqlx，檔名流水號；API 啟動時自動 migrate
|
||||
├── data/homes/ 每個電腦的家目錄（bind 進容器 /home/lazyboy）
|
||||
├── data/skills/<name>/SKILL.md 工作區共用的 slash 技能（執行時掛載）
|
||||
├── tests/ 跨語言的小測試（node:test、Python）
|
||||
├── scripts/ init-env、build-computer-image、dev
|
||||
├── docker-compose.yml 正式堆疊（Postgres + supervisor + API）
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title><link rel="icon" href="/favicon.svg" type="image/svg+xml"/><link rel="icon" href="/favicon.png" type="image/png" sizes="32x32"/><link rel="apple-touch-icon" href="/apple-touch-icon.png"/></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>
|
||||
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title><link rel="icon" href="/favicon.ico" sizes="any"/><link rel="icon" href="/favicon.svg" type="image/svg+xml"/><link rel="icon" href="/favicon.png" type="image/png" sizes="32x32"/><link rel="apple-touch-icon" href="/apple-touch-icon.png"/></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>
|
||||
|
|
|
|||
|
|
@ -17,16 +17,18 @@ import searchToX from "react-useanimations/lib/searchToX";
|
|||
import { api, ApiError } from "./api";
|
||||
import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar";
|
||||
import { t, type MessageKey } from "./i18n";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings, WorkspaceSettings } from "./types";
|
||||
import { ChatMarkdown, CopyMessageButton } from "./markdown";
|
||||
import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, type CronPreset, type ScheduleItem } from "./schedule";
|
||||
import { CallOverlay, PhoneIcon } from "./call";
|
||||
import { VoiceSettingsDialog } from "./voice-settings";
|
||||
|
||||
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false};
|
||||
const SESSION_STORE="lazyboy.sessionByBot";
|
||||
const PANE_STORE="lazyboy.rightPane";
|
||||
const WORKSPACE_STORE="lazyboy.workspace";
|
||||
type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts";
|
||||
type AccountDialog="phone"|"settings"|"model"|"about"|"help"|"feedback"|null;
|
||||
type AccountDialog="phone"|"settings"|"model"|"voice"|"about"|"help"|"feedback"|null;
|
||||
function readSessionStore():Record<string,string>{try{const raw=localStorage.getItem(SESSION_STORE);return raw?JSON.parse(raw) as Record<string,string>:{}}catch{return {}}}
|
||||
function writeSessionStore(botId:string,sessionId:string){const store=readSessionStore();store[botId]=sessionId;localStorage.setItem(SESSION_STORE,JSON.stringify(store))}
|
||||
function readPaneStore():{collapsed:boolean;part:RightPart}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer"};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"||value.part==="accounts"?value.part:"computer"}}catch{return{collapsed:false,part:"computer"}}}
|
||||
|
|
@ -89,7 +91,8 @@ export function App(){
|
|||
const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null);
|
||||
const [roomToDelete,setRoomToDelete]=useState<Room|null>(null); const [mcpServers,setMcpServers]=useState<McpServer[]>([]);
|
||||
const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState<AccountDialog>(null);
|
||||
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [skillQuery,setSkillQuery]=useState(""); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState<string|null>(null);
|
||||
const [voiceSettings,setVoiceSettings]=useState<VoiceSettings|null>(null); const [callOpen,setCallOpen]=useState(false);
|
||||
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [fileSkills,setFileSkills]=useState<FileSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [skillQuery,setSkillQuery]=useState(""); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState<string|null>(null);
|
||||
const [schedules,setSchedules]=useState<ScheduleItem[]>([]); const [scheduleDraft,setScheduleDraft]=useState<{name:string;instructions:string;enabled:boolean;preset:CronPreset;id?:string;timezone?:string;threadId?:string|null}|null>(null);
|
||||
const [scheduleError,setScheduleError]=useState<string|null>(null); const [scheduleSaving,setScheduleSaving]=useState(false); const [runningScheduleId,setRunningScheduleId]=useState<string|null>(null);
|
||||
const [workspaceName,setWorkspaceName]=useState(workspaceStart.name);
|
||||
|
|
@ -124,6 +127,13 @@ export function App(){
|
|||
const savedSkills=skills.filter(skill=>skill.status==="saved");
|
||||
const skillNeedle=skillQuery.trim().toLowerCase();
|
||||
const listedSkills=skillNeedle?savedSkills.filter(skill=>skill.name.toLowerCase().includes(skillNeedle)||(skill.playbook.whenToUse||"").toLowerCase().includes(skillNeedle)||skill.goal.toLowerCase().includes(skillNeedle)):savedSkills;
|
||||
const [slashIndex,setSlashIndex]=useState(0);
|
||||
const [slashDismissed,setSlashDismissed]=useState(false);
|
||||
useEffect(()=>{setSlashIndex(0);setSlashDismissed(false)},[draft]);
|
||||
const slashToken=draft.trimStart().split(/\s/,1)[0].slice(1).toLowerCase();
|
||||
const slashSuggestions=!slashDismissed&&/^\/[^\s]*$/.test(draft.trimStart())
|
||||
? [{name:"goal",description:t("goalCommandHint"),kind:"執行模式"},...fileSkills.map(skill=>({...skill,kind:"檔案技能"}))].filter(skill=>!slashToken||skill.name.startsWith(slashToken)).slice(0,8)
|
||||
: [];
|
||||
|
||||
const loadMcp=useCallback(async()=>{setMcpServers(await api<McpServer[]>("/api/mcp-servers").catch(()=>[] as McpServer[]))},[]);
|
||||
const loadBots=useCallback(async()=>{const [next,nextRooms]=await Promise.all([api<Bot[]>("/api/bots"),api<Room[]>("/api/rooms").catch(()=>[] as Room[])]);setBots(next);setRooms(nextRooms);setActiveRoomId(id=>id&&nextRooms.some(room=>room.id===id)?id:null);setActiveId(id=>id&&next.some(b=>b.id===id)?id:null);await loadMcp()},[loadMcp]);
|
||||
|
|
@ -144,6 +154,9 @@ export function App(){
|
|||
setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null)
|
||||
},[activeId,activeRoomId,activeSessionId]);
|
||||
useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e.message)})},[loadBots]);
|
||||
useEffect(()=>{if(!voiceSettings?.enabled)setCallOpen(false)},[voiceSettings?.enabled]);
|
||||
useEffect(()=>{api<VoiceSettings>("/api/voice/settings").then(setVoiceSettings).catch(()=>setVoiceSettings(null))},[]);
|
||||
useEffect(()=>{if(authRequired)return;api<FileSkill[]>("/api/file-skills").then(setFileSkills).catch(()=>setFileSkills([]))},[authRequired]);
|
||||
useEffect(()=>{if(activeId||activeRoomId||bots.length===0)return;setActiveId(bots[0].id)},[bots,activeId,activeRoomId]);
|
||||
useEffect(()=>{setMessages([]);loadSessions().catch(e=>setError(e.message))},[loadSessions]);
|
||||
useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current="";setPendingFiles(current=>{current.forEach(file=>file.preview&&URL.revokeObjectURL(file.preview));return []})},[activeSessionId]);
|
||||
|
|
@ -205,7 +218,7 @@ export function App(){
|
|||
function addPendingFiles(list:FileList|File[]){const incoming=[...list];if(!incoming.length)return;setError(null);setPendingFiles(current=>{const next=[...current];for(const file of incoming){if(next.length>=ATTACH_MAX){setError(t("attachTooMany"));break}if(file.size>ATTACH_MAX_BYTES){setError(t("attachTooLarge"));continue}if(!attachAllowed(file)){setError(t("attachType"));continue}next.push({id:clientNonce(),file,preview:file.type.startsWith("image/")?URL.createObjectURL(file):null})}return next})}
|
||||
function removePendingFile(id:string){setPendingFiles(current=>current.filter(item=>{if(item.id===id&&item.preview)URL.revokeObjectURL(item.preview);return item.id!==id}))}
|
||||
async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();const files=pendingFiles;if((!active&&!activeRoom)||!activeSessionId||(!text&&files.length===0))return;sendingRef.current=true;setDraft("");setPendingFiles([]);historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{try{const attachments=await Promise.all(files.map(async item=>({name:item.file.name,mimeType:item.file.type,content:await readAsBase64(item.file)})));await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce(),attachments})});sentHistoryRef.current.push(text||files[0]?.file.name||"");if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();files.forEach(item=>item.preview&&URL.revokeObjectURL(item.preview));await loadSessions()}catch(error){setPendingFiles(files);throw error}})}finally{sendingRef.current=false}}
|
||||
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
|
||||
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;if(slashSuggestions.length){if(event.key==="Escape"){event.preventDefault();setSlashDismissed(true);return}if(event.key==="ArrowDown"||event.key==="ArrowUp"){event.preventDefault();setSlashIndex(index=>(index+(event.key==="ArrowDown"?1:slashSuggestions.length-1))%slashSuggestions.length);return}if(event.key==="Tab"||(event.key==="Enter"&&!event.shiftKey)){event.preventDefault();setDraft(`/${slashSuggestions[slashIndex%slashSuggestions.length].name} `);return}}const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
|
||||
function selectSession(id:string){setActiveSessionId(id);setSessionMenuOpen(false);if(sessionStoreKey)writeSessionStore(sessionStoreKey,id)}
|
||||
async function createSession(){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{const session=await api<Session>(sessionsPath,{method:"POST",body:JSON.stringify({title:t("newConversation")})});writeSessionStore(sessionStoreKey,session.id);const next=await api<Session[]>(sessionsPath);setSessions(next);setActiveSessionId(session.id);setMessages([])}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
|
||||
async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})}
|
||||
|
|
@ -230,16 +243,16 @@ export function App(){
|
|||
async function pasteClipboard(){try{pasteText(await navigator.clipboard.readText())}catch{setClipboardOpen(true)}}
|
||||
async function copyClipboard(){try{await navigator.clipboard.writeText(desktopClipboard)}catch{setError(t("clipboardWriteBlocked"))}}
|
||||
async function inbox(bot:Bot,actionName:string,groupName?:string|null){await api(`/api/bots/${bot.id}/inbox`,{method:"POST",body:JSON.stringify({action:actionName,groupName})});await loadBots()}
|
||||
function openBot(bot:Bot){setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)}
|
||||
function openRoom(room:Room){setMobileNav(false);setSessionMenuOpen(false);if(rightPart==="settings")setRightPart("computer");if(room.id===activeRoomId){if(!activeSessionId){const stored=readSessionStore()[`room:${room.id}`];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveId(null);setBusyMembers([]);setActiveRoomId(room.id)}
|
||||
function openBot(bot:Bot){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)}
|
||||
function openRoom(room:Room){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(rightPart==="settings")setRightPart("computer");if(room.id===activeRoomId){if(!activeSessionId){const stored=readSessionStore()[`room:${room.id}`];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveId(null);setBusyMembers([]);setActiveRoomId(room.id)}
|
||||
async function deleteRoom(room:Room){setRoomToDelete(null);await action(async()=>{await api(`/api/rooms/${room.id}`,{method:"DELETE"});if(activeRoomId===room.id){setActiveRoomId(null);setActiveSessionId(null);setMessages([]);setBusyMembers([])}await loadBots()})}
|
||||
function openAccount(dialog:AccountDialog){setAccountOpen(false);setAccountDialog(dialog)}
|
||||
async function logout(){setAccountOpen(false);await api("/api/session",{method:"DELETE",body:"{}"}).catch(()=>{});setBots([]);setRooms([]);setMcpServers([]);setActiveId(null);setActiveRoomId(null);setAuthRequired(true)}
|
||||
async function changeComputer(operation:"boot"|"restart"){
|
||||
async function changeComputer(operation:"boot"|"restart"|"stop"){
|
||||
const botId=paneBotId;if(!botId)return;
|
||||
const previous=computer;setScreenUrl(null);setDesktopReady(false);
|
||||
setComputer(current=>({...current,state:operation==="boot"&¤t.state==="suspended"?"suspended":"booting"}));
|
||||
try{await api(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)})}
|
||||
setComputer(current=>({...current,state:operation==="stop"?current.state:operation==="boot"&¤t.state==="suspended"?"suspended":"booting"}));
|
||||
try{const status=await api<ComputerStatus>(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)});if(currentPaneRef.current===botId)setComputer(status)}
|
||||
catch(error){
|
||||
const status=await api<ComputerStatus>(`/api/computer/${botId}/status`,{signal:AbortSignal.timeout(5_000)}).catch(()=>previous);
|
||||
if(currentPaneRef.current===botId)setComputer(status);
|
||||
|
|
@ -247,6 +260,7 @@ export function App(){
|
|||
}
|
||||
}
|
||||
const startBoot=()=>changeComputer("boot");
|
||||
const stopComputer=()=>changeComputer("stop");
|
||||
const restartComputer=()=>changeComputer("restart");
|
||||
const connecting=computer.state==="running"&&Boolean(screenUrl)&&!desktopReady;
|
||||
const overlayLabel=hudLabel(computer,connecting,handingOff);
|
||||
|
|
@ -254,6 +268,7 @@ export function App(){
|
|||
const hud=paneBot&&overlayLabel?<ComputerHud bot={paneBot} label={overlayLabel}/>:null;
|
||||
const statusMembers=workingMembers;
|
||||
const topTools=<nav className="top-tools" aria-label={t("workTools")}>
|
||||
{active&&!activeRoomId?<span className="call-entry" tabIndex={!voiceSettings?.enabled?0:undefined} title={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined} aria-label={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined}><button type="button" className={`top-tool-button ${callOpen?"call-active":""}`} title={!voiceSettings?.enabled?t("voiceDisabledHint"):voiceSettings.ready?t("call"):t("setUpVoiceToCall")} aria-label={t("call")} disabled={!activeSessionId||!voiceSettings?.enabled} onClick={()=>{if(!voiceSettings?.enabled)return;if(!voiceSettings.ready){setAccountDialog("voice");return}setCallOpen(true)}}><PhoneIcon/></button></span>:null}
|
||||
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="computer"?"active":""}`} title={t("computer")} aria-label={t("computer")} onClick={()=>openPane("computer")}><Computer/></button>
|
||||
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="accounts"?"active":""}`} title={t("accounts")} aria-label={t("accounts")} onClick={()=>openPane("accounts")} disabled={!paneBot}><svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8"><rect x="5" y="11" width="14" height="10" rx="2"/><path d="M8 11V8a4 4 0 0 1 8 0v3"/></svg></button>
|
||||
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="memory"?"active":""}`} title={t("memory")} aria-label={t("memory")} onClick={()=>openPane("memory")} disabled={!paneBot}><Brain/></button>
|
||||
|
|
@ -282,6 +297,7 @@ export function App(){
|
|||
<button type="button" role="menuitem" onClick={()=>openAccount("phone")}><Smartphone/>{t("openOnPhone")}</button>
|
||||
<button type="button" role="menuitem" onClick={()=>openAccount("settings")}><Settings/>{t("settings")}</button>
|
||||
<button type="button" role="menuitem" onClick={()=>openAccount("model")}><BotIcon/>{t("modelSettings")}</button>
|
||||
<button type="button" role="menuitem" onClick={()=>openAccount("voice")}><PhoneIcon/>{t("voiceSettings")}</button>
|
||||
<button type="button" role="menuitem" disabled={!paneBot} onClick={()=>{setAccountOpen(false);openPane("memory")}}><Brain/>{t("memory")}</button>
|
||||
<button type="button" role="menuitem" onClick={()=>openAccount("about")}><Info/>{t("about")}</button>
|
||||
<button type="button" role="menuitem" onClick={()=>openAccount("help")}><CircleHelp/>{t("helpCenter")}</button>
|
||||
|
|
@ -298,12 +314,13 @@ export function App(){
|
|||
|
||||
<main className="chat-panel">
|
||||
<header className="topbar"><button className="icon-button mobile-menu" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="#dfdfe2"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} size={32} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
|
||||
{callOpen&&voiceSettings?.enabled&&active&&activeSessionId&&!activeRoomId?<CallOverlay bot={active} sessionId={activeSessionId} takeover={computer.takeoverRequested||computer.controlHolder==="user"} onHangUp={()=>setCallOpen(false)} onTakeOver={()=>{setRightPart("computer");setRightCollapsed(false);if(active)void action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}}/>:null}
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const chips=chipBlocks(message.blocks);const hideBody=isAutoAttachCaption(message.body,files);return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""} ${files.length?"with-files":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}{files.length>0&&<div className="msg-attachments">{files.map(file=><FileCard key={file.name} file={file}/>)}</div>}{chips.map((chip,index)=>chip.kind==="login"?<div className="login-chip" key={`${message.id}-login-${index}`}><div className="login-label">{t("loginNeedsYou")}</div><div className="login-site">{chip.site||message.body}</div>{chip.why?<div className="login-why">{t("loginWhy",{why:chip.why})}</div>:null}<button type="button" className="primary" onClick={()=>void openLoginScreen()}>{t("loginOpenScreen")}</button></div>:chip.kind==="schedule"?<div className="sched-chip" key={`${message.id}-sched-${index}`}><div className="sched-label">{t("scheduleChip")}</div><strong>{chip.name}</strong><small>{chip.human}</small></div>:<div className="sched-chip" key={`${message.id}-run-${index}`}><div className="sched-label">{t("scheduleRunChip")}</div><strong>{chip.name}</strong><small>{chip.human}</small></div>)}{!hideBody&&(message.role==="assistant"?<div className="message-stack"><div className="message-body md"><ChatMarkdown>{message.body}</ChatMarkdown></div>{message.body.trim()?<CopyMessageButton text={message.body}/>:null}</div>:<span className="message-body">{message.body}</span>)}{!hideBody&&message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
|
||||
{otherSessionBusy&&<div className="queue-hint">{t("anotherConversationQueued")}</div>}
|
||||
<div className={`composer-dock ${statusMembers.length?"has-status":""}`}>
|
||||
{statusMembers.map(member=>{const step=computer.busyStep&&member.id===computer.botId?computer.busyStep:null;const transition=isTransitionStep(step);const label=t("working",{name:member.name});return <div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-copy"><span className="working-label">{label}</span>{!transition&&step?<span className="working-step">{step}</span>:null}</span></div>})}
|
||||
<form className={`composer ${pendingFiles.length?"has-files":""}`} onSubmit={send} onDragOver={event=>{event.preventDefault()}} onDrop={event=>{event.preventDefault();if(event.dataTransfer.files.length)addPendingFiles(event.dataTransfer.files)}}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled={!activeSessionId||Boolean(teaching)} title={t("attachFileHint")} onClick={()=>{setPlusOpen(false);attachRef.current?.click()}}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button><button type="button" role="menuitem" disabled={!active} title={t("importSkillHint")} onClick={()=>{setPlusOpen(false);importRef.current?.click()}}><Upload/>{t("importSkill")}</button>{savedSkills.length>0&&<><hr/><div className="plus-menu-skills"><small className="plus-menu-label">{t("taughtSkills")}{savedSkills.length>5?` · ${savedSkills.length}`:""}</small>{savedSkills.length>=6&&<input className="plus-menu-search" value={skillQuery} onChange={e=>setSkillQuery(e.target.value)} placeholder={t("searchSkills")} aria-label={t("searchSkills")} onClick={e=>e.stopPropagation()}/>}<div className="plus-menu-skill-list">{listedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("exportSkillHint")} aria-label={t("exportSkill")} onClick={()=>downloadSkill(skill.name,skill.goal,skill.playbook)}><Download/></button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}{listedSkills.length===0&&<small className="plus-menu-empty">{t("noMatchingSkills")}</small>}</div></div></>}</div>}</div><input ref={importRef} className="skill-import-input" type="file" accept="application/json,.json" tabIndex={-1} aria-hidden="true" onChange={event=>{const file=event.target.files?.[0];event.currentTarget.value="";if(file)void importSkillFile(file)}}/><input ref={attachRef} className="skill-import-input attach-input" type="file" multiple accept={ATTACH_ACCEPT} tabIndex={-1} aria-hidden="true" onChange={event=>{const files=[...event.target.files||[]];event.currentTarget.value="";if(files.length)addPendingFiles(files)}}/>{pendingFiles.length>0&&<div className="composer-files">{pendingFiles.map(item=><FileCard key={item.id} file={{name:item.file.name,size:item.file.size}} preview={item.preview} onRemove={()=>removePendingFile(item.id)}/>)}</div>}<textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} onPaste={event=>{const files=event.clipboardData?.files;if(files&&files.length){event.preventDefault();addPendingFiles(files)}}} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||(!draft.trim()&&pendingFiles.length===0)||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
<form className={`composer ${pendingFiles.length?"has-files":""}`} onSubmit={send} onDragOver={event=>{event.preventDefault()}} onDrop={event=>{event.preventDefault();if(event.dataTransfer.files.length)addPendingFiles(event.dataTransfer.files)}}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled={!activeSessionId||Boolean(teaching)} title={t("attachFileHint")} onClick={()=>{setPlusOpen(false);attachRef.current?.click()}}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button><button type="button" role="menuitem" disabled={!active} title={t("importSkillHint")} onClick={()=>{setPlusOpen(false);importRef.current?.click()}}><Upload/>{t("importSkill")}</button>{savedSkills.length>0&&<><hr/><div className="plus-menu-skills"><small className="plus-menu-label">{t("taughtSkills")}{savedSkills.length>5?` · ${savedSkills.length}`:""}</small>{savedSkills.length>=6&&<input className="plus-menu-search" value={skillQuery} onChange={e=>setSkillQuery(e.target.value)} placeholder={t("searchSkills")} aria-label={t("searchSkills")} onClick={e=>e.stopPropagation()}/>}<div className="plus-menu-skill-list">{listedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("exportSkillHint")} aria-label={t("exportSkill")} onClick={()=>downloadSkill(skill.name,skill.goal,skill.playbook)}><Download/></button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}{listedSkills.length===0&&<small className="plus-menu-empty">{t("noMatchingSkills")}</small>}</div></div></>}</div>}</div><input ref={importRef} className="skill-import-input" type="file" accept="application/json,.json" tabIndex={-1} aria-hidden="true" onChange={event=>{const file=event.target.files?.[0];event.currentTarget.value="";if(file)void importSkillFile(file)}}/><input ref={attachRef} className="skill-import-input attach-input" type="file" multiple accept={ATTACH_ACCEPT} tabIndex={-1} aria-hidden="true" onChange={event=>{const files=[...event.target.files||[]];event.currentTarget.value="";if(files.length)addPendingFiles(files)}}/>{pendingFiles.length>0&&<div className="composer-files">{pendingFiles.map(item=><FileCard key={item.id} file={{name:item.file.name,size:item.file.size}} preview={item.preview} onRemove={()=>removePendingFile(item.id)}/>)}</div>}{slashSuggestions.length>0&&<div className="slash-suggestions" role="listbox" aria-label="指令與技能">{slashSuggestions.map((skill,index)=><button type="button" role="option" aria-selected={index===slashIndex} key={skill.name} onClick={()=>{setDraft(`/${skill.name} `);requestAnimationFrame(()=>document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus())}}><strong>/{skill.name}</strong><small>{skill.kind}</small><span>{skill.description}</span></button>)}</div>}<textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} onPaste={event=>{const files=event.clipboardData?.files;if(files&&files.length){event.preventDefault();addPendingFiles(files)}}} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||(!draft.trim()&&pendingFiles.length===0)||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
|
@ -318,7 +335,7 @@ export function App(){
|
|||
<div className={`side-part computer-part ${rightPart==="computer"?"":"hidden-part"}`}>
|
||||
<div className="computer-status-row">{paneBot?<span>{t("botComputer",{name:paneBot.name})}</span>:<span>{t("computer")}</span>}{computer.state==="booting"?<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/>:<i className={`state-dot ${computer.state}`}/>}<small>{stateLabel(computer.state)}</small></div>
|
||||
<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}{!computerOpen&&hud}</div>
|
||||
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}<p className="computer-login-hint">{t("computerLoginHint")}</p><p className="clipboard-status" role="status">{clipboardStatus}</p>{scheduleDraft?<ScheduleEditor draft={scheduleDraft} timezone={scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei"} saving={scheduleSaving} error={scheduleError} onChange={next=>setScheduleDraft(current=>current?{...current,...next}:next)} onBack={()=>setScheduleDraft(null)} onSave={()=>void saveScheduleDraft()} onDelete={scheduleDraft.id?()=>void action(async()=>{await api(`/api/schedules/${scheduleDraft.id}`,{method:"DELETE"});setScheduleDraft(null);await reloadSchedules()}):undefined}/>:<ScheduleList items={schedules} runningId={runningScheduleId} onCreate={()=>setScheduleDraft({name:"",instructions:"",enabled:true,preset:defaultCronPreset()})} onOpen={item=>setScheduleDraft({id:item.id,timezone:item.timezone,threadId:item.threadId,name:item.name,instructions:item.instructions,enabled:item.enabled,preset:presetFromCron(item.cron)})} onRun={item=>void action(async()=>{setRunningScheduleId(item.id);try{await api(`/api/schedules/${item.id}/run`,{method:"POST",body:"{}"});await loadSessions()}finally{setRunningScheduleId(null)}})}/>}</>}
|
||||
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<p className="computer-login-hint">{t("computerLoginHint")}</p><p className="clipboard-status" role="status">{clipboardStatus}</p>{scheduleDraft?<ScheduleEditor draft={scheduleDraft} timezone={scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei"} saving={scheduleSaving} error={scheduleError} onChange={next=>setScheduleDraft(current=>current?{...current,...next}:next)} onBack={()=>setScheduleDraft(null)} onSave={()=>void saveScheduleDraft()} onDelete={scheduleDraft.id?()=>void action(async()=>{await api(`/api/schedules/${scheduleDraft.id}`,{method:"DELETE"});setScheduleDraft(null);await reloadSchedules()}):undefined}/>:<ScheduleList items={schedules} runningId={runningScheduleId} onCreate={()=>setScheduleDraft({name:"",instructions:"",enabled:true,preset:defaultCronPreset()})} onOpen={item=>setScheduleDraft({id:item.id,timezone:item.timezone,threadId:item.threadId,name:item.name,instructions:item.instructions,enabled:item.enabled,preset:presetFromCron(item.cron)})} onRun={item=>void action(async()=>{setRunningScheduleId(item.id);try{await api(`/api/schedules/${item.id}/run`,{method:"POST",body:"{}"});await loadSessions()}finally{setRunningScheduleId(null)}})}/>}</>}
|
||||
</div>
|
||||
{rightPart==="accounts"&&paneBot&&<VaultPane bot={paneBot}/>}
|
||||
{rightPart==="memory"&&paneBot&&<MemoryPane bot={paneBot} changed={loadBots}/>}
|
||||
|
|
@ -328,7 +345,7 @@ export function App(){
|
|||
</>
|
||||
</aside>}
|
||||
|
||||
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" title={t("moreActions")}><Ellipsis/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen"><div className="overlay-desktop">{frame}{hud}</div></div>{error&&<div className="overlay-error">{error}</div>}</div>}
|
||||
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen"><div className="overlay-desktop">{frame}{hud}</div></div>{error&&<div className="overlay-error">{error}</div>}</div>}
|
||||
{teachOpen&&active&&<TeachDialog bot={active} busy={busy} close={()=>setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>}
|
||||
{editingSkill&&<SkillEditDialog key={editingSkill.id} skill={editingSkill} busy={busy} close={()=>setEditingSkillId(null)} save={(name,playbook)=>void updateSkill(editingSkill,name,playbook)} test={(name,playbook)=>void testSkill(editingSkill,name,playbook)} remove={()=>void deleteSkill(editingSkill)} exportFile={(name,playbook)=>downloadSkill(name,editingSkill.goal,playbook)}/>}
|
||||
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{pasteText(text);setClipboardOpen(false)}}/>}
|
||||
|
|
@ -344,6 +361,7 @@ export function App(){
|
|||
{accountDialog==="phone"&&<PhoneAccessDialog close={()=>setAccountDialog(null)}/>}
|
||||
{accountDialog==="settings"&&<WorkspaceSettingsDialog name={workspaceName} setName={setWorkspaceName} showHidden={showHidden} setShowHidden={setShowHidden} rightCollapsed={rightCollapsed} setRightCollapsed={setRightCollapsed} close={()=>setAccountDialog(null)}/>}
|
||||
{accountDialog==="model"&&<ModelSettingsDialog close={()=>setAccountDialog(null)}/>}
|
||||
{accountDialog==="voice"&&<VoiceSettingsDialog close={()=>{setAccountDialog(null);api<VoiceSettings>("/api/voice/settings").then(setVoiceSettings).catch(()=>setVoiceSettings(null))}}/>}
|
||||
{accountDialog==="about"&&<AboutDialog close={()=>setAccountDialog(null)}/>}
|
||||
{accountDialog==="help"&&<HelpDialog close={()=>setAccountDialog(null)}/>}
|
||||
{accountDialog==="feedback"&&<FeedbackDialog close={()=>setAccountDialog(null)}/>}
|
||||
|
|
@ -567,8 +585,37 @@ function ComputerHud({bot,label}:{bot:{id:string;name:string;avatarColor?:string
|
|||
return <div className="computer-hud" role="status" aria-label={`${bot.name}：${label}`}><span className="computer-signal" aria-hidden="true"><span className="computer-signal-face"><i/><i/></span></span><span className="computer-hud-label">{label}</span></div>
|
||||
}
|
||||
function EmptyComputer({state}:{state:ComputerStatus["state"]}){if(state==="booting"||state==="suspended")return <div className="empty-computer is-waiting" aria-hidden="true"/>;return <div className="empty-computer"><Computer/><strong>{stateLabel(state)}</strong><span>{t("computerPreviewHint")}</span></div>}
|
||||
function ControlButtons({computer,busy,action,active,sessionId,onBoot,onRestart}:{computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;active:Bot;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const working=Boolean(computer.usingComputer);const restart=<button className="outline restart-computer" disabled={busy} title={t("restartDocker")} onClick={()=>void action(onRestart||(()=>api(`/api/computer/${active.id}/restart`,{method:"POST",body:"{}"})))}><RefreshCw/> {t("restartDocker")}</button>;if(computer.state!=="running")return <div className="computer-actions"><button className="primary" disabled={busy||computer.state==="booting"} onClick={()=>void action(onBoot||(()=>api(`/api/computer/${active.id}/boot`,{method:"POST",body:"{}"})))}>{(busy||computer.state==="booting")&&<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle",marginRight:7}}/>}{computer.state==="booting"?t("bootingProgress"):t("openComputer")}</button>{(computer.state==="booting"||computer.state==="error")&&restart}</div>;if(working)return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button><button className="outline" disabled={busy} onClick={()=>action(async()=>{if(sessionId)await api(`/api/sessions/${sessionId}/stop`,{method:"POST",body:"{}"});else await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});})}><Square/>{t("stopTask")}</button>{restart}</div>;if(computer.controlHolder==="user")return <div className="computer-actions"><button className="outline" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseControl")}</button>{restart}</div>;return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeControl")}</button>{restart}</div>}
|
||||
function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const interactive=props.computer.controlHolder==="user";return <div className="control-bar"><ControlButtons {...props}/><button className="icon-button" disabled={!interactive} onClick={props.paste}><ClipboardPaste/></button><button className="icon-button" disabled={!interactive} onClick={props.copy}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button></div>}
|
||||
function ControlButtons({computer,busy,action,active,sessionId,onBoot,onRestart,onStop}:{computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;active:Bot;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>;onStop?:()=>Promise<void>}){
|
||||
const [menuOpen,setMenuOpen]=useState(false);
|
||||
const menuRef=useRef<HTMLDivElement>(null);
|
||||
useEffect(()=>{
|
||||
if(!menuOpen)return;
|
||||
const outside=(event:PointerEvent)=>{if(!menuRef.current?.contains(event.target as Node))setMenuOpen(false)};
|
||||
const escape=(event:KeyboardEvent)=>{if(event.key==="Escape"){event.stopPropagation();setMenuOpen(false);menuRef.current?.querySelector<HTMLButtonElement>("button")?.focus()}};
|
||||
document.addEventListener("pointerdown",outside);document.addEventListener("keydown",escape);
|
||||
return()=>{document.removeEventListener("pointerdown",outside);document.removeEventListener("keydown",escape)};
|
||||
},[menuOpen]);
|
||||
useEffect(()=>setMenuOpen(false),[active.id,computer.state]);
|
||||
const working=Boolean(computer.usingComputer);
|
||||
const running=computer.state==="running";
|
||||
const booting=computer.state==="booting";
|
||||
const run=(operation:"boot"|"restart"|"stop",handler?:()=>Promise<void>)=>{setMenuOpen(false);void action(handler||(()=>api(`/api/computer/${active.id}/${operation}`,{method:"POST",body:"{}"})))};
|
||||
return <div className="computer-actions">
|
||||
{!running?<button className="primary" disabled={busy||booting} onClick={()=>run("boot",onBoot)}>{booting?t("bootingProgress"):t("openComputer")}</button>
|
||||
:computer.controlHolder==="user"?<button className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${active.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseControl")}</button>
|
||||
:<button className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{working?t("takeOverNow"):t("takeControl")}</button>}
|
||||
{running&&working&&<button className="outline" disabled={busy} onClick={()=>void action(()=>api(sessionId?`/api/sessions/${sessionId}/stop`:`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"}))}><Square/>{t("stopTask")}</button>}
|
||||
{computer.state!=="stopped"&&<div className="computer-power" ref={menuRef}>
|
||||
<button type="button" className="outline computer-power-trigger" disabled={busy||booting} aria-expanded={menuOpen} aria-label={t("computerPower")} title={t("computerPower")} onClick={()=>setMenuOpen(open=>!open)}><Ellipsis/></button>
|
||||
{menuOpen&&<div className="computer-power-menu">
|
||||
<button type="button" disabled={busy} onClick={()=>run("restart",onRestart)}><RefreshCw/>{t("restartComputer")}</button>
|
||||
<button type="button" className="computer-power-stop" disabled={busy} onClick={()=>run("stop",onStop)}><Square/>{t("shutDownComputer")}</button>
|
||||
{computer.mode==="team"&&<small>{t("sharedPowerHint")}</small>}
|
||||
</div>}
|
||||
</div>}
|
||||
</div>;
|
||||
}
|
||||
function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>;onStop?:()=>Promise<void>}){const interactive=props.computer.controlHolder==="user";return <div className="control-bar"><ControlButtons {...props}/><button className="icon-button" disabled={!interactive} title={t("pasteClipboard")} onClick={props.paste}><ClipboardPaste/></button><button className="icon-button" disabled={!interactive} title={t("copyDesktopClipboard")} onClick={props.copy}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button></div>}
|
||||
function TeachDialog({bot,busy,close,start}:{bot:Bot;busy:boolean;close:()=>void;start:(goal:string)=>void}){
|
||||
const [goal,setGoal]=useState("");
|
||||
return <div className="modal-backdrop" onClick={close}><form className="dialog teach-dialog" onClick={e=>e.stopPropagation()} onSubmit={e=>{e.preventDefault();if(goal.trim())start(goal.trim())}}>
|
||||
|
|
@ -741,6 +788,7 @@ function HelpDialog({close}:{close:()=>void}){
|
|||
<section><h3>{t("helpMcpTitle")}</h3><p>{t("helpMcp")}</p></section>
|
||||
<section><h3>{t("helpSkillsTitle")}</h3><p>{t("helpSkills")}</p></section>
|
||||
<section><h3>{t("helpAttachTitle")}</h3><p>{t("helpAttach")}</p></section>
|
||||
<section><h3>{t("helpVoiceTitle")}</h3><p>{t("helpVoice")}</p></section>
|
||||
<section><h3>{t("helpShortcutsTitle")}</h3><p>{t("helpShortcuts")}</p></section>
|
||||
</div>
|
||||
<div className="dialog-actions"><button className="primary" onClick={close}>{t("close")}</button></div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
export const VOICE_SAMPLE_RATE = 24000;
|
||||
|
||||
export function floatToPcm16(input: Float32Array): Uint8Array {
|
||||
const pcm = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i += 1) {
|
||||
const sample = Math.max(-1, Math.min(1, input[i] ?? 0));
|
||||
pcm[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
||||
}
|
||||
return new Uint8Array(pcm.buffer);
|
||||
}
|
||||
|
||||
export function pcm16ToFloat(bytes: ArrayBuffer): Float32Array {
|
||||
const pcm = new Int16Array(bytes.byteLength % 2 === 0 ? bytes : bytes.slice(0, bytes.byteLength - 1));
|
||||
const out = new Float32Array(pcm.length);
|
||||
for (let i = 0; i < pcm.length; i += 1) {
|
||||
out[i] = (pcm[i] ?? 0) / 32768;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function resample(input: Float32Array, fromRate: number, toRate: number): Float32Array {
|
||||
if (fromRate === toRate || input.length === 0) return input;
|
||||
const ratio = fromRate / toRate;
|
||||
const length = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Float32Array(length);
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
const src = i * ratio;
|
||||
const left = Math.floor(src);
|
||||
const frac = src - left;
|
||||
const a = input[left] ?? 0;
|
||||
const b = input[Math.min(left + 1, input.length - 1)] ?? 0;
|
||||
out[i] = a + (b - a) * frac;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const WORKLET = `
|
||||
class CaptureProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const channel = inputs[0] && inputs[0][0];
|
||||
if (channel && channel.length) this.port.postMessage(channel);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor("lazyboy-capture", CaptureProcessor);
|
||||
`;
|
||||
|
||||
export type CallAudioHandlers = {
|
||||
onCapture: (pcm: Uint8Array) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export class CallAudio {
|
||||
private stopped = false;
|
||||
private context: AudioContext | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private worklet: AudioWorkletNode | null = null;
|
||||
private nextTime = 0;
|
||||
private playing: AudioBufferSourceNode[] = [];
|
||||
private handlers: CallAudioHandlers;
|
||||
|
||||
constructor(handlers: CallAudioHandlers) {
|
||||
this.handlers = handlers;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
if (this.stopped) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
this.stream = stream;
|
||||
const context = new AudioContext();
|
||||
this.context = context;
|
||||
if (context.state === "suspended") await context.resume();
|
||||
const blob = new Blob([WORKLET], { type: "application/javascript" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
try {
|
||||
await context.audioWorklet.addModule(url);
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
if (this.stopped) return;
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const worklet = new AudioWorkletNode(context, "lazyboy-capture");
|
||||
worklet.port.onmessage = (event) => {
|
||||
const samples = event.data as Float32Array;
|
||||
const resampled = resample(samples, context.sampleRate, VOICE_SAMPLE_RATE);
|
||||
this.handlers.onCapture(floatToPcm16(resampled));
|
||||
};
|
||||
source.connect(worklet);
|
||||
worklet.connect(context.destination);
|
||||
this.worklet = worklet;
|
||||
this.nextTime = context.currentTime;
|
||||
}
|
||||
|
||||
play(pcm: ArrayBuffer): void {
|
||||
const context = this.context;
|
||||
if (!context) return;
|
||||
const samples = resample(pcm16ToFloat(pcm), VOICE_SAMPLE_RATE, context.sampleRate);
|
||||
if (!samples.length) return;
|
||||
const buffer = context.createBuffer(1, samples.length, context.sampleRate);
|
||||
buffer.getChannelData(0).set(samples);
|
||||
const node = context.createBufferSource();
|
||||
node.buffer = buffer;
|
||||
node.connect(context.destination);
|
||||
const startAt = Math.max(context.currentTime, this.nextTime);
|
||||
node.start(startAt);
|
||||
this.nextTime = startAt + buffer.duration;
|
||||
this.playing.push(node);
|
||||
node.onended = () => {
|
||||
this.playing = this.playing.filter((item) => item !== node);
|
||||
};
|
||||
}
|
||||
|
||||
interrupt(): void {
|
||||
for (const node of this.playing) {
|
||||
try { node.stop(); } catch { /* already stopped */ }
|
||||
}
|
||||
this.playing = [];
|
||||
if (this.context) this.nextTime = this.context.currentTime;
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
this.interrupt();
|
||||
this.worklet?.disconnect();
|
||||
this.worklet = null;
|
||||
this.stream?.getTracks().forEach((track) => track.stop());
|
||||
this.stream = null;
|
||||
void this.context?.close();
|
||||
this.context = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
.call-overlay{position:absolute;inset:0;z-index:40;display:grid;place-items:center;background:rgba(5,5,6,.82);padding:24px}
|
||||
.call-card{width:min(420px,100%);display:flex;flex-direction:column;align-items:center;gap:10px;padding:28px 24px 22px;border:1px solid var(--border);border-radius:28px;background:#121214;box-shadow:0 30px 80px rgba(0,0,0,.55);text-align:center}
|
||||
.call-kicker{font-size:12px;letter-spacing:.14em;text-transform:uppercase;color:var(--faint)}
|
||||
.call-name{margin-top:4px;font-size:22px}
|
||||
.call-phase{min-height:1.4em;color:var(--accent);font-size:14px}
|
||||
.call-phase.working{color:#ffe08a}
|
||||
.call-caption{min-height:3.2em;margin:4px 0 0;color:var(--muted);font-size:14.5px;line-height:1.5}
|
||||
.call-error{margin:0;color:var(--danger);font-size:13px}
|
||||
.call-time{color:var(--faint);font-variant-numeric:tabular-nums;font-size:13px}
|
||||
.call-actions{display:flex;gap:10px;margin-top:8px}
|
||||
.call-interrupt,.call-hangup{min-width:108px;height:40px;border:0;border-radius:999px;cursor:pointer}
|
||||
.call-interrupt{background:var(--surface);color:var(--ink)}
|
||||
.call-hangup{background:var(--danger);color:#fff;font-weight:600}
|
||||
.call-takeover{margin-top:8px}
|
||||
.call-hint{margin:8px 0 0;color:var(--faint);font-size:12px}
|
||||
.top-tool-button.call-active{background:rgba(62,197,168,.16);color:var(--accent)}
|
||||
|
||||
.call-entry{display:inline-flex;border-radius:9px}
|
||||
.call-entry:focus-visible{outline:2px solid var(--accent);outline-offset:2px}
|
||||
.call-entry button:disabled{pointer-events:none}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
import { useEffect, useRef, useState } from "react";
|
||||
import { Avatar } from "./avatar";
|
||||
import { CallAudio } from "./call-audio";
|
||||
import { t } from "./i18n";
|
||||
import type { AvatarShape, Bot } from "./types";
|
||||
|
||||
export type CallPhase = "connecting" | "listening" | "speaking" | "working";
|
||||
|
||||
type ComputerEvent = { status?: string; step?: string | null; takeover?: boolean };
|
||||
type ServerEvent =
|
||||
| { type: "ready"; callId?: string; voice?: string }
|
||||
| { type: "transcript"; role: "user" | "assistant"; text: string; final?: boolean }
|
||||
| { type: "speech"; state: "started" | "stopped" }
|
||||
| { type: "computer"; status?: string; step?: string | null; takeover?: boolean }
|
||||
| { type: "error"; message: string };
|
||||
|
||||
function wsUrl(sessionId: string): string {
|
||||
const protocol = location.protocol === "https:" ? "wss:" : "ws:";
|
||||
return `${protocol}//${location.host}/api/sessions/${sessionId}/call`;
|
||||
}
|
||||
|
||||
function phaseLabel(phase: CallPhase, computer: ComputerEvent | null): string {
|
||||
if (computer?.takeover) return t("takeOverToContinue");
|
||||
if (phase === "connecting") return t("callConnecting");
|
||||
if (phase === "speaking") return t("speaking");
|
||||
if (phase === "working" || computer?.status === "running" || computer?.status === "queued") {
|
||||
return t("workingOnComputer");
|
||||
}
|
||||
return t("listening");
|
||||
}
|
||||
|
||||
export function PhoneIcon({ size = 16 }: { size?: number }) {
|
||||
return (
|
||||
<svg width={size} height={size} viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" aria-hidden="true">
|
||||
<path d="M6.5 3.8c.5-1 1.6-1.4 2.6-1l2 1c.8.4 1.2 1.3 1 2.2l-.6 2.2a2 2 0 0 0 .6 1.9l3.8 3.8c.5.5 1.3.7 1.9.6l2.2-.6c.9-.2 1.8.2 2.2 1l1 2c.4 1 .1 2.1-.9 2.6-2.3 1.2-7.2 1-11.6-3.4S4.1 7.6 5.3 5.3Z" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
export function CallOverlay({
|
||||
bot,
|
||||
sessionId,
|
||||
takeover,
|
||||
onHangUp,
|
||||
onTakeOver,
|
||||
}: {
|
||||
bot: Bot;
|
||||
sessionId: string;
|
||||
takeover: boolean;
|
||||
onHangUp: () => void;
|
||||
onTakeOver: () => void;
|
||||
}) {
|
||||
const [phase, setPhase] = useState<CallPhase>("connecting");
|
||||
const [caption, setCaption] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const [computer, setComputer] = useState<ComputerEvent | null>(null);
|
||||
const audioRef = useRef<CallAudio | null>(null);
|
||||
const socketRef = useRef<WebSocket | null>(null);
|
||||
const hanging = useRef(false);
|
||||
|
||||
function hangUp() {
|
||||
hanging.current = true;
|
||||
audioRef.current?.stop();
|
||||
audioRef.current = null;
|
||||
socketRef.current?.close();
|
||||
socketRef.current = null;
|
||||
onHangUp();
|
||||
}
|
||||
|
||||
function interrupt() {
|
||||
audioRef.current?.interrupt();
|
||||
setPhase("listening");
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
hanging.current = false;
|
||||
let disposed = false;
|
||||
const started = Date.now();
|
||||
const tick = window.setInterval(() => setElapsed(Math.floor((Date.now() - started) / 1000)), 1000);
|
||||
const audio = new CallAudio({
|
||||
onCapture: (pcm) => {
|
||||
if (socketRef.current?.readyState === WebSocket.OPEN) socketRef.current.send(pcm);
|
||||
},
|
||||
onError: (message) => setError(message),
|
||||
});
|
||||
audioRef.current = audio;
|
||||
const socket = new WebSocket(wsUrl(sessionId));
|
||||
socket.binaryType = "arraybuffer";
|
||||
socketRef.current = socket;
|
||||
socket.onopen = () => {
|
||||
void audio.start().catch((err) => {
|
||||
if (disposed) return;
|
||||
audio.stop();
|
||||
socket.close();
|
||||
setError(err instanceof Error && /NotAllowedError|PermissionDenied/.test(err.name + err.message) ? t("micDenied") : t("micFailed"));
|
||||
});
|
||||
setPhase("listening");
|
||||
};
|
||||
socket.onmessage = (event) => {
|
||||
if (typeof event.data !== "string") {
|
||||
audio.play(event.data as ArrayBuffer);
|
||||
return;
|
||||
}
|
||||
let payload: ServerEvent;
|
||||
try { payload = JSON.parse(event.data) as ServerEvent; } catch { return; }
|
||||
if (payload.type === "speech") {
|
||||
if (payload.state === "started") {
|
||||
audio.interrupt();
|
||||
setPhase("listening");
|
||||
} else setPhase("listening");
|
||||
return;
|
||||
}
|
||||
if (payload.type === "transcript") {
|
||||
setCaption(payload.text);
|
||||
if (payload.role === "assistant") setPhase(payload.final ? "listening" : "speaking");
|
||||
return;
|
||||
}
|
||||
if (payload.type === "computer") {
|
||||
setComputer(payload);
|
||||
if (payload.status === "running" || payload.status === "queued" || payload.takeover) setPhase("working");
|
||||
return;
|
||||
}
|
||||
if (payload.type === "error") setError(payload.message);
|
||||
};
|
||||
socket.onerror = () => { if (!disposed && !hanging.current) setError(t("callFailed")); };
|
||||
socket.onclose = () => {
|
||||
audio.stop();
|
||||
if (!disposed && !hanging.current) setError((previous) => previous || t("callFailed"));
|
||||
};
|
||||
function onKey(event: KeyboardEvent) {
|
||||
if (event.key === "Escape") { event.preventDefault(); hangUp(); }
|
||||
if (event.key === " " && !event.repeat && event.target === document.body) {
|
||||
event.preventDefault();
|
||||
interrupt();
|
||||
}
|
||||
}
|
||||
window.addEventListener("keydown", onKey);
|
||||
return () => {
|
||||
disposed = true;
|
||||
hanging.current = true;
|
||||
window.clearInterval(tick);
|
||||
window.removeEventListener("keydown", onKey);
|
||||
audio.stop();
|
||||
socket.close();
|
||||
};
|
||||
}, [sessionId, bot.id]);
|
||||
|
||||
const minutes = Math.floor(elapsed / 60);
|
||||
const seconds = String(elapsed % 60).padStart(2, "0");
|
||||
const showTakeover = takeover || Boolean(computer?.takeover);
|
||||
|
||||
return (
|
||||
<div className="call-overlay" role="dialog" aria-label={t("call")}>
|
||||
<div className="call-card" data-testid="call-view">
|
||||
<div className="call-kicker">{t("call")}</div>
|
||||
<Avatar lookId={bot.id} name={bot.name} color={bot.avatarColor} shape={bot.avatarShape as AvatarShape} active online size={72} />
|
||||
<strong className="call-name">{bot.name}</strong>
|
||||
<div className={`call-phase ${phase}`}>{phaseLabel(phase, computer)}</div>
|
||||
<p className="call-caption">{caption || t("callPrompt")}</p>
|
||||
{error ? <p className="call-error">{error}</p> : null}
|
||||
<div className="call-time">{minutes}:{seconds}</div>
|
||||
<div className="call-actions">
|
||||
<button type="button" className="call-interrupt" onClick={interrupt}>{t("interrupt")}</button>
|
||||
<button type="button" className="call-hangup" onClick={hangUp}>{t("hangUp")}</button>
|
||||
</div>
|
||||
{showTakeover ? (
|
||||
<button type="button" className="primary call-takeover" onClick={onTakeOver}>{t("takeOverNow")}</button>
|
||||
) : null}
|
||||
<p className="call-hint">{t("callShortcuts")}</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -30,6 +30,11 @@
|
|||
.composer{position:relative;left:auto;right:auto;bottom:auto;width:var(--chat-col);min-height:62px;align-items:center;padding:9px 10px;transform:none}
|
||||
.composer.has-files{flex-wrap:wrap;align-items:flex-end;padding-top:10px}
|
||||
.composer textarea{align-self:center;box-sizing:border-box;height:42px;min-height:42px;max-height:126px;padding:11px 4px;line-height:20px}
|
||||
.slash-suggestions{position:absolute;left:58px;right:58px;bottom:calc(100% + 8px);z-index:8;display:grid;gap:2px;padding:6px;border:1px solid var(--border);border-radius:14px;background:#18181b;box-shadow:0 14px 34px rgba(0,0,0,.35)}
|
||||
.slash-suggestions button{display:flex;align-items:baseline;gap:10px;width:100%;padding:8px 10px;border:0;border-radius:9px;background:transparent;color:var(--ink);text-align:left;cursor:pointer}
|
||||
.slash-suggestions button:hover,.slash-suggestions button:focus-visible{background:rgba(255,255,255,.08);outline:0}
|
||||
.slash-suggestions strong{min-width:110px;color:var(--accent);font-weight:650}
|
||||
.slash-suggestions span{overflow:hidden;color:var(--muted);font-size:12px;text-overflow:ellipsis;white-space:nowrap}
|
||||
.composer-files{display:flex;flex-wrap:wrap;gap:8px;flex:1 0 100%;order:-1;padding:2px 8px 10px 46px}
|
||||
.file-card{display:flex;align-items:center;gap:8px;max-width:min(240px,100%);height:48px;padding:2px 8px 2px 2px;border:1px solid #2a2a2a;border-radius:14px;background:#1a1a1a;color:var(--ink)}
|
||||
.file-card-thumb,.file-card-badge{flex:0 0 44px;width:44px;height:44px;border-radius:11px;object-fit:cover}
|
||||
|
|
@ -65,3 +70,6 @@
|
|||
.message.assistant.spoken>.message-body,.message.assistant.spoken>.message-stack{grid-column:2;max-width:82%;min-width:0}
|
||||
.message.assistant.spoken .message-stack>.message-body{max-width:100%;line-height:1.52;border-radius:6px 18px 18px 18px}
|
||||
.composer-plus.open{color:var(--ink);background:rgba(255,255,255,.08)}
|
||||
|
||||
.slash-suggestions button[aria-selected="true"]{background:rgba(255,255,255,.08)}
|
||||
.slash-suggestions small{flex-shrink:0;color:var(--muted);font-size:11px}
|
||||
|
|
|
|||
|
|
@ -55,3 +55,16 @@
|
|||
.computer-hud-label{color:#c9ddd5;background:none;animation:none;white-space:normal;text-align:center;max-width:90%;line-height:1.5}
|
||||
|
||||
.clipboard-status{margin:0;min-height:1.4em;font-size:12px;line-height:1.4;color:var(--muted)}
|
||||
|
||||
/* Keep control actions prominent and group less frequent power actions. */
|
||||
.control-bar>.computer-actions{justify-content:flex-start}
|
||||
.computer-power{position:relative;flex-shrink:0;margin-left:auto}
|
||||
.computer-power-trigger{padding:8px;display:inline-flex;align-items:center;justify-content:center}
|
||||
.computer-power-trigger svg{width:18px;height:18px}
|
||||
.computer-power-menu{position:absolute;right:0;bottom:calc(100% + 8px);z-index:40;width:180px;padding:6px;border:1px solid var(--line,#343438);border-radius:12px;background:var(--surface,#242426);box-shadow:0 8px 28px #0005}
|
||||
.computer-power-menu button{display:flex;align-items:center;gap:9px;width:100%;padding:10px;border:0;border-radius:7px;background:transparent;color:var(--ink);text-align:left;cursor:pointer}
|
||||
.computer-power-menu button:hover{background:var(--surface-hover,#ffffff0c)}
|
||||
.computer-power-menu svg{width:16px;height:16px}
|
||||
.computer-power-menu .computer-power-stop{color:#e98282}
|
||||
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line,#343438);color:var(--muted);font-size:11px;line-height:1.6}
|
||||
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export const zhTW = {
|
|||
attachTooMany: "一次最多 4 個附件", attachTooLarge: "單檔不能超過 10 MB", attachType: "這個檔案類型不能附加",
|
||||
attachRemove: "移除 {name}", attachedFile: "附件 {name}", attachDrop: "放到這裡附加",
|
||||
teachTask: "教它一項任務", teachTaskHint: "你示範一次，它學成技能", teachNeedsBot: "先選擇一個機器人", taughtSkills: "已學會的技能",
|
||||
goalCommandHint: "持續規劃並執行，直到驗證完成或需要你介入",
|
||||
searchSkills: "搜尋技能", noMatchingSkills: "沒有符合的技能",
|
||||
teachDialogLead: "接下來畫面交給你操作，{name} 會在旁邊看：記錄你點了哪些控制項、輸入了什麼、去了哪些頁面，之後整理成一個「知道目的與流程」的技能，而不是死記座標。",
|
||||
teachGoalLabel: "你要示範什麼？（一句話說目標）", teachGoalPlaceholder: "例如：到 STAR 訓練系統，把指定課程的影片看完並完成測驗",
|
||||
|
|
@ -63,7 +64,7 @@ export const zhTW = {
|
|||
mcpNoResults: "找不到符合的 MCP。換個關鍵字，或改用自訂接入。", mcpCustom: "自訂接入", mcpBackToList: "回到列表",
|
||||
mcpConnectNamed: "接入 {name}", mcpKeyHint: "這個 MCP 需要憑證才能連。",
|
||||
noMcp: "還沒有 MCP。點上面的「選擇 MCP」從市集接入。", toolsCount: "{count} 個工具", disabled: "已關閉", disconnected: "未連線", noTools: "沒有可用工具", reconnect: "重新連線", disable: "停用", enable: "啟用",
|
||||
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後，畫面會顯示在這裡。", bootingProgress: "啟動中…", restartDocker: "重啟 Docker", stopAndTakeOver: "停止並接管",
|
||||
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後，畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦，操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管",
|
||||
hudBooting: "電腦啟動中…", hudWaking: "喚醒中…", hudConnecting: "連線中…", hudHandoff: "換手中…",
|
||||
pasteToRemoteComputer: "貼到遠端電腦", pasteRemoteHelp: "把外面的文字貼在這裡，再送進 VNC。這個方式在區網 HTTP 也能使用。", pasteTextPlaceholder: "在此貼上文字…", pasteIntoVnc: "貼入 VNC",
|
||||
botNamePlaceholder: "例如：研究助理", sharedComputerHint: "與其他機器人共用環境", privateComputerHint: "全新的獨立 Docker", create: "建立",
|
||||
|
|
@ -92,7 +93,18 @@ export const zhTW = {
|
|||
helpMcpTitle: "MCP 外掛", helpMcp: "左下「外掛程式」接入 MCP server。連上的工具會顯示在畫面上，對話時 Agent 可以使用。",
|
||||
helpSkillsTitle: "技能", helpSkills: "＋ → 教它一項任務，示範一次就會整理成技能。示範結束後可以匯出 JSON，或把別人的技能檔匯入，換一個機器人也適用。",
|
||||
helpAttachTitle: "附件", helpAttach: "＋ → 附加檔案。圖片這則訊息就會給模型看，不會存進對話紀錄。若機器人電腦要打開原檔，會暫放 inbox/，兩小時後自動刪，避免把磁碟塞滿。",
|
||||
helpShortcutsTitle: "快捷鍵", helpShortcuts: "Enter 送出，Shift+Enter 換行。正在回覆時送出鈕會變成停止。",
|
||||
helpShortcutsTitle: "快捷鍵", helpShortcuts: "Enter 送出，Shift+Enter 換行。正在回覆時送出鈕會變成停止。通話中空白鍵插話，Esc 掛斷。",
|
||||
helpVoiceTitle: "語音通話", helpVoice: "1:1 對話標題列的電話可以打給機器人。用說話下指令，它會去操作自己的電腦；能不能做會立刻用聲音回你，中途可以插嘴改指令。",
|
||||
call: "通話", hangUp: "掛斷", interrupt: "插話", listening: "正在聽你說…", speaking: "正在說話…",
|
||||
callConnecting: "連線中…", workingOnComputer: "正在電腦上做…", callPrompt: "直接說要做什麼。沉默一下就會送出。",
|
||||
callShortcuts: "空白鍵插話 · Esc 掛斷", callFailed: "通話中斷了。", micDenied: "需要麥克風權限才能通話。", micFailed: "麥克風打不開。",
|
||||
takeOverToContinue: "需要你接手畫面。", setUpVoiceToCall: "先設定語音才能通話",
|
||||
voiceEnabled: "啟用語音通話", voiceEnabledHint: "預設關閉。開啟並儲存後，才能使用通話按鈕。", voiceDisabledHint: "語音通話尚未啟用，請到「設定 → 語音」開啟。",
|
||||
voiceSettings: "語音", voiceSettingsHint: "語音跟文字模型分開。xAI 若已有聊天金鑰，通常不用再填一次。按分鐘計費。",
|
||||
voiceProvider: "語音供應商", voiceModel: "語音模型", voiceName: "聲音",
|
||||
providerOpenai: "OpenAI", providerXaiVoiceHint: "Grok Voice。即時通話，可插嘴，並把電腦工作交給文字模型去做。",
|
||||
providerOpenaiVoiceHint: "OpenAI Realtime。同一套通話介面，之後可切換。",
|
||||
voiceReusesTextKey: "目前會重用文字模型已存的 xAI 金鑰。",
|
||||
feedbackDescription: "寫下問題、想法或想要的功能。這是本機工作區，內容會複製到剪貼簿，方便你貼到 issue 或訊息裡。", feedbackPlaceholder: "例如：群組對話希望可以指定誰先發言…", copyContent: "複製內容",
|
||||
loginTitle: "登入 LazyBoy", loginDescription: "輸入伺服器設定的共享存取 token。", accessToken: "存取 token", verifying: "驗證中…", login: "登入",
|
||||
agentComputer: "Agent 電腦", url: "URL", stdio: "stdio", http: "HTTP", sse: "SSE",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import "./avatar.css";
|
|||
import "./chat.css";
|
||||
import "./computer.css";
|
||||
import "./schedule.css";
|
||||
import "./call.css";
|
||||
import "./responsive.css";
|
||||
import { App } from "./App";
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(<App />);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ export interface PlaybookInput { name:string; description?:string; example?:stri
|
|||
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
|
||||
export type TaughtSkillStatus = "recording"|"drafting"|"draft"|"saved"|"failed"|"cancelled";
|
||||
export interface TaughtSkill { id:string; botId:string; threadId:string|null; name:string; goal:string; status:TaughtSkillStatus; playbook:Playbook; error:string|null; startedAt:string|null; expiresAt:string|null; stoppedAt:string|null; createdAt:string; updatedAt:string; eventCount:number; frameCount:number }
|
||||
export interface FileSkill { name:string; description:string }
|
||||
export interface MemoryItem { id:string; sessionId:string|null; sourceRunId:string|null; sourceMessageId:string|null; content:string; importance:number; revision:number; createdAt:string; updatedAt:string }
|
||||
export type McpTransport = "stdio" | "http" | "sse";
|
||||
export interface McpTool { name:string; exposedName:string; description:string }
|
||||
|
|
@ -22,6 +23,22 @@ export interface McpServer { id:string; name:string; transport:McpTransport; com
|
|||
export interface McpSecretField { name:string; required:boolean; secret:boolean; hint:string }
|
||||
export interface McpCatalogEntry { id:string; title:string; description:string; transport:McpTransport; command:string|null; args:string[]; url:string|null; envKeys:McpSecretField[]; headerKeys:McpSecretField[]; source:"featured"|"registry"; remote:boolean }
|
||||
export type ModelProviderId = "xai" | "opencode-go" | "openai-compatible";
|
||||
export type VoiceProviderId = "xai" | "openai" | "scripted";
|
||||
export interface VoiceSettings {
|
||||
enabled: boolean;
|
||||
provider: VoiceProviderId;
|
||||
modelId: string;
|
||||
voiceId: string;
|
||||
ready: boolean;
|
||||
missing?: string | null;
|
||||
apiKeySet: boolean;
|
||||
envKeySet: boolean;
|
||||
envKeyName: string;
|
||||
reusesTextKey: boolean;
|
||||
providers: { id: VoiceProviderId; name: string; envKeyName: string }[];
|
||||
models: { id: string; name: string }[];
|
||||
voices: { id: string; name: string }[];
|
||||
}
|
||||
export interface WorkspaceProvider { id:ModelProviderId; name:string; needsBaseUrl:boolean; needsKey:boolean; defaultBaseUrl:string|null; defaultModel:string|null }
|
||||
export interface WorkspaceModel { id:string; name:string }
|
||||
export interface WorkspaceSettings { provider:ModelProviderId; modelId:string; baseUrl:string; apiKeySet:boolean; envKeySet:boolean; envKeyName:string; providers:WorkspaceProvider[]; models:WorkspaceModel[] }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,144 @@
|
|||
import { useEffect, useState } from "react";
|
||||
import { api } from "./api";
|
||||
import { t } from "./i18n";
|
||||
import { X } from "./animated-icons";
|
||||
import type { VoiceProviderId, VoiceSettings } from "./types";
|
||||
|
||||
export type { VoiceProviderId, VoiceSettings };
|
||||
|
||||
export function VoiceSettingsDialog({ close }: { close: () => void }) {
|
||||
const [settings, setSettings] = useState<VoiceSettings | null>(null);
|
||||
const [enabled, setEnabled] = useState(false);
|
||||
const [provider, setProvider] = useState<VoiceProviderId>("xai");
|
||||
const [modelId, setModelId] = useState("");
|
||||
const [voiceId, setVoiceId] = useState("");
|
||||
const [apiKey, setApiKey] = useState("");
|
||||
const [clearKey, setClearKey] = useState(false);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
api<VoiceSettings>("/api/voice/settings")
|
||||
.then((value) => {
|
||||
setSettings(value);
|
||||
setEnabled(value.enabled);
|
||||
setProvider(value.provider);
|
||||
setModelId(value.modelId);
|
||||
setVoiceId(value.voiceId);
|
||||
})
|
||||
.catch((err) => setError(err instanceof Error ? err.message : t("loadFailed")));
|
||||
}, []);
|
||||
|
||||
const current = settings?.providers.find((item) => item.id === provider);
|
||||
const models = (settings?.provider === provider ? settings.models : null)
|
||||
?? (provider === "openai"
|
||||
? [{ id: "gpt-realtime", name: "GPT Realtime" }]
|
||||
: [{ id: "grok-voice-latest", name: "Grok Voice Latest" }, { id: "grok-voice-think-fast-2.0", name: "Grok Voice Think Fast 2.0" }]);
|
||||
const voices = (settings?.provider === provider ? settings.voices : null)
|
||||
?? (provider === "openai"
|
||||
? [{ id: "marin", name: "Marin" }, { id: "alloy", name: "Alloy" }, { id: "verse", name: "Verse" }]
|
||||
: [{ id: "eve", name: "Eve" }, { id: "ara", name: "Ara" }, { id: "leo", name: "Leo" }, { id: "rex", name: "Rex" }, { id: "sal", name: "Sal" }]);
|
||||
|
||||
function pickProvider(id: VoiceProviderId) {
|
||||
setProvider(id);
|
||||
setError("");
|
||||
if (id === "openai") {
|
||||
setModelId("gpt-realtime");
|
||||
setVoiceId("marin");
|
||||
return;
|
||||
}
|
||||
setModelId("grok-voice-latest");
|
||||
setVoiceId("eve");
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="modal-backdrop">
|
||||
<form
|
||||
className="dialog settings-dialog"
|
||||
onSubmit={async (event) => {
|
||||
event.preventDefault();
|
||||
setBusy(true);
|
||||
setError("");
|
||||
try {
|
||||
await api("/api/voice/settings", {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify({
|
||||
enabled,
|
||||
provider,
|
||||
modelId: modelId.trim(),
|
||||
voiceId: voiceId.trim(),
|
||||
apiKey: clearKey ? "" : apiKey.trim() || null,
|
||||
clearApiKey: clearKey,
|
||||
}),
|
||||
});
|
||||
close();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("settingsFailed"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div className="dialog-title">
|
||||
<h2>{t("voiceSettings")}</h2>
|
||||
<button type="button" onClick={close}><X /></button>
|
||||
</div>
|
||||
<p className="dialog-lead">{t("voiceSettingsHint")}</p>
|
||||
<label className="memory-toggle">
|
||||
<input type="checkbox" role="switch" checked={enabled} disabled={!settings || busy} onChange={(event) => setEnabled(event.target.checked)} />
|
||||
{t("voiceEnabled")}
|
||||
</label>
|
||||
<p className="dialog-lead">{t("voiceEnabledHint")}</p>
|
||||
{enabled ? <>
|
||||
<fieldset className="provider-fieldset">
|
||||
<legend>{t("voiceProvider")}</legend>
|
||||
<div className="provider-grid">
|
||||
{(settings?.providers || [
|
||||
{ id: "xai" as const, name: t("providerXai"), envKeyName: "XAI_API_KEY" },
|
||||
{ id: "openai" as const, name: t("providerOpenai"), envKeyName: "OPENAI_API_KEY" },
|
||||
]).map((item) => (
|
||||
<button type="button" key={item.id} className={provider === item.id ? "picked" : ""} onClick={() => pickProvider(item.id)}>
|
||||
{item.id === "xai" ? t("providerXai") : item.id === "openai" ? t("providerOpenai") : item.name}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<p className="dialog-lead">{provider === "openai" ? t("providerOpenaiVoiceHint") : t("providerXaiVoiceHint")}</p>
|
||||
</fieldset>
|
||||
<label>
|
||||
{t("voiceModel")}
|
||||
{models.length ? (
|
||||
<select value={models.some((item) => item.id === modelId) ? modelId : modelId} onChange={(event) => setModelId(event.target.value)}>
|
||||
{models.map((item) => <option key={item.id} value={item.id}>{item.name}</option>)}
|
||||
{!models.some((item) => item.id === modelId) && modelId ? <option value={modelId}>{modelId}</option> : null}
|
||||
</select>
|
||||
) : (
|
||||
<input value={modelId} onChange={(event) => setModelId(event.target.value)} />
|
||||
)}
|
||||
</label>
|
||||
<label>
|
||||
{t("voiceName")}
|
||||
{voices.length ? (
|
||||
<select value={voiceId} onChange={(event) => setVoiceId(event.target.value)}>
|
||||
{voices.map((item) => <option key={item.id} value={item.id}>{item.name}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<input value={voiceId} onChange={(event) => setVoiceId(event.target.value)} />
|
||||
)}
|
||||
</label>
|
||||
<label>
|
||||
{t("apiKey")}
|
||||
<input type="password" autoComplete="off" value={apiKey} onChange={(event) => { setApiKey(event.target.value); setClearKey(false); }} placeholder={settings?.apiKeySet ? t("apiKeyStored") : t("apiKeyPlaceholder")} />
|
||||
</label>
|
||||
{settings?.reusesTextKey && !apiKey && !clearKey ? <p className="dialog-lead">{t("voiceReusesTextKey")}</p> : null}
|
||||
{settings?.envKeySet && !apiKey && !clearKey ? <p className="dialog-lead">{t("usingEnvKey", { name: current?.envKeyName || settings.envKeyName })}</p> : null}
|
||||
{settings?.apiKeySet ? <label className="memory-toggle"><input type="checkbox" checked={clearKey} onChange={(event) => setClearKey(event.target.checked)} /> {t("clearApiKey")}</label> : null}
|
||||
</> : null}
|
||||
{error ? <div className="pane-error">{error}</div> : null}
|
||||
<div className="dialog-actions">
|
||||
<button type="button" className="outline" onClick={close}>{t("cancel")}</button>
|
||||
<button className="primary" disabled={busy || !settings || !modelId.trim() || !voiceId.trim()}>{busy ? t("saving") : t("save")}</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
export default defineConfig({plugins:[react()],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":"http://127.0.0.1:3101","/view":{target:"http://127.0.0.1:3101",ws:true}}}});
|
||||
export default defineConfig({plugins:[react()],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":{target:"http://127.0.0.1:3101",ws:true},"/view":{target:"http://127.0.0.1:3101",ws:true}}}});
|
||||
|
|
|
|||
|
|
@ -722,9 +722,12 @@ pub async fn stop(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
|
|||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.ok_or_else(|| "computer not found".to_string())?;
|
||||
if computer_has_active_work(state, &computer_id).await {
|
||||
return Err("電腦仍有工作或示範進行中，請先停止工作再關閉電腦。".into());
|
||||
}
|
||||
if let Some(provider_ref) = &computer.provider_ref {
|
||||
let ctx = adapter_context(actor, bot_id, "stop");
|
||||
let _ = state
|
||||
state
|
||||
.sandbox
|
||||
.stop(
|
||||
&lazyboy_control::ComputerRef {
|
||||
|
|
@ -736,7 +739,8 @@ pub async fn stop(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
|
|||
},
|
||||
&ctx,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.map_err(|error| format!("無法關閉電腦：{error}"))?;
|
||||
}
|
||||
sqlx::query(
|
||||
"UPDATE computers SET state = 'stopped', control_holder = 'none', control_lease_id = NULL,
|
||||
|
|
|
|||
|
|
@ -95,6 +95,11 @@ pub struct SpaceRow {
|
|||
pub default_model_id: String,
|
||||
pub default_model_base_url: Option<String>,
|
||||
pub default_model_api_key: Option<String>,
|
||||
pub voice_enabled: bool,
|
||||
pub voice_provider: Option<String>,
|
||||
pub voice_model_id: Option<String>,
|
||||
pub voice_id: Option<String>,
|
||||
pub voice_api_key: Option<String>,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
|
|
@ -125,7 +130,8 @@ impl Db {
|
|||
pub async fn get_space(&self, actor: &Actor) -> Result<Option<SpaceRow>, sqlx::Error> {
|
||||
sqlx::query_as(
|
||||
"SELECT id, user_id, name, default_model_provider, default_model_id,
|
||||
default_model_base_url, default_model_api_key
|
||||
default_model_base_url, default_model_api_key,
|
||||
voice_enabled, voice_provider, voice_model_id, voice_id, voice_api_key
|
||||
FROM spaces WHERE id = $1 AND user_id = $2",
|
||||
)
|
||||
.bind(&actor.space_id)
|
||||
|
|
@ -178,6 +184,51 @@ impl Db {
|
|||
self.get_space(actor).await?.ok_or(sqlx::Error::RowNotFound)
|
||||
}
|
||||
|
||||
pub async fn update_voice_settings(
|
||||
&self,
|
||||
actor: &Actor,
|
||||
enabled: Option<bool>,
|
||||
provider: &str,
|
||||
model_id: &str,
|
||||
voice_id: &str,
|
||||
api_key: Option<Option<&str>>,
|
||||
) -> Result<SpaceRow, sqlx::Error> {
|
||||
match api_key {
|
||||
Some(key) => {
|
||||
sqlx::query(
|
||||
"UPDATE spaces
|
||||
SET voice_provider = $3, voice_model_id = $4, voice_id = $5, voice_api_key = $6, voice_enabled = COALESCE($7, voice_enabled)
|
||||
WHERE id = $1 AND user_id = $2",
|
||||
)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.bind(provider)
|
||||
.bind(model_id)
|
||||
.bind(voice_id)
|
||||
.bind(key)
|
||||
.bind(enabled)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
sqlx::query(
|
||||
"UPDATE spaces
|
||||
SET voice_provider = $3, voice_model_id = $4, voice_id = $5, voice_enabled = COALESCE($6, voice_enabled)
|
||||
WHERE id = $1 AND user_id = $2",
|
||||
)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.bind(provider)
|
||||
.bind(model_id)
|
||||
.bind(voice_id)
|
||||
.bind(enabled)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
self.get_space(actor).await?.ok_or(sqlx::Error::RowNotFound)
|
||||
}
|
||||
|
||||
pub async fn list_bots(
|
||||
&self,
|
||||
actor: &Actor,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
//! Read-only, workspace-wide SKILL.md commands.
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FileSkill {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
#[serde(skip_serializing)]
|
||||
pub instructions: String,
|
||||
}
|
||||
|
||||
fn valid_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name != "goal"
|
||||
&& name != "skills"
|
||||
&& name != "help"
|
||||
&& name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
}
|
||||
|
||||
fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
|
||||
let file_type = std::fs::symlink_metadata(&path).ok()?.file_type();
|
||||
if file_type.is_symlink() || !file_type.is_file() {
|
||||
return None;
|
||||
}
|
||||
if std::fs::metadata(&path).ok()?.len() > 512 * 1024 {
|
||||
return None;
|
||||
}
|
||||
let text = std::fs::read_to_string(&path).ok()?;
|
||||
let (frontmatter, instructions) = if let Some(rest) = text.strip_prefix("---\n") {
|
||||
let (header, body) = rest.split_once("\n---")?;
|
||||
(
|
||||
header,
|
||||
body.trim_start_matches(|ch| ch == '\n' || ch == '\r')
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
("", text.trim().to_string())
|
||||
};
|
||||
let description = frontmatter.lines().find_map(|line| {
|
||||
let (key, value) = line.split_once(':')?;
|
||||
(key.trim() == "description")
|
||||
.then(|| value.trim().trim_matches(|ch| ch == '"' || ch == '\'').to_string())
|
||||
}).unwrap_or_else(|| instructions.lines().next().unwrap_or("自訂技能").chars().take(120).collect());
|
||||
(!instructions.is_empty()).then_some(FileSkill { name, description, instructions })
|
||||
}
|
||||
|
||||
pub fn list(data_dir: &str) -> Vec<FileSkill> {
|
||||
let root = Path::new(data_dir).join("skills");
|
||||
let Ok(entries) = std::fs::read_dir(&root) else { return Vec::new() };
|
||||
let mut skills = entries.filter_map(Result::ok).filter_map(|entry| {
|
||||
let file_type = entry.file_type().ok()?;
|
||||
if !file_type.is_dir() { return None; }
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
valid_name(&name).then(|| parse_file(entry.path().join("SKILL.md"), name)).flatten()
|
||||
}).collect::<Vec<_>>();
|
||||
skills.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
skills
|
||||
}
|
||||
|
||||
pub fn slash(prompt: &str, data_dir: &str) -> Option<(FileSkill, String)> {
|
||||
let mut words = prompt.trim().splitn(2, char::is_whitespace);
|
||||
let command = words.next()?.strip_prefix('/')?;
|
||||
if !valid_name(command) { return None; }
|
||||
let skill = list(data_dir).into_iter().find(|skill| skill.name == command)?;
|
||||
Some((skill, words.next().unwrap_or("").trim().to_string()))
|
||||
}
|
||||
|
||||
pub fn index(data_dir: &str) -> String {
|
||||
let skills = list(data_dir);
|
||||
if skills.is_empty() { return String::new(); }
|
||||
let lines = skills.iter().map(|skill| format!("/{:<18} {}", skill.name, skill.description)).collect::<Vec<_>>().join("\n");
|
||||
format!("可用的檔案技能（唯讀）：\n{lines}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn loads_only_safe_skill_names_and_resolves_arguments() {
|
||||
let root = std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
|
||||
fs::create_dir_all(root.join("skills/open-site")).unwrap();
|
||||
fs::write(root.join("skills/open-site/SKILL.md"), "---\ndescription: Open a site\n---\nUse the browser.\n").unwrap();
|
||||
fs::create_dir_all(root.join("skills/goal")).unwrap();
|
||||
let root_text = root.to_str().unwrap();
|
||||
assert_eq!(slash("/open-site example.com", root_text).unwrap().1, "example.com");
|
||||
assert!(slash("/goal do it", root_text).is_none());
|
||||
assert!(slash("/missing", root_text).is_none());
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
}
|
||||
|
|
@ -2,9 +2,11 @@ mod attachments;
|
|||
mod auth;
|
||||
mod computer;
|
||||
mod db;
|
||||
mod file_skills;
|
||||
mod mcp;
|
||||
mod mcp_catalog;
|
||||
mod memory;
|
||||
mod retention;
|
||||
mod rooms;
|
||||
mod routes;
|
||||
mod runs;
|
||||
|
|
@ -15,15 +17,17 @@ mod skills;
|
|||
mod state;
|
||||
mod tools;
|
||||
mod vault;
|
||||
mod voice;
|
||||
mod voice_call;
|
||||
mod web_static;
|
||||
mod workspace;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use state::AppState;
|
||||
use tower_http::services::ServeDir;
|
||||
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[tokio::main]
|
||||
|
|
@ -51,6 +55,9 @@ async fn main() {
|
|||
mcp_state.mcp.reconnect_all(mcp_state.pool(), &actor).await;
|
||||
});
|
||||
|
||||
let retention_state = state.clone();
|
||||
tokio::spawn(async move { retention::retention_loop(retention_state).await; });
|
||||
|
||||
let worker_state = state.clone();
|
||||
tokio::spawn(async move {
|
||||
runs::worker_loop(worker_state).await;
|
||||
|
|
@ -73,14 +80,13 @@ async fn main() {
|
|||
}
|
||||
|
||||
let web_dir = std::env::var("LAZYBOY_WEB_DIR").unwrap_or_else(|_| "apps/web/dist".into());
|
||||
let app = Router::new()
|
||||
let app = web_static::static_router(&web_dir)
|
||||
.route(
|
||||
"/api/health",
|
||||
axum::routing::get(|| async { axum::Json(serde_json::json!({"ok": true})) }),
|
||||
)
|
||||
.merge(auth::public_router(state.clone()))
|
||||
.merge(routes::router(state.clone()))
|
||||
.fallback_service(ServeDir::new(web_dir))
|
||||
.layer(DefaultBodyLimit::max(28 * 1024 * 1024))
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
//! Bounded maintenance of expendable diagnostics, never user-authored content.
|
||||
use std::time::{Duration};
|
||||
use sqlx::PgPool;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn days(name: &str, default: i32) -> i32 {
|
||||
std::env::var(name).ok().and_then(|v| v.parse::<i32>().ok())
|
||||
.filter(|v| (1..=3650).contains(v)).unwrap_or(default)
|
||||
}
|
||||
|
||||
pub async fn retention_loop(state: AppState) {
|
||||
let rules = [
|
||||
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
|
||||
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
|
||||
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
|
||||
("recordings", include_str!("retention/recordings.sql"), days("LAZYBOY_RECORDING_RETENTION_DAYS", 30)),
|
||||
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||
("leases", include_str!("retention/leases.sql"), 7),
|
||||
("profile_locks", include_str!("retention/profile_locks.sql"), 7),
|
||||
];
|
||||
loop {
|
||||
for (name, query, age) in rules {
|
||||
let mut removed = 0;
|
||||
// Limit both transaction size and work per hour; defer excess backlog.
|
||||
for _ in 0..20 {
|
||||
match batch(state.pool(), query, age).await {
|
||||
Ok(count) => { removed += count; if count < 1000 { break; } }
|
||||
Err(error) => { tracing::warn!(name, %error, "retention batch failed; retry next hour"); break; }
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
|
||||
}
|
||||
if let Err(error) = clean_frames(&state, rules[3].2).await {
|
||||
tracing::warn!(%error, "recording file retention failed; retry next hour");
|
||||
}
|
||||
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
|
||||
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB").ok().and_then(|v| v.parse::<i64>().ok()).filter(|v| *v > 0 && *v < 1_000_000).unwrap_or(1024);
|
||||
tracing::info!(bytes, "database size after retention");
|
||||
if bytes > warn_mb * 1024 * 1024 { tracing::warn!(bytes, warn_mb, "database exceeds configured size warning; review retained conversations and memories"); }
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(3600)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn batch(pool: &PgPool, query: &str, age: i32) -> Result<u64, sqlx::Error> {
|
||||
let mut tx = pool.begin().await?;
|
||||
// One maintenance writer, even when multiple API processes start together.
|
||||
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)").fetch_one(&mut *tx).await?;
|
||||
if !acquired { return Ok(0); }
|
||||
sqlx::query("SET LOCAL statement_timeout = '10s'").execute(&mut *tx).await?;
|
||||
sqlx::query("SET LOCAL lock_timeout = '1s'").execute(&mut *tx).await?;
|
||||
let result = sqlx::query(query).bind(age).bind(1000_i64).execute(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let root = std::path::Path::new(&state.data_dir).join("teach");
|
||||
let mut dirs = match tokio::fs::read_dir(&root).await {
|
||||
Ok(dirs) => dirs,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let mut cleaned = 0;
|
||||
while let Some(entry) = dirs.next_entry().await? {
|
||||
if cleaned >= 1000 { break; }
|
||||
// Ignore symlinks and unexpected names; never traverse a user's home.
|
||||
if !entry.file_type().await?.is_dir() { continue; }
|
||||
let id = entry.file_name().to_string_lossy().into_owned();
|
||||
if uuid::Uuid::parse_str(&id).is_err() { continue; }
|
||||
let eligible: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT status IN ('saved','failed','draft') AND updated_at < now() - make_interval(days => $2) FROM taught_skills WHERE id=$1"
|
||||
).bind(&id).bind(age).fetch_optional(state.pool()).await?;
|
||||
let old_orphan = eligible.is_none() && entry.metadata().await?.modified()?.elapsed().unwrap_or(Duration::ZERO) > Duration::from_secs(age as u64 * 86400);
|
||||
if eligible == Some(true) || old_orphan {
|
||||
tokio::fs::remove_dir_all(entry.path()).await?;
|
||||
cleaned += 1;
|
||||
}
|
||||
}
|
||||
if cleaned > 0 { tracing::info!(cleaned, "removed expired teaching frame directories"); }
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
UPDATE runs SET checkpoint = '{}'::jsonb WHERE id IN (
|
||||
SELECT id FROM runs WHERE status IN ('completed','failed','cancelled')
|
||||
AND COALESCE(completed_at,updated_at) < now() - make_interval(days => $1)
|
||||
AND checkpoint <> '{}'::jsonb
|
||||
ORDER BY COALESCE(completed_at,updated_at) LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
DELETE FROM memory_items WHERE id IN (
|
||||
SELECT id FROM memory_items WHERE deleted_at < now() - make_interval(days => $1)
|
||||
ORDER BY deleted_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
DELETE FROM events WHERE id IN (
|
||||
SELECT id FROM events WHERE created_at < now() - make_interval(days => $1)
|
||||
ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
DELETE FROM computer_execution_leases WHERE id IN (
|
||||
SELECT l.id FROM computer_execution_leases l WHERE l.expires_at < now() - make_interval(days => $1)
|
||||
AND NOT EXISTS (SELECT 1 FROM runs r WHERE r.id=l.run_id AND r.status NOT IN ('completed','failed','cancelled'))
|
||||
ORDER BY l.expires_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
DELETE FROM computer_profile_locks WHERE (computer_id,profile_key) IN (
|
||||
SELECT l.computer_id,l.profile_key FROM computer_profile_locks l
|
||||
WHERE l.expires_at < now() - make_interval(days => $1)
|
||||
AND NOT EXISTS (SELECT 1 FROM runs r WHERE r.id=l.run_id AND r.status NOT IN ('completed','failed','cancelled'))
|
||||
ORDER BY l.expires_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
UPDATE taught_skills SET recording = '{}'::jsonb WHERE id IN (
|
||||
SELECT id FROM taught_skills WHERE status IN ('saved','failed','draft')
|
||||
AND updated_at < now() - make_interval(days => $1) AND recording <> '{}'::jsonb
|
||||
ORDER BY updated_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
DELETE FROM memory_revisions WHERE (memory_id,revision) IN (
|
||||
SELECT r.memory_id,r.revision FROM memory_revisions r JOIN memory_items m ON m.id=r.memory_id
|
||||
WHERE r.revision < m.revision AND (
|
||||
r.created_at < now() - make_interval(days => $1)
|
||||
OR r.revision <= m.revision - 10
|
||||
) ORDER BY r.created_at LIMIT $2 FOR UPDATE OF r SKIP LOCKED
|
||||
)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
DELETE FROM runs WHERE id IN (
|
||||
SELECT id FROM runs WHERE status IN ('completed','failed','cancelled')
|
||||
AND COALESCE(completed_at,updated_at) < now() - make_interval(days => $1)
|
||||
ORDER BY COALESCE(completed_at,updated_at) LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -18,7 +18,9 @@ pub fn router(state: AppState) -> Router {
|
|||
.merge(crate::rooms::router())
|
||||
.merge(crate::mcp::router())
|
||||
.merge(crate::workspace::router())
|
||||
.merge(crate::voice::router())
|
||||
.merge(crate::skills::router())
|
||||
.route("/api/file-skills", get(file_skills))
|
||||
.merge(crate::vault::router())
|
||||
.merge(crate::schedules::router())
|
||||
.route("/api/bots", get(list_bots).post(create_bot))
|
||||
|
|
@ -51,6 +53,10 @@ pub fn router(state: AppState) -> Router {
|
|||
.with_state(state)
|
||||
}
|
||||
|
||||
async fn file_skills(State(state): State<AppState>) -> Json<Vec<crate::file_skills::FileSkill>> {
|
||||
Json(crate::file_skills::list(&state.data_dir))
|
||||
}
|
||||
|
||||
async fn actor(state: &AppState) -> Result<Actor, StatusCode> {
|
||||
state
|
||||
.bootstrap()
|
||||
|
|
@ -602,12 +608,12 @@ async fn restart(
|
|||
async fn stop(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<String>,
|
||||
) -> Result<Json<Value>, StatusCode> {
|
||||
let actor = actor(&state).await?;
|
||||
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
|
||||
let actor = actor(&state).await.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
|
||||
computer::stop(&state, &actor, &id)
|
||||
.await
|
||||
.map(|status| Json(serde_json::to_value(status).unwrap()))
|
||||
.map_err(|_| StatusCode::BAD_REQUEST)
|
||||
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message":error}))))
|
||||
}
|
||||
|
||||
async fn screen_url(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use lazyboy_harness::execution::{ExecutionMode, GoalOutcome, goal_request, goal_outcome, GOAL_INSTRUCTIONS, GOAL_CONTINUE};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
@ -130,7 +131,31 @@ pub async fn send(
|
|||
}));
|
||||
}
|
||||
}
|
||||
let run_id = Uuid::new_v4().to_string();
|
||||
// A message sent while a single-bot /goal is active is steering for that
|
||||
// run. Reuse its id so it is delivered by the persistent loop rather than
|
||||
// creating a duplicate queued run that would repeat the work afterwards.
|
||||
let merged_goal_run: Option<String> = if room_id.is_none() {
|
||||
sqlx::query_scalar(
|
||||
"SELECT id FROM runs
|
||||
WHERE bot_id=$1 AND thread_id=$2
|
||||
AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
AND btrim(prompt) ~ '^/goal($|[[:space:]])'
|
||||
ORDER BY created_at ASC LIMIT 1",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.bind(thread_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let merged_goal = merged_goal_run.is_some();
|
||||
let run_id = merged_goal_run.unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
if merged_goal {
|
||||
sqlx::query("UPDATE runs SET status='queued', retry_count=0, updated_at=now() WHERE id=$1 AND status='waiting_input'")
|
||||
.bind(&run_id).execute(&mut *tx).await.map_err(|error| error.to_string())?;
|
||||
}
|
||||
let message_id = Uuid::new_v4().to_string();
|
||||
let seq: i32 = sqlx::query_scalar(
|
||||
"UPDATE threads
|
||||
|
|
@ -176,6 +201,9 @@ pub async fn send(
|
|||
member_ids.push(bot_id.to_string());
|
||||
}
|
||||
for (index, member_id) in member_ids.iter().enumerate() {
|
||||
if merged_goal && index == 0 {
|
||||
continue;
|
||||
}
|
||||
let member_run = if index == 0 {
|
||||
run_id.clone()
|
||||
} else {
|
||||
|
|
@ -213,7 +241,10 @@ pub async fn send(
|
|||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let queued_behind_active: bool = sqlx::query_scalar(
|
||||
let queued_behind_active = if merged_goal {
|
||||
true
|
||||
} else {
|
||||
sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT 1 FROM runs WHERE bot_id=$1 AND id<>$2
|
||||
AND status IN ('queued','leased','running','waiting_input','waiting_takeover'))",
|
||||
)
|
||||
|
|
@ -221,7 +252,8 @@ pub async fn send(
|
|||
.bind(&run_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
.map_err(|error| error.to_string())?
|
||||
};
|
||||
if let Some(room_id) = room_id.as_deref() {
|
||||
sqlx::query("UPDATE rooms SET updated_at=now() WHERE id=$1")
|
||||
.bind(room_id)
|
||||
|
|
@ -508,12 +540,31 @@ async fn execute_run(
|
|||
});
|
||||
}
|
||||
}
|
||||
let goal_mode = goal_request(prompt).is_some();
|
||||
let goal_text = prompt
|
||||
.trim()
|
||||
.strip_prefix("/goal")
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
let file_skill = if !resume_after_takeover && !goal_mode {
|
||||
crate::file_skills::slash(prompt, &state.data_dir)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let initial_prompt = if goal_mode {
|
||||
format!("Execute this goal until it is verified complete:\n{}", goal_text)
|
||||
} else if let Some((skill, args)) = &file_skill {
|
||||
format!("Run the /{} skill with these arguments: {}", skill.name, args)
|
||||
} else {
|
||||
prompt.to_string()
|
||||
};
|
||||
let mut first = if resume_after_takeover {
|
||||
vec![UserContent::text(
|
||||
"The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch.",
|
||||
)]
|
||||
} else {
|
||||
vec![UserContent::text(prompt)]
|
||||
vec![UserContent::text(initial_prompt)]
|
||||
};
|
||||
let blocks: Vec<Value> = if resume_after_takeover {
|
||||
Vec::new()
|
||||
|
|
@ -537,12 +588,19 @@ async fn execute_run(
|
|||
if !resume_after_takeover {
|
||||
// The user named a taught skill: hand the model the full playbook up
|
||||
// front so it does not have to guess or call use_skill first.
|
||||
if let Some(skill) = crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await {
|
||||
if let Some(skill) = if goal_mode || file_skill.is_some() { None } else { crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await } {
|
||||
first.push(UserContent::text(crate::skills::format_playbook_for_run(
|
||||
&skill,
|
||||
)));
|
||||
skill_check = Some(crate::skills::skill_check_hint(&skill));
|
||||
}
|
||||
if let Some((skill, args)) = &file_skill {
|
||||
first.push(UserContent::text(format!(
|
||||
"File skill /{} (read-only source, follow its instructions):\n{}\nArguments: {}",
|
||||
skill.name, skill.instructions, args
|
||||
)));
|
||||
skill_check = Some(format!("完成 /{} 的指令，並用工具驗證結果。", skill.name));
|
||||
}
|
||||
}
|
||||
let mut earlier_replies: Vec<String> = assistant_texts(&history);
|
||||
if skill_check.is_some() {
|
||||
|
|
@ -560,7 +618,7 @@ async fn execute_run(
|
|||
// Greetings and small talk must not even *see* desktop tools: models
|
||||
// otherwise "check the screen" or `ls` the home on "hi" and boot Docker.
|
||||
let chat_only =
|
||||
!resume_after_takeover && skill_check.is_none() && !workspace_file && is_plain_chat(prompt);
|
||||
!resume_after_takeover && !goal_mode && file_skill.is_none() && skill_check.is_none() && !workspace_file && is_plain_chat(prompt);
|
||||
if chat_only {
|
||||
// Memory is recalled separately and injected into the preamble below.
|
||||
// Do not expose even memory tools here: a plain greeting must be one
|
||||
|
|
@ -570,14 +628,16 @@ async fn execute_run(
|
|||
}
|
||||
// Taught skills run long (a 24-page course is 24 clicks); plain chats stay
|
||||
// bounded tighter so a confused model cannot burn budget for as long.
|
||||
let max_turns: u32 = if skill_check.is_some() {
|
||||
80
|
||||
let execution_mode = if goal_mode {
|
||||
ExecutionMode::Goal
|
||||
} else if skill_check.is_some() && file_skill.is_none() {
|
||||
ExecutionMode::Bounded(80)
|
||||
} else if chat_only {
|
||||
4
|
||||
ExecutionMode::Bounded(4)
|
||||
} else {
|
||||
40
|
||||
ExecutionMode::Bounded(40)
|
||||
};
|
||||
let mut nudges: u8 = 0;
|
||||
let mut nudges: u32 = 0;
|
||||
let mut screenshots: u32 = 0;
|
||||
let mut screenshot_bytes: u64 = 0;
|
||||
if resume_after_takeover {
|
||||
|
|
@ -618,6 +678,11 @@ async fn execute_run(
|
|||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0)
|
||||
.min(u32::MAX as u64) as u32;
|
||||
// Messages sent to this same thread while a /goal run is working are
|
||||
// steering input. Keep the run alive and deliver each new message once
|
||||
// before the next model turn instead of waiting for a second run to win
|
||||
// the bot lease.
|
||||
let mut steering_seq = checkpoint.get("steeringSeq").and_then(Value::as_i64).map(|seq| seq as i32).unwrap_or(current_seq);
|
||||
let mut used_gui = false;
|
||||
let memory = if ctx.memory_enabled {
|
||||
match state
|
||||
|
|
@ -677,11 +742,20 @@ async fn execute_run(
|
|||
preamble.push_str("\n\n");
|
||||
preamble.push_str(&memory);
|
||||
}
|
||||
if goal_mode {
|
||||
preamble.push_str("\n\n");
|
||||
preamble.push_str(GOAL_INSTRUCTIONS);
|
||||
}
|
||||
if !chat_only {
|
||||
if let Some(index) = crate::skills::skills_preamble(&skills) {
|
||||
preamble.push_str("\n\n");
|
||||
preamble.push_str(&index);
|
||||
}
|
||||
let file_skill_index = crate::file_skills::index(&state.data_dir);
|
||||
if !file_skill_index.is_empty() {
|
||||
preamble.push_str("\n\n");
|
||||
preamble.push_str(&file_skill_index);
|
||||
}
|
||||
if let Ok(accounts) = crate::vault::list_on(state.pool(), actor, bot_id).await {
|
||||
if !accounts.is_empty() {
|
||||
let names = accounts
|
||||
|
|
@ -696,8 +770,37 @@ async fn execute_run(
|
|||
}
|
||||
}
|
||||
|
||||
for _ in turns..max_turns {
|
||||
turns += 1;
|
||||
while execution_mode.allows_turn(turns) {
|
||||
turns = turns.saturating_add(1);
|
||||
if goal_mode {
|
||||
let steering: Vec<(i32, String)> = sqlx::query_as(
|
||||
"SELECT seq, body FROM messages
|
||||
WHERE thread_id=$1 AND role='user' AND seq>$2
|
||||
ORDER BY seq ASC LIMIT 12",
|
||||
)
|
||||
.bind(thread_id)
|
||||
.bind(steering_seq)
|
||||
.fetch_all(state.pool())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if let Some((last_seq, _)) = steering.last() {
|
||||
steering_seq = *last_seq;
|
||||
let guidance = steering
|
||||
.iter()
|
||||
.map(|(_, body)| body.trim())
|
||||
.filter(|body| !body.is_empty())
|
||||
.collect::<Vec<_>>();
|
||||
if !guidance.is_empty() {
|
||||
let text = format!(
|
||||
"The user added this guidance in the same goal thread. Incorporate it into the current goal and continue verifying the result:\n{}",
|
||||
guidance.join("\n")
|
||||
);
|
||||
if let Message::User { content } = &mut pending {
|
||||
content.push(UserContent::text(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(halt) = renew_or_halt(state, run_id, lease_owner).await? {
|
||||
return finish_halt(
|
||||
state,
|
||||
|
|
@ -761,7 +864,12 @@ async fn execute_run(
|
|||
let parroted = earlier_replies
|
||||
.iter()
|
||||
.any(|earlier| earlier == final_text.trim());
|
||||
let nudge = if parroted {
|
||||
let nudge = if goal_mode {
|
||||
match goal_outcome(&final_text) {
|
||||
GoalOutcome::Continue => Some(GOAL_CONTINUE.to_string()),
|
||||
GoalOutcome::Complete | GoalOutcome::NeedsInput => None,
|
||||
}
|
||||
} else if parroted {
|
||||
Some(
|
||||
"Your reply repeats an earlier message word for word, so it cannot describe the current screen. Below is what the screen shows RIGHT NOW. Act on it with a tool call. Waiting is done by calling wait or by clicking the control (the click waits for it to enable), never by replying. Reply in text only once the task is finished or you are truly blocked (say why).".to_string(),
|
||||
)
|
||||
|
|
@ -773,8 +881,8 @@ async fn execute_run(
|
|||
})
|
||||
};
|
||||
match nudge {
|
||||
Some(text) if nudges < 6 && turns + 2 < max_turns => {
|
||||
nudges += 1;
|
||||
Some(text) if goal_mode || (nudges < 6 && execution_mode.allows_turn(turns.saturating_add(2))) => {
|
||||
nudges = nudges.saturating_add(1);
|
||||
tracing::info!(
|
||||
run_id,
|
||||
turn = turns,
|
||||
|
|
@ -958,7 +1066,7 @@ async fn execute_run(
|
|||
results.extend(screenshot_parts(png));
|
||||
}
|
||||
pending = Message::User { content: results };
|
||||
save_harness_checkpoint(state, run_id, lease_owner, &history, &pending, turns).await?;
|
||||
save_harness_checkpoint(state, run_id, lease_owner, &history, &pending, turns, steering_seq).await?;
|
||||
}
|
||||
|
||||
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id = $1")
|
||||
|
|
@ -980,15 +1088,26 @@ async fn execute_run(
|
|||
)
|
||||
.await;
|
||||
}
|
||||
let needs_input = goal_mode && goal_outcome(&final_text) == GoalOutcome::NeedsInput;
|
||||
if needs_input {
|
||||
let next = Message::User { content: vec![UserContent::text("The goal was paused for required user input. Read the user's new information and continue from completed work.")] };
|
||||
save_harness_checkpoint(state, run_id, lease_owner, &history, &next, turns, steering_seq).await?;
|
||||
}
|
||||
let final_text = final_text
|
||||
.replace("[GOAL_COMPLETE]", "")
|
||||
.replace("[GOAL_BLOCKED]", "")
|
||||
.trim()
|
||||
.to_string();
|
||||
append_bot_message(state, thread_id, run_id, bot_id, &final_text).await?;
|
||||
let completed = sqlx::query(
|
||||
"UPDATE runs
|
||||
SET status='completed', completed_at=now(), updated_at=now(),
|
||||
SET status=$3, completed_at=CASE WHEN $3='completed' THEN now() ELSE NULL END, updated_at=now(),
|
||||
lease_owner=NULL, lease_expires_at=NULL
|
||||
WHERE id=$1 AND lease_owner=$2 AND status='running'",
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(lease_owner)
|
||||
.bind(if needs_input { "waiting_input" } else { "completed" })
|
||||
.execute(state.pool())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
|
@ -1001,7 +1120,7 @@ async fn execute_run(
|
|||
state,
|
||||
thread_id,
|
||||
run_id,
|
||||
"run.completed",
|
||||
if needs_input { "run.paused" } else { "run.completed" },
|
||||
turns,
|
||||
screenshots,
|
||||
screenshot_bytes,
|
||||
|
|
@ -1199,6 +1318,7 @@ async fn save_harness_checkpoint(
|
|||
history: &[Message],
|
||||
pending: &Message,
|
||||
turns: u32,
|
||||
steering_seq: i32,
|
||||
) -> Result<(), String> {
|
||||
let mut history = history.to_vec();
|
||||
let mut pending = pending.clone();
|
||||
|
|
@ -1210,7 +1330,7 @@ async fn save_harness_checkpoint(
|
|||
});
|
||||
}
|
||||
}
|
||||
let value = json!({"harnessHistory":history,"harnessPending":pending,"toolsStarted":false,"harnessTurns":turns});
|
||||
let value = json!({"harnessHistory":history,"harnessPending":pending,"toolsStarted":false,"harnessTurns":turns,"steeringSeq":steering_seq});
|
||||
// Large/unsupported checkpoints fail closed: keep the uncertain-effects flag.
|
||||
if value.to_string().len() > 1024 * 1024 {
|
||||
return Ok(());
|
||||
|
|
|
|||
|
|
@ -1,15 +1,40 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use lazyboy_control::SandboxProvider;
|
||||
use lazyboy_sandbox::{DockerSandbox, FakeSandbox};
|
||||
use sqlx::PgPool;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::auth::AuthConfig;
|
||||
use crate::db::{Actor, Db};
|
||||
use crate::mcp::McpHub;
|
||||
use crate::memory::MemoryService;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CallRegistry {
|
||||
inner: Arc<Mutex<HashMap<String, String>>>,
|
||||
}
|
||||
|
||||
impl CallRegistry {
|
||||
pub async fn try_begin(&self, bot_id: &str, call_id: &str) -> bool {
|
||||
let mut map = self.inner.lock().await;
|
||||
if map.contains_key(bot_id) {
|
||||
return false;
|
||||
}
|
||||
map.insert(bot_id.to_string(), call_id.to_string());
|
||||
true
|
||||
}
|
||||
|
||||
pub async fn end(&self, bot_id: &str, call_id: &str) {
|
||||
let mut map = self.inner.lock().await;
|
||||
if map.get(bot_id).is_some_and(|held| held == call_id) {
|
||||
map.remove(bot_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub db: Db,
|
||||
|
|
@ -18,6 +43,7 @@ pub struct AppState {
|
|||
pub auth: AuthConfig,
|
||||
pub memory: MemoryService,
|
||||
pub mcp: McpHub,
|
||||
pub calls: CallRegistry,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
|
|
@ -39,6 +65,7 @@ impl AppState {
|
|||
auth: AuthConfig::from_env(),
|
||||
memory: MemoryService::from_env(),
|
||||
mcp: McpHub::new(),
|
||||
calls: CallRegistry::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1432,5 +1432,6 @@ fn schedule_state(ctx: &ToolCtx) -> crate::state::AppState {
|
|||
auth: crate::auth::AuthConfig::from_env(),
|
||||
memory: ctx.memory.clone(),
|
||||
mcp: ctx.mcp.clone(),
|
||||
calls: crate::state::CallRegistry::default(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,297 @@
|
|||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use lazyboy_contracts::{VoiceProvider, catalog_voices, computer_voice_tools};
|
||||
use lazyboy_harness::{
|
||||
CredentialChain, ResolveVoiceRequest, resolve_voice, scripted_voice_enabled, voice_catalog,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::db::{Actor, SpaceRow};
|
||||
use crate::state::AppState;
|
||||
use crate::voice_call;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/voice/settings", get(get_settings).patch(update_settings))
|
||||
.route("/api/sessions/{id}/call", get(voice_call::call_ws))
|
||||
}
|
||||
|
||||
async fn actor(state: &AppState) -> Result<Actor, StatusCode> {
|
||||
state
|
||||
.bootstrap()
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
|
||||
pub fn voice_credential_chain(
|
||||
provider: VoiceProvider,
|
||||
voice_api_key: Option<&str>,
|
||||
text_provider: &str,
|
||||
text_api_key: Option<&str>,
|
||||
env_key: Option<&str>,
|
||||
) -> CredentialChain {
|
||||
let nonempty = |value: Option<&str>| {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|item| !item.is_empty())
|
||||
.map(str::to_string)
|
||||
};
|
||||
let space = nonempty(voice_api_key).or_else(|| {
|
||||
if text_provider == provider.as_str() {
|
||||
nonempty(text_api_key)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
CredentialChain {
|
||||
bot: None,
|
||||
space,
|
||||
env: nonempty(env_key),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn voice_instructions(bot_name: &str, bot_instructions: &str) -> String {
|
||||
let mut text = format!(
|
||||
"You are {bot_name}, on a live voice call. Speak shortly and naturally in the user's language. Do not read markdown aloud.\n\
|
||||
You cannot see the screen. Computer progress arrives as text from tools.\n\
|
||||
If the user wants you to operate the computer, answer yes or no immediately, then call start_computer_task. Never pretend you already clicked.\n\
|
||||
If a task is already running and they change their mind, call follow_up_computer_task or stop_computer_task.\n\
|
||||
If a tool returns blocked because they have the screen, tell them to finish on the right-hand desktop."
|
||||
);
|
||||
let extra = bot_instructions.trim();
|
||||
if !extra.is_empty() {
|
||||
text.push_str("\n\nBot instructions:\n");
|
||||
text.push_str(extra);
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
fn listed_providers() -> Vec<VoiceProvider> {
|
||||
let mut providers = VoiceProvider::selectable().to_vec();
|
||||
if scripted_voice_enabled() {
|
||||
providers.push(VoiceProvider::Scripted);
|
||||
}
|
||||
providers
|
||||
}
|
||||
|
||||
fn settings_json(space: &SpaceRow) -> Value {
|
||||
let requested = space
|
||||
.voice_provider
|
||||
.as_deref()
|
||||
.unwrap_or("xai")
|
||||
.parse::<VoiceProvider>()
|
||||
.unwrap_or(VoiceProvider::Xai);
|
||||
let provider = if requested == VoiceProvider::Scripted && !scripted_voice_enabled() {
|
||||
VoiceProvider::Xai
|
||||
} else {
|
||||
requested
|
||||
};
|
||||
let env_key = std::env::var(provider.env_key_name())
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
let credentials = voice_credential_chain(
|
||||
provider,
|
||||
space.voice_api_key.as_deref(),
|
||||
&space.default_model_provider,
|
||||
space.default_model_api_key.as_deref(),
|
||||
env_key.as_deref(),
|
||||
);
|
||||
let resolved = resolve_voice(ResolveVoiceRequest {
|
||||
provider,
|
||||
model_id: space.voice_model_id.clone(),
|
||||
voice_id: space.voice_id.clone(),
|
||||
credentials,
|
||||
});
|
||||
let (model_id, voice_id, ready, missing) = match &resolved {
|
||||
Ok(value) => (value.model_id.clone(), value.voice_id.clone(), true, None),
|
||||
Err(error) => (
|
||||
space
|
||||
.voice_model_id
|
||||
.clone()
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| provider.default_model_id().to_string()),
|
||||
space
|
||||
.voice_id
|
||||
.clone()
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| provider.default_voice_id().to_string()),
|
||||
false,
|
||||
Some(error.to_string()),
|
||||
),
|
||||
};
|
||||
let catalog = voice_catalog(provider);
|
||||
json!({
|
||||
"provider": provider.as_str(),
|
||||
"modelId": model_id,
|
||||
"voiceId": voice_id,
|
||||
"enabled": space.voice_enabled,
|
||||
"ready": space.voice_enabled && ready,
|
||||
"missing": missing,
|
||||
"apiKeySet": space.voice_api_key.as_deref().is_some_and(|value| !value.is_empty()),
|
||||
"envKeySet": env_key.is_some(),
|
||||
"envKeyName": provider.env_key_name(),
|
||||
"reusesTextKey": provider.as_str() == space.default_model_provider
|
||||
&& space.voice_api_key.as_deref().is_none_or(|value| value.is_empty())
|
||||
&& space.default_model_api_key.as_deref().is_some_and(|value| !value.is_empty()),
|
||||
"providers": listed_providers().iter().map(|item| json!({
|
||||
"id": item.as_str(),
|
||||
"name": match item {
|
||||
VoiceProvider::Xai => "xAI",
|
||||
VoiceProvider::Openai => "OpenAI",
|
||||
VoiceProvider::Scripted => "Scripted",
|
||||
},
|
||||
"envKeyName": item.env_key_name(),
|
||||
})).collect::<Vec<_>>(),
|
||||
"models": catalog.models.iter().map(|item| json!({"id": item.id, "name": item.name})).collect::<Vec<_>>(),
|
||||
"voices": catalog.voices.iter().map(|item| json!({"id": item.id, "name": item.name})).collect::<Vec<_>>(),
|
||||
"tools": computer_voice_tools(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_settings(State(state): State<AppState>) -> Result<Json<Value>, StatusCode> {
|
||||
let actor = actor(&state).await?;
|
||||
let space = state
|
||||
.db
|
||||
.get_space(&actor)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.ok_or(StatusCode::NOT_FOUND)?;
|
||||
Ok(Json(settings_json(&space)))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UpdateVoiceSettings {
|
||||
#[serde(default)]
|
||||
enabled: Option<bool>,
|
||||
provider: String,
|
||||
model_id: String,
|
||||
voice_id: String,
|
||||
#[serde(default)]
|
||||
api_key: Option<String>,
|
||||
#[serde(default)]
|
||||
clear_api_key: bool,
|
||||
}
|
||||
|
||||
async fn update_settings(
|
||||
State(state): State<AppState>,
|
||||
Json(input): Json<UpdateVoiceSettings>,
|
||||
) -> Result<Json<Value>, StatusCode> {
|
||||
let actor = actor(&state).await?;
|
||||
let provider: VoiceProvider = input
|
||||
.provider
|
||||
.parse()
|
||||
.map_err(|_| StatusCode::BAD_REQUEST)?;
|
||||
if !listed_providers().contains(&provider) {
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
let model_id = input.model_id.trim();
|
||||
let voice_id = input.voice_id.trim();
|
||||
if model_id.is_empty() || voice_id.is_empty() {
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
if catalog_voices(provider).iter().all(|(id, _)| *id != voice_id)
|
||||
&& provider != VoiceProvider::Scripted
|
||||
{
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
let supplied = input
|
||||
.api_key
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty());
|
||||
let api_key = if input.clear_api_key {
|
||||
Some(None)
|
||||
} else {
|
||||
supplied.map(Some)
|
||||
};
|
||||
state
|
||||
.db
|
||||
.update_voice_settings(&actor, input.enabled, provider.as_str(), model_id, voice_id, api_key)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
get_settings(State(state)).await
|
||||
}
|
||||
|
||||
pub fn resolve_space_voice(
|
||||
space: &SpaceRow,
|
||||
) -> Result<(VoiceProvider, lazyboy_harness::ResolvedVoice), String> {
|
||||
if !space.voice_enabled {
|
||||
return Err("語音通話尚未啟用，請到「設定 → 語音」開啟。".into());
|
||||
}
|
||||
let requested = space
|
||||
.voice_provider
|
||||
.as_deref()
|
||||
.unwrap_or("xai")
|
||||
.parse::<VoiceProvider>()
|
||||
.unwrap_or(VoiceProvider::Xai);
|
||||
let provider = if requested == VoiceProvider::Scripted && !scripted_voice_enabled() {
|
||||
VoiceProvider::Xai
|
||||
} else {
|
||||
requested
|
||||
};
|
||||
let env_key = std::env::var(provider.env_key_name())
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
let credentials = voice_credential_chain(
|
||||
provider,
|
||||
space.voice_api_key.as_deref(),
|
||||
&space.default_model_provider,
|
||||
space.default_model_api_key.as_deref(),
|
||||
env_key.as_deref(),
|
||||
);
|
||||
let resolved = resolve_voice(ResolveVoiceRequest {
|
||||
provider,
|
||||
model_id: space.voice_model_id.clone(),
|
||||
voice_id: space.voice_id.clone(),
|
||||
credentials,
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok((provider, resolved))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn voice_key_reuses_matching_text_provider_and_does_not_cross_providers() {
|
||||
let reused = voice_credential_chain(
|
||||
VoiceProvider::Xai,
|
||||
None,
|
||||
"xai",
|
||||
Some("text-key"),
|
||||
Some("env-key"),
|
||||
);
|
||||
assert_eq!(reused.resolve(), Some("text-key"));
|
||||
|
||||
let env_only = voice_credential_chain(
|
||||
VoiceProvider::Xai,
|
||||
None,
|
||||
"opencode-go",
|
||||
Some("go-key"),
|
||||
Some("env-xai"),
|
||||
);
|
||||
assert_eq!(env_only.resolve(), Some("env-xai"));
|
||||
|
||||
let dedicated = voice_credential_chain(
|
||||
VoiceProvider::Openai,
|
||||
Some("voice-openai"),
|
||||
"xai",
|
||||
Some("text-xai"),
|
||||
Some("env-openai"),
|
||||
);
|
||||
assert_eq!(dedicated.resolve(), Some("voice-openai"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn instructions_include_bot_name_and_custom_text() {
|
||||
let text = voice_instructions("阿明", "Always prefer the browser.");
|
||||
assert!(text.contains("阿明"));
|
||||
assert!(text.contains("start_computer_task"));
|
||||
assert!(text.contains("Always prefer the browser."));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,616 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::ws::{Message as AxumMessage, WebSocket, WebSocketUpgrade};
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use lazyboy_contracts::SessionAttachment;
|
||||
use lazyboy_harness::{VoiceConnectRequest, VoiceEvent, VoiceSocket, create_voice};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::db::Actor;
|
||||
use crate::state::AppState;
|
||||
use crate::voice::{resolve_space_voice, voice_instructions};
|
||||
|
||||
struct PreparedCall {
|
||||
call_id: String,
|
||||
bot_id: String,
|
||||
bot_name: String,
|
||||
provider: lazyboy_contracts::VoiceProvider,
|
||||
connect: VoiceConnectRequest,
|
||||
}
|
||||
|
||||
pub async fn call_ws(
|
||||
State(state): State<AppState>,
|
||||
Path(session_id): Path<String>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> axum::response::Response {
|
||||
let actor = match state.bootstrap().await {
|
||||
Ok(actor) => actor,
|
||||
Err(_) => {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"message":"internal error"})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
match prepare_call(&state, &actor, &session_id).await {
|
||||
Ok(prep) => {
|
||||
if !state.calls.try_begin(&prep.bot_id, &prep.call_id).await {
|
||||
return (
|
||||
StatusCode::CONFLICT,
|
||||
Json(json!({"message":"already on a call"})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
let provider = create_voice(prep.provider);
|
||||
let socket = match provider.connect(prep.connect.clone()).await {
|
||||
Ok(socket) => socket,
|
||||
Err(error) => {
|
||||
state.calls.end(&prep.bot_id, &prep.call_id).await;
|
||||
return (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(json!({"message": error.to_string()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
ws.on_upgrade(move |client| run_call(state, actor, session_id, prep, client, socket))
|
||||
.into_response()
|
||||
}
|
||||
Err((status, message)) => (status, Json(json!({"message": message}))).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_call(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
session_id: &str,
|
||||
) -> Result<PreparedCall, (StatusCode, String)> {
|
||||
let row: Option<(String, Option<String>, String, String)> = sqlx::query_as(
|
||||
"SELECT t.bot_id, t.room_id, b.name, b.instructions
|
||||
FROM threads t JOIN bots b ON b.id=t.bot_id
|
||||
WHERE t.id=$1 AND t.space_id=$2 AND t.user_id=$3 AND t.status='active'",
|
||||
)
|
||||
.bind(session_id)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
|
||||
let Some((bot_id, room_id, bot_name, bot_instructions)) = row else {
|
||||
return Err((StatusCode::NOT_FOUND, "session not found".into()));
|
||||
};
|
||||
if room_id.is_some() {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"group calls are not supported".into(),
|
||||
));
|
||||
}
|
||||
let space = state
|
||||
.db
|
||||
.get_space(actor)
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
.ok_or((StatusCode::NOT_FOUND, "workspace not found".into()))?;
|
||||
let (provider, resolved) =
|
||||
resolve_space_voice(&space).map_err(|message| (StatusCode::CONFLICT, message))?;
|
||||
let history = recent_text_history(state, session_id).await.unwrap_or_default();
|
||||
let mut connect = VoiceConnectRequest::from_resolved(
|
||||
&resolved,
|
||||
voice_instructions(&bot_name, &bot_instructions),
|
||||
);
|
||||
connect.history = history;
|
||||
Ok(PreparedCall {
|
||||
call_id: Uuid::new_v4().to_string(),
|
||||
bot_id,
|
||||
bot_name,
|
||||
provider,
|
||||
connect,
|
||||
})
|
||||
}
|
||||
|
||||
async fn recent_text_history(
|
||||
state: &AppState,
|
||||
session_id: &str,
|
||||
) -> Result<Vec<(String, String)>, sqlx::Error> {
|
||||
let rows: Vec<(String, String)> = sqlx::query_as(
|
||||
"SELECT role, body FROM messages
|
||||
WHERE thread_id=$1 AND role IN ('user','assistant') AND body <> ''
|
||||
ORDER BY seq DESC LIMIT 10",
|
||||
)
|
||||
.bind(session_id)
|
||||
.fetch_all(state.pool())
|
||||
.await?;
|
||||
Ok(rows.into_iter().rev().collect())
|
||||
}
|
||||
|
||||
async fn run_call(
|
||||
state: AppState,
|
||||
actor: Actor,
|
||||
session_id: String,
|
||||
prep: PreparedCall,
|
||||
client: WebSocket,
|
||||
mut provider: Box<dyn VoiceSocket>,
|
||||
) {
|
||||
let call_id = prep.call_id.clone();
|
||||
let bot_id = prep.bot_id.clone();
|
||||
let (client_write, mut client_read) = client.split();
|
||||
let client_write = Arc::new(Mutex::new(client_write));
|
||||
let _ = send_json(
|
||||
&client_write,
|
||||
json!({
|
||||
"type": "ready",
|
||||
"callId": call_id,
|
||||
"botId": bot_id,
|
||||
"botName": prep.bot_name,
|
||||
"provider": prep.provider.as_str(),
|
||||
"voice": prep.connect.voice_id,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut user_partial = String::new();
|
||||
let mut assistant_partial = String::new();
|
||||
let mut last_progress = String::new();
|
||||
let mut last_spoken_at = std::time::Instant::now()
|
||||
.checked_sub(Duration::from_secs(30))
|
||||
.unwrap_or_else(std::time::Instant::now);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
client_msg = client_read.next() => {
|
||||
match client_msg {
|
||||
None | Some(Ok(AxumMessage::Close(_))) => break,
|
||||
Some(Err(_)) => break,
|
||||
Some(Ok(AxumMessage::Ping(payload))) => {
|
||||
let _ = client_write.lock().await.send(AxumMessage::Pong(payload)).await;
|
||||
}
|
||||
Some(Ok(AxumMessage::Pong(_))) => {}
|
||||
Some(Ok(AxumMessage::Binary(bytes))) => {
|
||||
if provider.send(VoiceEvent::AudioPcm(bytes.to_vec())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Some(Ok(AxumMessage::Text(_))) => {}
|
||||
}
|
||||
}
|
||||
provider_msg = provider.recv() => {
|
||||
match provider_msg {
|
||||
Ok(None) | Err(_) => break,
|
||||
Ok(Some(event)) => {
|
||||
if handle_provider_event(
|
||||
&state,
|
||||
&actor,
|
||||
&session_id,
|
||||
&prep,
|
||||
&mut provider,
|
||||
&client_write,
|
||||
event,
|
||||
&mut user_partial,
|
||||
&mut assistant_partial,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_millis(800)) => {
|
||||
let _ = watch_computer(
|
||||
&state,
|
||||
&actor,
|
||||
&session_id,
|
||||
&bot_id,
|
||||
prep.provider,
|
||||
&mut provider,
|
||||
&client_write,
|
||||
&mut last_progress,
|
||||
&mut last_spoken_at,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state.calls.end(&bot_id, &call_id).await;
|
||||
}
|
||||
|
||||
async fn handle_provider_event(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
session_id: &str,
|
||||
prep: &PreparedCall,
|
||||
provider: &mut Box<dyn VoiceSocket>,
|
||||
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
|
||||
event: VoiceEvent,
|
||||
user_partial: &mut String,
|
||||
assistant_partial: &mut String,
|
||||
) -> Result<(), ()> {
|
||||
match event {
|
||||
VoiceEvent::AudioPcm(bytes) => {
|
||||
client_write
|
||||
.lock()
|
||||
.await
|
||||
.send(AxumMessage::Binary(bytes.into()))
|
||||
.await
|
||||
.map_err(|_| ())?;
|
||||
}
|
||||
VoiceEvent::SpeechStarted => {
|
||||
send_json(client_write, json!({"type":"speech","state":"started"})).await?;
|
||||
}
|
||||
VoiceEvent::SpeechStopped => {
|
||||
send_json(client_write, json!({"type":"speech","state":"stopped"})).await?;
|
||||
}
|
||||
VoiceEvent::InputTranscript { text, final_ } => {
|
||||
if final_ {
|
||||
let body = if text.trim().is_empty() {
|
||||
user_partial.trim().to_string()
|
||||
} else {
|
||||
text.trim().to_string()
|
||||
};
|
||||
user_partial.clear();
|
||||
if !body.is_empty() {
|
||||
let _ = persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
|
||||
send_json(
|
||||
client_write,
|
||||
json!({"type":"transcript","role":"user","text":body,"final":true}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
user_partial.push_str(&text);
|
||||
send_json(
|
||||
client_write,
|
||||
json!({"type":"transcript","role":"user","text":user_partial,"final":false}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
VoiceEvent::OutputTranscript { text, final_ } => {
|
||||
if final_ {
|
||||
let body = if text.trim().is_empty() {
|
||||
assistant_partial.trim().to_string()
|
||||
} else {
|
||||
text.trim().to_string()
|
||||
};
|
||||
assistant_partial.clear();
|
||||
if !body.is_empty() {
|
||||
let _ =
|
||||
persist_transcript(state, session_id, "assistant", &body, &prep.call_id)
|
||||
.await;
|
||||
send_json(
|
||||
client_write,
|
||||
json!({"type":"transcript","role":"assistant","text":body,"final":true}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
assistant_partial.push_str(&text);
|
||||
send_json(
|
||||
client_write,
|
||||
json!({"type":"transcript","role":"assistant","text":assistant_partial,"final":false}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
VoiceEvent::FunctionCall {
|
||||
call_id,
|
||||
name,
|
||||
arguments,
|
||||
} => {
|
||||
let output = dispatch_voice_tool(state, actor, &prep.bot_id, session_id, &prep.call_id, &name, &arguments)
|
||||
.await;
|
||||
let _ = provider
|
||||
.send(VoiceEvent::FunctionCallOutput {
|
||||
call_id,
|
||||
output: output.to_string(),
|
||||
})
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
let _ = provider.send(VoiceEvent::ResponseCreate).await;
|
||||
if let Some(status) = output.get("status").and_then(Value::as_str) {
|
||||
send_json(
|
||||
client_write,
|
||||
json!({
|
||||
"type":"computer",
|
||||
"status": status,
|
||||
"step": output.get("step"),
|
||||
"takeover": output.get("takeover").and_then(Value::as_bool).unwrap_or(false)
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
VoiceEvent::Error { message } => {
|
||||
send_json(client_write, json!({"type":"error","message":message})).await?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn dispatch_voice_tool(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot_id: &str,
|
||||
session_id: &str,
|
||||
call_id: &str,
|
||||
name: &str,
|
||||
arguments: &str,
|
||||
) -> Value {
|
||||
match name {
|
||||
"computer_status" => computer_tool_status(state, actor, bot_id, session_id).await,
|
||||
"stop_computer_task" => match crate::sessions::cancel_session_runs(state, session_id).await
|
||||
{
|
||||
Ok(ids) => json!({"status":"cancelled","runIds": ids}),
|
||||
Err(error) => json!({"status":"error","reason": error}),
|
||||
},
|
||||
"start_computer_task" | "follow_up_computer_task" => {
|
||||
let prompt = serde_json::from_str::<Value>(arguments)
|
||||
.ok()
|
||||
.and_then(|value| {
|
||||
value
|
||||
.get("prompt")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let prompt = prompt.trim().to_string();
|
||||
if prompt.is_empty() {
|
||||
return json!({"status":"error","reason":"missing prompt"});
|
||||
}
|
||||
start_or_follow(state, actor, bot_id, session_id, call_id, &prompt).await
|
||||
}
|
||||
other => json!({"status":"error","reason": format!("unknown tool {other}")}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_or_follow(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot_id: &str,
|
||||
session_id: &str,
|
||||
call_id: &str,
|
||||
prompt: &str,
|
||||
) -> Value {
|
||||
if crate::skills::recording_skill(state.pool(), bot_id)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
return json!({"status":"blocked","reason":"teaching"});
|
||||
}
|
||||
let status = computer_tool_status(state, actor, bot_id, session_id).await;
|
||||
if status
|
||||
.get("takeover")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
|| status.get("status").and_then(Value::as_str) == Some("takeover")
|
||||
{
|
||||
return json!({"status":"blocked","reason":"takeover","takeover":true});
|
||||
}
|
||||
if status.get("status").and_then(Value::as_str) == Some("user_control") {
|
||||
return json!({"status":"blocked","reason":"user_control"});
|
||||
}
|
||||
let blocks = vec![json!({"kind":"voice","callId": call_id})];
|
||||
match crate::runs::send(
|
||||
state,
|
||||
actor,
|
||||
bot_id,
|
||||
session_id,
|
||||
prompt,
|
||||
Some(&Uuid::new_v4().to_string()),
|
||||
&blocks,
|
||||
&[] as &[SessionAttachment],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => json!({
|
||||
"status": if result.get("queuedBehindActive").and_then(Value::as_bool).unwrap_or(false) {
|
||||
"followed_up"
|
||||
} else {
|
||||
"queued"
|
||||
},
|
||||
"runId": result.get("runId"),
|
||||
}),
|
||||
Err(error) => json!({"status":"error","reason": error}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn computer_tool_status(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot_id: &str,
|
||||
session_id: &str,
|
||||
) -> Value {
|
||||
let holder: Option<String> = sqlx::query_scalar(
|
||||
"SELECT c.control_holder FROM computers c
|
||||
JOIN bots b ON b.computer_id=c.id
|
||||
WHERE b.id=$1 AND b.space_id=$2 AND b.user_id=$3",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
if holder.as_deref() == Some("user") {
|
||||
return json!({"status":"user_control","takeover":false});
|
||||
}
|
||||
let row: Option<(String, String, Option<String>)> = sqlx::query_as(
|
||||
"SELECT id, status, checkpoint->>'step'
|
||||
FROM runs
|
||||
WHERE bot_id=$1 AND thread_id=$2
|
||||
AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
ORDER BY CASE status
|
||||
WHEN 'running' THEN 0 WHEN 'leased' THEN 1
|
||||
WHEN 'waiting_takeover' THEN 2 WHEN 'waiting_input' THEN 3
|
||||
ELSE 4 END,
|
||||
created_at ASC
|
||||
LIMIT 1",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.bind(session_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
match row {
|
||||
Some((run_id, status, step)) if status == "waiting_takeover" => json!({
|
||||
"status":"takeover",
|
||||
"runId": run_id,
|
||||
"step": step,
|
||||
"takeover": true
|
||||
}),
|
||||
Some((run_id, status, step)) => json!({
|
||||
"status": status,
|
||||
"runId": run_id,
|
||||
"step": step,
|
||||
"takeover": false
|
||||
}),
|
||||
None => json!({"status":"idle","takeover":false}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn watch_computer(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
session_id: &str,
|
||||
bot_id: &str,
|
||||
voice_provider: lazyboy_contracts::VoiceProvider,
|
||||
provider: &mut Box<dyn VoiceSocket>,
|
||||
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
|
||||
last_progress: &mut String,
|
||||
last_spoken_at: &mut std::time::Instant,
|
||||
) -> Result<(), ()> {
|
||||
let snapshot = computer_tool_status(state, actor, bot_id, session_id).await;
|
||||
let key = snapshot.to_string();
|
||||
if key == *last_progress {
|
||||
return Ok(());
|
||||
}
|
||||
let previous = last_progress.clone();
|
||||
*last_progress = key;
|
||||
send_json(
|
||||
client_write,
|
||||
json!({
|
||||
"type":"computer",
|
||||
"status": snapshot.get("status"),
|
||||
"step": snapshot.get("step"),
|
||||
"takeover": snapshot.get("takeover")
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
if let Some(line) = speakable_progress(&previous, &snapshot) {
|
||||
let urgent = snapshot.get("status").and_then(Value::as_str)
|
||||
== Some("takeover")
|
||||
|| snapshot.get("status").and_then(Value::as_str) == Some("failed");
|
||||
if urgent || last_spoken_at.elapsed() > Duration::from_secs(6) {
|
||||
let _ = provider.send(VoiceEvent::SpeakNow { text: line }).await;
|
||||
if voice_provider == lazyboy_contracts::VoiceProvider::Openai {
|
||||
let _ = provider.send(VoiceEvent::ResponseCreate).await;
|
||||
}
|
||||
*last_spoken_at = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn speakable_progress(previous: &str, snapshot: &Value) -> Option<String> {
|
||||
let status = snapshot.get("status")?.as_str()?;
|
||||
if previous.contains(&format!("\"status\":\"{status}\"")) {
|
||||
return None;
|
||||
}
|
||||
match status {
|
||||
"takeover" => Some("需要你接手畫面登入或驗證。".into()),
|
||||
"user_control" => Some("畫面現在在你手上，完成後再叫我繼續。".into()),
|
||||
"failed" => Some("這次電腦操作沒做成。".into()),
|
||||
"idle" if previous.contains("running") || previous.contains("queued") => {
|
||||
Some("電腦上的工作做完了。".into())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn persist_transcript(
|
||||
state: &AppState,
|
||||
session_id: &str,
|
||||
role: &str,
|
||||
body: &str,
|
||||
call_id: &str,
|
||||
) -> Result<(), String> {
|
||||
let mut tx = state
|
||||
.pool()
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let seq: i32 = sqlx::query_scalar(
|
||||
"UPDATE threads SET next_message_seq=next_message_seq+1, updated_at=now()
|
||||
WHERE id=$1 RETURNING next_message_seq-1",
|
||||
)
|
||||
.bind(session_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let message_id = Uuid::new_v4().to_string();
|
||||
sqlx::query(
|
||||
"INSERT INTO messages (id,thread_id,seq,role,body,blocks)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)",
|
||||
)
|
||||
.bind(&message_id)
|
||||
.bind(session_id)
|
||||
.bind(seq)
|
||||
.bind(role)
|
||||
.bind(body)
|
||||
.bind(json!([{"kind":"voice","callId": call_id}]))
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
tx.commit().await.map_err(|error| error.to_string())?;
|
||||
let _ = crate::sessions::append_event(
|
||||
state,
|
||||
session_id,
|
||||
"message.created",
|
||||
json!({"id":message_id,"seq":seq,"role":role,"body":body,"voice":true}),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_json(
|
||||
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
|
||||
value: Value,
|
||||
) -> Result<(), ()> {
|
||||
client_write
|
||||
.lock()
|
||||
.await
|
||||
.send(AxumMessage::Text(value.to_string().into()))
|
||||
.await
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::speakable_progress;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn takeover_and_idle_after_work_are_spoken_once() {
|
||||
let takeover = json!({"status":"takeover","takeover":true});
|
||||
assert_eq!(
|
||||
speakable_progress("{\"status\":\"running\"}", &takeover).as_deref(),
|
||||
Some("需要你接手畫面登入或驗證。")
|
||||
);
|
||||
assert!(speakable_progress(&takeover.to_string(), &takeover).is_none());
|
||||
let idle = json!({"status":"idle"});
|
||||
assert!(speakable_progress("{\"status\":\"running\"}", &idle).unwrap().contains("做完"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use axum::Router;
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
|
||||
/// Vite puts built files in `dist/` and source assets in `public/`.
|
||||
/// `LAZYBOY_WEB_DIR` is sometimes the source tree (`apps/web`) and sometimes
|
||||
/// the build output (`apps/web/dist` or Docker `/web`).
|
||||
pub fn resolve_web_roots(configured: &str) -> (PathBuf, Option<PathBuf>) {
|
||||
let configured = PathBuf::from(configured);
|
||||
let dist = if configured.file_name().is_some_and(|name| name == "dist") {
|
||||
configured.clone()
|
||||
} else {
|
||||
configured.join("dist")
|
||||
};
|
||||
let primary = if dist.join("index.html").is_file() {
|
||||
dist
|
||||
} else {
|
||||
configured.clone()
|
||||
};
|
||||
let public = [
|
||||
configured.join("public"),
|
||||
configured
|
||||
.parent()
|
||||
.map(|parent| parent.join("public"))
|
||||
.unwrap_or_default(),
|
||||
]
|
||||
.into_iter()
|
||||
.find(|path| path.is_dir());
|
||||
(primary, public)
|
||||
}
|
||||
|
||||
pub fn static_router(configured: &str) -> Router {
|
||||
let (primary, public) = resolve_web_roots(configured);
|
||||
let fallback = public.clone().unwrap_or_else(|| primary.clone());
|
||||
let files = ServeDir::new(&primary).fallback(ServeDir::new(fallback));
|
||||
let mut router = Router::new().fallback_service(files);
|
||||
if let Some(icon) = first_existing(&[
|
||||
primary.join("favicon.ico"),
|
||||
primary.join("favicon.png"),
|
||||
public
|
||||
.as_ref()
|
||||
.map(|path| path.join("favicon.ico"))
|
||||
.unwrap_or_default(),
|
||||
public
|
||||
.as_ref()
|
||||
.map(|path| path.join("favicon.png"))
|
||||
.unwrap_or_default(),
|
||||
]) {
|
||||
router = router.route_service("/favicon.ico", ServeFile::new(icon));
|
||||
}
|
||||
router
|
||||
}
|
||||
|
||||
fn first_existing(paths: &[PathBuf]) -> Option<PathBuf> {
|
||||
paths.iter().find(|path| path.is_file()).cloned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"lazyboy-web-static-{}-{name}",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = fs::remove_dir_all(&path);
|
||||
fs::create_dir_all(&path).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_tree_falls_back_to_public_assets() {
|
||||
let root = scratch("source");
|
||||
fs::write(root.join("index.html"), "<!doctype html>").unwrap();
|
||||
fs::create_dir_all(root.join("public")).unwrap();
|
||||
fs::write(root.join("public/favicon.svg"), "<svg></svg>").unwrap();
|
||||
let (primary, public) = resolve_web_roots(root.to_str().unwrap());
|
||||
assert_eq!(primary, root);
|
||||
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefers_vite_dist_when_present() {
|
||||
let root = scratch("built");
|
||||
fs::write(root.join("index.html"), "source").unwrap();
|
||||
fs::create_dir_all(root.join("dist")).unwrap();
|
||||
fs::write(root.join("dist/index.html"), "built").unwrap();
|
||||
fs::create_dir_all(root.join("public")).unwrap();
|
||||
let (primary, public) = resolve_web_roots(root.to_str().unwrap());
|
||||
assert_eq!(primary, root.join("dist"));
|
||||
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dist_dir_uses_sibling_public() {
|
||||
let root = scratch("dist-only");
|
||||
let dist = root.join("dist");
|
||||
fs::create_dir_all(&dist).unwrap();
|
||||
fs::write(dist.join("index.html"), "built").unwrap();
|
||||
fs::create_dir_all(root.join("public")).unwrap();
|
||||
let (primary, public) = resolve_web_roots(dist.to_str().unwrap());
|
||||
assert_eq!(primary, dist);
|
||||
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ mod model;
|
|||
mod room;
|
||||
mod run;
|
||||
mod session;
|
||||
mod voice;
|
||||
|
||||
pub use action::*;
|
||||
pub use bot::*;
|
||||
|
|
@ -15,5 +16,6 @@ pub use model::*;
|
|||
pub use room::*;
|
||||
pub use run::*;
|
||||
pub use session::*;
|
||||
pub use voice::*;
|
||||
|
||||
pub type Id = String;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,200 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum VoiceProvider {
|
||||
Xai,
|
||||
Openai,
|
||||
Scripted,
|
||||
}
|
||||
|
||||
impl VoiceProvider {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Xai => "xai",
|
||||
Self::Openai => "openai",
|
||||
Self::Scripted => "scripted",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn env_key_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Xai => "XAI_API_KEY",
|
||||
Self::Openai => "OPENAI_API_KEY",
|
||||
Self::Scripted => "",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn requires_api_key(self) -> bool {
|
||||
!matches!(self, Self::Scripted)
|
||||
}
|
||||
|
||||
pub fn default_model_id(self) -> &'static str {
|
||||
match self {
|
||||
Self::Xai => DEFAULT_XAI_VOICE_MODEL,
|
||||
Self::Openai => DEFAULT_OPENAI_VOICE_MODEL,
|
||||
Self::Scripted => "scripted-voice",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_voice_id(self) -> &'static str {
|
||||
match self {
|
||||
Self::Xai => "eve",
|
||||
Self::Openai => "marin",
|
||||
Self::Scripted => "scripted",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn realtime_url(self, model_id: &str) -> String {
|
||||
match self {
|
||||
Self::Xai => format!("wss://api.x.ai/v1/realtime?model={model_id}"),
|
||||
Self::Openai => format!("wss://api.openai.com/v1/realtime?model={model_id}"),
|
||||
Self::Scripted => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn selectable() -> &'static [Self] {
|
||||
&[Self::Xai, Self::Openai]
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for VoiceProvider {
|
||||
type Err = UnknownVoiceProvider;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"xai" => Ok(Self::Xai),
|
||||
"openai" => Ok(Self::Openai),
|
||||
"scripted" => Ok(Self::Scripted),
|
||||
other => Err(UnknownVoiceProvider(other.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
#[error("unknown voice provider: {0}")]
|
||||
pub struct UnknownVoiceProvider(pub String);
|
||||
|
||||
pub const DEFAULT_XAI_VOICE_MODEL: &str = "grok-voice-latest";
|
||||
pub const DEFAULT_OPENAI_VOICE_MODEL: &str = "gpt-realtime";
|
||||
pub const VOICE_SAMPLE_RATE: u32 = 24_000;
|
||||
|
||||
pub fn catalog_voice_models(provider: VoiceProvider) -> &'static [(&'static str, &'static str)] {
|
||||
match provider {
|
||||
VoiceProvider::Xai => &[
|
||||
("grok-voice-latest", "Grok Voice Latest"),
|
||||
("grok-voice-think-fast-2.0", "Grok Voice Think Fast 2.0"),
|
||||
("grok-voice-think-fast-1.0", "Grok Voice Think Fast 1.0"),
|
||||
],
|
||||
VoiceProvider::Openai => &[("gpt-realtime", "GPT Realtime")],
|
||||
VoiceProvider::Scripted => &[("scripted-voice", "Scripted")],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn catalog_voices(provider: VoiceProvider) -> &'static [(&'static str, &'static str)] {
|
||||
match provider {
|
||||
VoiceProvider::Xai => &[
|
||||
("eve", "Eve"),
|
||||
("ara", "Ara"),
|
||||
("leo", "Leo"),
|
||||
("rex", "Rex"),
|
||||
("sal", "Sal"),
|
||||
],
|
||||
VoiceProvider::Openai => &[
|
||||
("marin", "Marin"),
|
||||
("alloy", "Alloy"),
|
||||
("verse", "Verse"),
|
||||
("cedar", "Cedar"),
|
||||
],
|
||||
VoiceProvider::Scripted => &[("scripted", "Scripted")],
|
||||
}
|
||||
}
|
||||
|
||||
/// Custom functions the voice model may call. LazyBoy executes them; they return immediately.
|
||||
pub fn computer_voice_tools() -> Vec<Value> {
|
||||
vec![
|
||||
function_tool(
|
||||
"start_computer_task",
|
||||
"Start a computer task on the user's Linux desktop. Returns immediately — speak to the user first, do not wait for the work to finish. Use when they want you to operate the computer (open a site, click, download, run a command).",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "What to do on the computer, in the user's language."
|
||||
}
|
||||
},
|
||||
"required": ["prompt"]
|
||||
}),
|
||||
),
|
||||
function_tool(
|
||||
"follow_up_computer_task",
|
||||
"Add or change instructions for the computer task that is already running. Use when the user interrupts with a correction.",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "The new or extra instruction."
|
||||
}
|
||||
},
|
||||
"required": ["prompt"]
|
||||
}),
|
||||
),
|
||||
function_tool(
|
||||
"stop_computer_task",
|
||||
"Cancel the computer task that is running.",
|
||||
json!({ "type": "object", "properties": {} }),
|
||||
),
|
||||
function_tool(
|
||||
"computer_status",
|
||||
"Check whether a computer task is running, queued, needs the user to take over the screen, or is idle.",
|
||||
json!({ "type": "object", "properties": {} }),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn function_tool(name: &str, description: &str, parameters: Value) -> Value {
|
||||
json!({
|
||||
"type": "function",
|
||||
"name": name,
|
||||
"description": description,
|
||||
"parameters": parameters
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn voice_provider_roundtrip() {
|
||||
assert_eq!("xai".parse::<VoiceProvider>().unwrap(), VoiceProvider::Xai);
|
||||
assert_eq!(
|
||||
"openai".parse::<VoiceProvider>().unwrap(),
|
||||
VoiceProvider::Openai
|
||||
);
|
||||
assert!("anthropic".parse::<VoiceProvider>().is_err());
|
||||
assert_eq!(VoiceProvider::Xai.default_voice_id(), "eve");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn computer_tools_are_named() {
|
||||
let tools = computer_voice_tools();
|
||||
let names: Vec<_> = tools
|
||||
.iter()
|
||||
.filter_map(|tool| tool.get("name")?.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
[
|
||||
"start_computer_task",
|
||||
"follow_up_computer_task",
|
||||
"stop_computer_task",
|
||||
"computer_status"
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -8,7 +8,16 @@ publish.workspace = true
|
|||
[dependencies]
|
||||
lazyboy-contracts.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
rig-core.workspace = true
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util.workspace = true
|
||||
base64.workspace = true
|
||||
http = "1"
|
||||
|
||||
reqwest.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls-native-certs = "0.8"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
//! Execution policy belongs to the harness, independently of task playbooks.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ExecutionMode {
|
||||
Bounded(u32),
|
||||
Goal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum GoalOutcome {
|
||||
Continue,
|
||||
Complete,
|
||||
NeedsInput,
|
||||
}
|
||||
|
||||
pub fn goal_request(prompt: &str) -> Option<&str> {
|
||||
let rest = prompt.trim().strip_prefix("/goal")?;
|
||||
(rest.is_empty() || rest.starts_with(char::is_whitespace)).then(|| rest.trim())
|
||||
}
|
||||
|
||||
impl ExecutionMode {
|
||||
pub fn allows_turn(self, turns: u32) -> bool {
|
||||
match self {
|
||||
Self::Goal => true,
|
||||
Self::Bounded(limit) => turns < limit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Only a standalone terminal marker is an outcome, not a quoted mention.
|
||||
pub fn goal_outcome(reply: &str) -> GoalOutcome {
|
||||
match reply.trim().lines().last().map(str::trim) {
|
||||
Some("[GOAL_COMPLETE]") => GoalOutcome::Complete,
|
||||
Some("[GOAL_BLOCKED]") => GoalOutcome::NeedsInput,
|
||||
_ => GoalOutcome::Continue,
|
||||
}
|
||||
}
|
||||
|
||||
pub const GOAL_INSTRUCTIONS: &str = "Persistent goal execution: plan the requested work, execute it, and verify each requested outcome. Intermediate progress replies do not finish the run. Preserve completed work and incorporate user steering. End your final reply with a standalone [GOAL_COMPLETE] line only when all outcomes are verified; explain the verification. When required information or human action is missing, explain exactly what is needed and end with a standalone [GOAL_BLOCKED] line. For login, CAPTCHA or 2FA use request_takeover. Never claim completion merely because you planned the work.";
|
||||
pub const GOAL_CONTINUE: &str = "The goal remains active. Continue the plan with tools and verify the outcome. Finish only with a standalone [GOAL_COMPLETE] line after verification, or [GOAL_BLOCKED] when required human input is missing.";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn goals_are_unbounded_while_normal_runs_remain_bounded() {
|
||||
assert!(ExecutionMode::Goal.allows_turn(40));
|
||||
assert!(ExecutionMode::Goal.allows_turn(u32::MAX));
|
||||
assert!(!ExecutionMode::Bounded(40).allows_turn(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn goal_command_requires_a_token_boundary() {
|
||||
assert_eq!(goal_request(" /goal\nfinish this"), Some("finish this"));
|
||||
assert_eq!(goal_request("/goalkeeper"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_and_quoted_markers_do_not_complete_a_goal() {
|
||||
assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue);
|
||||
assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete);
|
||||
assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
mod resolve;
|
||||
pub mod execution;
|
||||
mod voice;
|
||||
|
||||
pub use resolve::*;
|
||||
pub use voice::*;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,731 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use lazyboy_contracts::{
|
||||
VoiceProvider, catalog_voice_models, catalog_voices, computer_voice_tools,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
|
||||
use crate::{CredentialChain, ModelError};
|
||||
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
pub enum VoiceError {
|
||||
#[error("{0}")]
|
||||
Message(String),
|
||||
#[error("missing credential for {provider} ({env_key})")]
|
||||
MissingCredential { provider: String, env_key: String },
|
||||
#[error("unknown voice provider: {0}")]
|
||||
UnknownProvider(String),
|
||||
}
|
||||
|
||||
impl From<ModelError> for VoiceError {
|
||||
fn from(error: ModelError) -> Self {
|
||||
match error {
|
||||
ModelError::MissingCredential { provider, env_key } => {
|
||||
Self::MissingCredential { provider, env_key }
|
||||
}
|
||||
other => Self::Message(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct VoiceCatalogEntry {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct VoiceCatalog {
|
||||
pub provider: VoiceProvider,
|
||||
pub models: Vec<VoiceCatalogEntry>,
|
||||
pub voices: Vec<VoiceCatalogEntry>,
|
||||
}
|
||||
|
||||
pub fn voice_catalog(provider: VoiceProvider) -> VoiceCatalog {
|
||||
VoiceCatalog {
|
||||
provider,
|
||||
models: catalog_voice_models(provider)
|
||||
.iter()
|
||||
.map(|(id, name)| VoiceCatalogEntry {
|
||||
id: (*id).into(),
|
||||
name: (*name).into(),
|
||||
})
|
||||
.collect(),
|
||||
voices: catalog_voices(provider)
|
||||
.iter()
|
||||
.map(|(id, name)| VoiceCatalogEntry {
|
||||
id: (*id).into(),
|
||||
name: (*name).into(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ResolveVoiceRequest {
|
||||
pub provider: VoiceProvider,
|
||||
pub model_id: Option<String>,
|
||||
pub voice_id: Option<String>,
|
||||
pub credentials: CredentialChain,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ResolvedVoice {
|
||||
pub provider: VoiceProvider,
|
||||
pub model_id: String,
|
||||
pub voice_id: String,
|
||||
pub api_key: String,
|
||||
}
|
||||
|
||||
pub fn resolve_voice(request: ResolveVoiceRequest) -> Result<ResolvedVoice, VoiceError> {
|
||||
let api_key = match request.credentials.resolve() {
|
||||
Some(key) => key.to_string(),
|
||||
None if request.provider.requires_api_key() => {
|
||||
return Err(VoiceError::MissingCredential {
|
||||
provider: request.provider.as_str().to_string(),
|
||||
env_key: request.provider.env_key_name().to_string(),
|
||||
});
|
||||
}
|
||||
None => String::new(),
|
||||
};
|
||||
let model_id = request
|
||||
.model_id
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| request.provider.default_model_id().to_string());
|
||||
let voice_id = request
|
||||
.voice_id
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| request.provider.default_voice_id().to_string());
|
||||
Ok(ResolvedVoice {
|
||||
provider: request.provider,
|
||||
model_id,
|
||||
voice_id,
|
||||
api_key,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum VoiceEvent {
|
||||
AudioPcm(Vec<u8>),
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
InputTranscript { text: String, final_: bool },
|
||||
OutputTranscript { text: String, final_: bool },
|
||||
FunctionCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
},
|
||||
FunctionCallOutput { call_id: String, output: String },
|
||||
SpeakNow { text: String },
|
||||
InjectContext { text: String },
|
||||
ResponseCreate,
|
||||
Error { message: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VoiceConnectRequest {
|
||||
pub api_key: String,
|
||||
pub model_id: String,
|
||||
pub voice_id: String,
|
||||
pub instructions: String,
|
||||
pub tools: Vec<Value>,
|
||||
pub history: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl VoiceConnectRequest {
|
||||
pub fn from_resolved(resolved: &ResolvedVoice, instructions: String) -> Self {
|
||||
Self {
|
||||
api_key: resolved.api_key.clone(),
|
||||
model_id: resolved.model_id.clone(),
|
||||
voice_id: resolved.voice_id.clone(),
|
||||
instructions,
|
||||
tools: computer_voice_tools(),
|
||||
history: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait VoiceRealtime: Send + Sync {
|
||||
fn provider(&self) -> VoiceProvider;
|
||||
async fn connect(
|
||||
&self,
|
||||
request: VoiceConnectRequest,
|
||||
) -> Result<Box<dyn VoiceSocket>, VoiceError>;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait VoiceSocket: Send {
|
||||
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError>;
|
||||
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError>;
|
||||
}
|
||||
|
||||
pub fn create_voice(provider: VoiceProvider) -> Box<dyn VoiceRealtime> {
|
||||
match provider {
|
||||
VoiceProvider::Scripted => Box::new(ScriptedVoice),
|
||||
other => Box::new(HostedVoice { provider: other }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scripted_voice_enabled() -> bool {
|
||||
matches!(
|
||||
std::env::var("LAZYBOY_VOICE_SCRIPTED").as_deref(),
|
||||
Ok("1" | "true" | "yes")
|
||||
)
|
||||
}
|
||||
|
||||
fn native_roots() -> Result<rustls::RootCertStore, VoiceError> {
|
||||
let mut roots = rustls::RootCertStore::empty();
|
||||
let certs = rustls_native_certs::load_native_certs();
|
||||
roots.add_parsable_certificates(certs.certs);
|
||||
if roots.is_empty() {
|
||||
return Err(VoiceError::Message("No trusted TLS certificates available".into()));
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
|
||||
struct HostedVoice {
|
||||
provider: VoiceProvider,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceRealtime for HostedVoice {
|
||||
fn provider(&self) -> VoiceProvider {
|
||||
self.provider
|
||||
}
|
||||
|
||||
async fn connect(
|
||||
&self,
|
||||
request: VoiceConnectRequest,
|
||||
) -> Result<Box<dyn VoiceSocket>, VoiceError> {
|
||||
let url = self.provider.realtime_url(&request.model_id);
|
||||
let mut http_request = url
|
||||
.as_str()
|
||||
.into_client_request()
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?;
|
||||
let header = format!("Bearer {}", request.api_key);
|
||||
http_request.headers_mut().insert(
|
||||
"Authorization",
|
||||
http::HeaderValue::from_str(&header)
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?,
|
||||
);
|
||||
// Choose explicitly: the dependency graph enables both ring and aws-lc-rs.
|
||||
// Rustls's automatic provider selection panics in that configuration.
|
||||
let tls = rustls::ClientConfig::builder_with_provider(
|
||||
Arc::new(rustls::crypto::ring::default_provider()),
|
||||
)
|
||||
.with_safe_default_protocol_versions()
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?
|
||||
.with_root_certificates(native_roots()?)
|
||||
.with_no_client_auth();
|
||||
let (stream, _) = tokio_tungstenite::connect_async_tls_with_config(
|
||||
http_request, None, false, Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?;
|
||||
let (write, read) = stream.split();
|
||||
let socket = HostedSocket {
|
||||
provider: self.provider,
|
||||
write: Mutex::new(write),
|
||||
read,
|
||||
};
|
||||
socket
|
||||
.send_raw(Message::Text(session_update_json(self.provider, &request).into()))
|
||||
.await?;
|
||||
for (role, text) in &request.history {
|
||||
if text.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let item = json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": role,
|
||||
"content": [{
|
||||
"type": if *role == "assistant" { "output_text" } else { "input_text" },
|
||||
"text": text
|
||||
}]
|
||||
}
|
||||
});
|
||||
socket
|
||||
.send_raw(Message::Text(item.to_string().into()))
|
||||
.await?;
|
||||
}
|
||||
Ok(Box::new(socket))
|
||||
}
|
||||
}
|
||||
|
||||
type HostedWrite =
|
||||
futures_util::stream::SplitSink<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>, Message>;
|
||||
type HostedRead =
|
||||
futures_util::stream::SplitStream<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>>;
|
||||
|
||||
struct HostedSocket {
|
||||
provider: VoiceProvider,
|
||||
write: Mutex<HostedWrite>,
|
||||
read: HostedRead,
|
||||
}
|
||||
|
||||
impl HostedSocket {
|
||||
async fn send_raw(&self, message: Message) -> Result<(), VoiceError> {
|
||||
self.write
|
||||
.lock()
|
||||
.await
|
||||
.send(message)
|
||||
.await
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceSocket for HostedSocket {
|
||||
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError> {
|
||||
match encode_provider_event(self.provider, &event) {
|
||||
Some(message) => self.send_raw(message).await,
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError> {
|
||||
loop {
|
||||
match self.read.next().await {
|
||||
None => return Ok(None),
|
||||
Some(Err(error)) => return Err(VoiceError::Message(error.to_string())),
|
||||
Some(Ok(Message::Close(_))) => return Ok(None),
|
||||
Some(Ok(Message::Ping(payload))) => {
|
||||
let _ = self.send_raw(Message::Pong(payload)).await;
|
||||
}
|
||||
Some(Ok(Message::Pong(_))) => {}
|
||||
Some(Ok(Message::Frame(_))) => {}
|
||||
Some(Ok(Message::Binary(bytes))) => {
|
||||
return Ok(Some(VoiceEvent::AudioPcm(bytes.to_vec())));
|
||||
}
|
||||
Some(Ok(Message::Text(text))) => {
|
||||
if let Some(event) = parse_provider_event(&text) {
|
||||
return Ok(Some(event));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn session_update_json(provider: VoiceProvider, request: &VoiceConnectRequest) -> String {
|
||||
let mut session = json!({
|
||||
"voice": request.voice_id,
|
||||
"instructions": request.instructions,
|
||||
"turn_detection": { "type": "server_vad" },
|
||||
"tools": request.tools,
|
||||
"audio": {
|
||||
"input": {
|
||||
"format": { "type": "audio/pcm", "rate": 24000 },
|
||||
"transport": "binary"
|
||||
},
|
||||
"output": {
|
||||
"format": { "type": "audio/pcm", "rate": 24000 },
|
||||
"transport": "binary"
|
||||
}
|
||||
}
|
||||
});
|
||||
if provider == VoiceProvider::Openai {
|
||||
session.as_object_mut().unwrap().remove("voice");
|
||||
session.as_object_mut().unwrap().remove("turn_detection");
|
||||
session["type"] = json!("realtime");
|
||||
session["audio"] = json!({
|
||||
"input": {
|
||||
"format": { "type": "audio/pcm", "rate": 24000 },
|
||||
"turn_detection": { "type": "server_vad" }
|
||||
},
|
||||
"output": {
|
||||
"format": { "type": "audio/pcm", "rate": 24000 },
|
||||
"voice": request.voice_id
|
||||
}
|
||||
});
|
||||
}
|
||||
json!({ "type": "session.update", "session": session }).to_string()
|
||||
}
|
||||
|
||||
pub fn encode_provider_event(provider: VoiceProvider, event: &VoiceEvent) -> Option<Message> {
|
||||
match event {
|
||||
VoiceEvent::AudioPcm(bytes) if provider == VoiceProvider::Openai => Some(Message::Text(
|
||||
json!({ "type": "input_audio_buffer.append", "audio": base64::engine::general_purpose::STANDARD.encode(bytes) }).to_string().into(),
|
||||
)),
|
||||
VoiceEvent::AudioPcm(bytes) => Some(Message::Binary(bytes.clone().into())),
|
||||
VoiceEvent::FunctionCallOutput { call_id, output } => Some(Message::Text(
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "function_call_output",
|
||||
"call_id": call_id,
|
||||
"output": output
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
)),
|
||||
VoiceEvent::ResponseCreate => Some(Message::Text(
|
||||
json!({ "type": "response.create" }).to_string().into(),
|
||||
)),
|
||||
VoiceEvent::InjectContext { text } => Some(Message::Text(
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{ "type": "input_text", "text": text }]
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
)),
|
||||
VoiceEvent::SpeakNow { text } if provider == VoiceProvider::Xai => Some(Message::Text(
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "force_message",
|
||||
"role": "assistant",
|
||||
"interruptible": true,
|
||||
"content": [{ "type": "output_text", "text": text }]
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
)),
|
||||
VoiceEvent::SpeakNow { text } => Some(Message::Text(
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{ "type": "output_text", "text": text }]
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
|
||||
let event: Value = serde_json::from_str(text).ok()?;
|
||||
let kind = event.get("type")?.as_str()?;
|
||||
match kind {
|
||||
"input_audio_buffer.speech_started" => Some(VoiceEvent::SpeechStarted),
|
||||
"input_audio_buffer.speech_stopped" => Some(VoiceEvent::SpeechStopped),
|
||||
"conversation.item.input_audio_transcription.delta"
|
||||
| "response.input_audio_transcription.delta" => {
|
||||
let text = event.get("delta")?.as_str()?.to_string();
|
||||
Some(VoiceEvent::InputTranscript {
|
||||
text,
|
||||
final_: false,
|
||||
})
|
||||
}
|
||||
"conversation.item.input_audio_transcription.completed"
|
||||
| "conversation.item.input_audio_transcription.done"
|
||||
| "response.input_audio_transcription.completed" => {
|
||||
let text = event
|
||||
.get("transcript")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| event.get("text").and_then(Value::as_str))?
|
||||
.to_string();
|
||||
Some(VoiceEvent::InputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
})
|
||||
}
|
||||
"response.output_audio.delta" | "response.audio.delta" => {
|
||||
let encoded = event.get("delta").and_then(Value::as_str)?;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.ok()?;
|
||||
Some(VoiceEvent::AudioPcm(bytes))
|
||||
}
|
||||
"response.output_audio_transcript.delta" | "response.audio_transcript.delta" => {
|
||||
let text = event.get("delta")?.as_str()?.to_string();
|
||||
Some(VoiceEvent::OutputTranscript {
|
||||
text,
|
||||
final_: false,
|
||||
})
|
||||
}
|
||||
"response.output_audio_transcript.done" | "response.audio_transcript.done" => {
|
||||
let text = event
|
||||
.get("transcript")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| event.get("text").and_then(Value::as_str))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
Some(VoiceEvent::OutputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
})
|
||||
}
|
||||
"response.function_call_arguments.done" => {
|
||||
let call_id = event.get("call_id")?.as_str()?.to_string();
|
||||
let name = event.get("name")?.as_str()?.to_string();
|
||||
let arguments = event
|
||||
.get("arguments")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("{}")
|
||||
.to_string();
|
||||
Some(VoiceEvent::FunctionCall {
|
||||
call_id,
|
||||
name,
|
||||
arguments,
|
||||
})
|
||||
}
|
||||
"error" => {
|
||||
let message = event
|
||||
.pointer("/error/message")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| event.get("message").and_then(Value::as_str))
|
||||
.unwrap_or("voice error")
|
||||
.to_string();
|
||||
Some(VoiceEvent::Error { message })
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
struct ScriptedVoice;
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceRealtime for ScriptedVoice {
|
||||
fn provider(&self) -> VoiceProvider {
|
||||
VoiceProvider::Scripted
|
||||
}
|
||||
|
||||
async fn connect(
|
||||
&self,
|
||||
_request: VoiceConnectRequest,
|
||||
) -> Result<Box<dyn VoiceSocket>, VoiceError> {
|
||||
Ok(Box::new(ScriptedSocket::new()))
|
||||
}
|
||||
}
|
||||
|
||||
struct ScriptedSocket {
|
||||
incoming: Mutex<mpsc::UnboundedSender<VoiceEvent>>,
|
||||
outgoing: mpsc::UnboundedReceiver<VoiceEvent>,
|
||||
heard: Arc<Mutex<bool>>,
|
||||
}
|
||||
|
||||
impl ScriptedSocket {
|
||||
fn new() -> Self {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
Self {
|
||||
incoming: Mutex::new(tx),
|
||||
outgoing: rx,
|
||||
heard: Arc::new(Mutex::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn beep() -> Vec<u8> {
|
||||
// 80 ms of 24 kHz PCM16 silence so tests have a non-empty clip.
|
||||
vec![0; 24000 / 12 * 2]
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceSocket for ScriptedSocket {
|
||||
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError> {
|
||||
match event {
|
||||
VoiceEvent::AudioPcm(bytes) if !bytes.is_empty() => {
|
||||
let mut heard = self.heard.lock().await;
|
||||
if !*heard {
|
||||
*heard = true;
|
||||
let tx = self.incoming.lock().await;
|
||||
let _ = tx.send(VoiceEvent::SpeechStarted);
|
||||
let _ = tx.send(VoiceEvent::InputTranscript {
|
||||
text: "hello from the test microphone".into(),
|
||||
final_: true,
|
||||
});
|
||||
let _ = tx.send(VoiceEvent::FunctionCall {
|
||||
call_id: "call_scripted".into(),
|
||||
name: "start_computer_task".into(),
|
||||
arguments: json!({ "prompt": "open the browser" }).to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
VoiceEvent::FunctionCallOutput { .. } => {
|
||||
let tx = self.incoming.lock().await;
|
||||
let _ = tx.send(VoiceEvent::OutputTranscript {
|
||||
text: "好，我去電腦上開。".into(),
|
||||
final_: true,
|
||||
});
|
||||
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
|
||||
}
|
||||
VoiceEvent::SpeakNow { text } | VoiceEvent::InjectContext { text } => {
|
||||
let tx = self.incoming.lock().await;
|
||||
let _ = tx.send(VoiceEvent::OutputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
});
|
||||
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
|
||||
}
|
||||
VoiceEvent::ResponseCreate => {}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError> {
|
||||
Ok(self.outgoing.recv().await)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn voice_tls_config_uses_an_explicit_provider() {
|
||||
let config = rustls::ClientConfig::builder_with_provider(
|
||||
Arc::new(rustls::crypto::ring::default_provider()),
|
||||
).with_safe_default_protocol_versions().unwrap()
|
||||
.with_root_certificates(native_roots().unwrap()).with_no_client_auth();
|
||||
assert!(!config.crypto_provider().cipher_suites.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_audio_uses_json_and_nested_session_settings() {
|
||||
let request = VoiceConnectRequest {
|
||||
api_key: String::new(), model_id: "gpt-realtime".into(),
|
||||
voice_id: "marin".into(), instructions: "test".into(),
|
||||
tools: vec![], history: vec![],
|
||||
};
|
||||
let session: Value = serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
|
||||
assert!(session["session"].get("voice").is_none());
|
||||
assert!(session["session"].get("turn_detection").is_none());
|
||||
assert_eq!(session.pointer("/session/audio/input/format/rate"), Some(&json!(24000)));
|
||||
assert_eq!(session.pointer("/session/audio/input/turn_detection/type"), Some(&json!("server_vad")));
|
||||
let event = VoiceEvent::AudioPcm(vec![0, 1, 2, 3]);
|
||||
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { panic!("expected JSON audio") };
|
||||
let encoded: Value = serde_json::from_str(&text).unwrap();
|
||||
assert_eq!(encoded["type"], "input_audio_buffer.append");
|
||||
assert_eq!(encoded["audio"], "AAECAw==");
|
||||
assert!(matches!(encode_provider_event(VoiceProvider::Xai, &event), Some(Message::Binary(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_voice_defaults_and_requires_key() {
|
||||
let missing = resolve_voice(ResolveVoiceRequest {
|
||||
provider: VoiceProvider::Xai,
|
||||
model_id: None,
|
||||
voice_id: None,
|
||||
credentials: CredentialChain {
|
||||
bot: None,
|
||||
space: None,
|
||||
env: None,
|
||||
},
|
||||
});
|
||||
assert!(matches!(missing, Err(VoiceError::MissingCredential { .. })));
|
||||
|
||||
let ok = resolve_voice(ResolveVoiceRequest {
|
||||
provider: VoiceProvider::Xai,
|
||||
model_id: None,
|
||||
voice_id: None,
|
||||
credentials: CredentialChain {
|
||||
bot: None,
|
||||
space: Some("sk".into()),
|
||||
env: None,
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(ok.model_id, "grok-voice-latest");
|
||||
assert_eq!(ok.voice_id, "eve");
|
||||
assert_eq!(ok.api_key, "sk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_xai_and_openai_event_aliases() {
|
||||
let started = parse_provider_event(
|
||||
r#"{"type":"input_audio_buffer.speech_started"}"#,
|
||||
);
|
||||
assert!(matches!(started, Some(VoiceEvent::SpeechStarted)));
|
||||
|
||||
let transcript = parse_provider_event(
|
||||
r#"{"type":"conversation.item.input_audio_transcription.completed","transcript":"hello"}"#,
|
||||
);
|
||||
match transcript {
|
||||
Some(VoiceEvent::InputTranscript { text, final_ }) => {
|
||||
assert_eq!(text, "hello");
|
||||
assert!(final_);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
|
||||
let old_audio = parse_provider_event(
|
||||
r#"{"type":"response.audio.delta","delta":"AQID"}"#,
|
||||
);
|
||||
assert!(matches!(old_audio, Some(VoiceEvent::AudioPcm(_))));
|
||||
|
||||
let tool = parse_provider_event(
|
||||
r#"{"type":"response.function_call_arguments.done","call_id":"1","name":"start_computer_task","arguments":"{}"}"#,
|
||||
);
|
||||
match tool {
|
||||
Some(VoiceEvent::FunctionCall { name, .. }) => {
|
||||
assert_eq!(name, "start_computer_task");
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scripted_voice_emits_transcript_and_tool() {
|
||||
let provider = create_voice(VoiceProvider::Scripted);
|
||||
let mut socket = provider
|
||||
.connect(VoiceConnectRequest {
|
||||
api_key: String::new(),
|
||||
model_id: "scripted-voice".into(),
|
||||
voice_id: "scripted".into(),
|
||||
instructions: "test".into(),
|
||||
tools: computer_voice_tools(),
|
||||
history: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
socket
|
||||
.send(VoiceEvent::AudioPcm(vec![0, 1, 2, 3]))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut kinds = Vec::new();
|
||||
for _ in 0..3 {
|
||||
match socket.recv().await.unwrap() {
|
||||
Some(VoiceEvent::SpeechStarted) => kinds.push("start"),
|
||||
Some(VoiceEvent::InputTranscript { final_: true, .. }) => kinds.push("in"),
|
||||
Some(VoiceEvent::FunctionCall { name, .. }) => {
|
||||
assert_eq!(name, "start_computer_task");
|
||||
kinds.push("tool");
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(kinds, ["start", "in", "tool"]);
|
||||
socket
|
||||
.send(VoiceEvent::FunctionCallOutput {
|
||||
call_id: "call_scripted".into(),
|
||||
output: json!({"status":"queued"}).to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
match socket.recv().await.unwrap() {
|
||||
Some(VoiceEvent::OutputTranscript { text, final_ }) => {
|
||||
assert!(final_);
|
||||
assert!(text.contains("電腦"));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
match socket.recv().await.unwrap() {
|
||||
Some(VoiceEvent::AudioPcm(bytes)) => assert!(!bytes.is_empty()),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ use bollard::container::{
|
|||
StartContainerOptions, StopContainerOptions,
|
||||
};
|
||||
use bollard::exec::{CreateExecOptions, StartExecOptions, StartExecResults};
|
||||
use bollard::models::{HostConfig, PortBinding};
|
||||
use bollard::models::{HostConfig, HostConfigLogConfig, PortBinding};
|
||||
use bollard::network::CreateNetworkOptions;
|
||||
use futures_util::StreamExt;
|
||||
use lazyboy_control::{
|
||||
|
|
@ -117,7 +117,7 @@ impl DockerHost {
|
|||
let mut labels = HashMap::new();
|
||||
labels.insert("lazyboy.homeKey".into(), home_key.to_string());
|
||||
labels.insert("lazyboy.spaceId".into(), space_id.to_string());
|
||||
labels.insert("lazyboy.controlVersion".into(), "2".into());
|
||||
labels.insert("lazyboy.controlVersion".into(), "3".into());
|
||||
|
||||
let mut port_bindings = HashMap::new();
|
||||
let mut exposed = HashMap::new();
|
||||
|
|
@ -134,13 +134,37 @@ impl DockerHost {
|
|||
}
|
||||
|
||||
let host_config = HostConfig {
|
||||
binds: Some(vec![format!("{home_path}:{HOME}")]),
|
||||
log_config: Some(HostConfigLogConfig {
|
||||
typ: Some("json-file".into()),
|
||||
config: Some(HashMap::from([
|
||||
("max-size".into(), "10m".into()),
|
||||
("max-file".into(), "3".into()),
|
||||
])),
|
||||
}),
|
||||
binds: Some(
|
||||
std::iter::once(format!("{home_path}:{HOME}"))
|
||||
.chain(lxcfs_binds())
|
||||
.collect(),
|
||||
),
|
||||
port_bindings: Some(port_bindings),
|
||||
memory: Some(computer_memory_bytes()),
|
||||
nano_cpus: Some(computer_nano_cpus()),
|
||||
pids_limit: Some(computer_pids_limit()),
|
||||
cap_drop: Some(vec!["ALL".into()]),
|
||||
security_opt: Some(vec!["no-new-privileges:true".into()]),
|
||||
cap_drop: if computer_sudo_enabled() {
|
||||
None
|
||||
} else {
|
||||
Some(vec!["ALL".into()])
|
||||
},
|
||||
cap_add: if computer_sudo_enabled() {
|
||||
None
|
||||
} else {
|
||||
Some(vec!["SETUID".into(), "SETGID".into()])
|
||||
},
|
||||
security_opt: if computer_sudo_enabled() {
|
||||
None
|
||||
} else {
|
||||
Some(vec!["no-new-privileges:true".into()])
|
||||
},
|
||||
privileged: Some(false),
|
||||
shm_size: Some(512 * 1024 * 1024),
|
||||
network_mode: Some(network),
|
||||
|
|
@ -149,7 +173,10 @@ impl DockerHost {
|
|||
|
||||
let config = Config {
|
||||
image: Some(self.image.clone()),
|
||||
user: Some("1000:1000".into()),
|
||||
// The entrypoint starts as root so it can apply the env-only sudo
|
||||
// policy, then drops the desktop process to the unprivileged user.
|
||||
// Exec requests below still run explicitly as 1000:1000.
|
||||
user: None,
|
||||
hostname: Some(name.clone()),
|
||||
env: Some(vec![
|
||||
"DISPLAY=:1".into(),
|
||||
|
|
@ -158,6 +185,10 @@ impl DockerHost {
|
|||
"LAZYBOY_CONTROL_TOKEN={}",
|
||||
scoped_control_token(&self.control_token, home_key)
|
||||
),
|
||||
format!(
|
||||
"LAZYBOY_COMPUTER_SUDO={}",
|
||||
if computer_sudo_enabled() { "true" } else { "false" }
|
||||
),
|
||||
]),
|
||||
labels: Some(labels),
|
||||
exposed_ports: Some(exposed),
|
||||
|
|
@ -624,7 +655,7 @@ PY"#,
|
|||
.and_then(|c| c.labels.as_ref())
|
||||
.and_then(|l| l.get("lazyboy.controlVersion"))
|
||||
.map(String::as_str)
|
||||
!= Some("2")
|
||||
!= Some("3")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
|
@ -999,6 +1030,26 @@ fn computer_pids_limit() -> i64 {
|
|||
.unwrap_or(2048)
|
||||
}
|
||||
|
||||
fn computer_sudo_enabled() -> bool {
|
||||
matches!(std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), Ok("1" | "true" | "yes"))
|
||||
}
|
||||
|
||||
/// LXCFS supplies cgroup-aware /proc views so tools such as htop and free
|
||||
/// report the Agent container's quota instead of the Docker host. It is
|
||||
/// optional because Docker Desktop (macOS/Windows) does not ship LXCFS.
|
||||
fn lxcfs_binds() -> impl Iterator<Item = String> {
|
||||
let root = std::env::var("LAZYBOY_LXCFS_ROOT").ok();
|
||||
["cpuinfo", "loadavg", "meminfo", "stat", "swaps", "uptime"]
|
||||
.into_iter()
|
||||
.filter_map(move |name| {
|
||||
let root = root.as_deref()?;
|
||||
let source = PathBuf::from(root).join("proc").join(name);
|
||||
source
|
||||
.is_file()
|
||||
.then(|| format!("{}:/proc/{name}:ro", source.display()))
|
||||
})
|
||||
}
|
||||
|
||||
fn docker_already(error: &str, needle: &str) -> bool {
|
||||
let error = error.to_ascii_lowercase();
|
||||
error.contains("409") || error.contains(needle)
|
||||
|
|
|
|||
|
|
@ -47,12 +47,18 @@ services:
|
|||
environment:
|
||||
SANDBOX_SUPERVISOR_TOKEN: ${SANDBOX_SUPERVISOR_TOKEN:?Set SANDBOX_SUPERVISOR_TOKEN in .env}
|
||||
LAZYBOY_COMPUTER_IMAGE: lazyboy/computer:local
|
||||
LAZYBOY_COMPUTER_CPUS: ${LAZYBOY_COMPUTER_CPUS:-2}
|
||||
LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
|
||||
LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
|
||||
LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
|
||||
LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
|
||||
SUPERVISOR_BIND: 0.0.0.0:7091
|
||||
DATA_DIR: /data
|
||||
HOST_DATA_DIR: ${LAZYBOY_HOST_DATA_DIR:-${PWD}/data}
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./data:/data
|
||||
- ${LAZYBOY_LXCFS_ROOT:-./data/lxcfs}:/var/lib/lxcfs:ro
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
depends_on:
|
||||
|
|
@ -88,6 +94,12 @@ services:
|
|||
LAZYBOY_MEMORY_MODEL_CACHE: /data/fastembed
|
||||
LAZYBOY_MEMORY_TOP_K: ${LAZYBOY_MEMORY_TOP_K:-8}
|
||||
LAZYBOY_MEMORY_BYTE_BUDGET: ${LAZYBOY_MEMORY_BYTE_BUDGET:-6000}
|
||||
LAZYBOY_EVENT_RETENTION_DAYS: ${LAZYBOY_EVENT_RETENTION_DAYS:-30}
|
||||
LAZYBOY_CHECKPOINT_RETENTION_DAYS: ${LAZYBOY_CHECKPOINT_RETENTION_DAYS:-7}
|
||||
LAZYBOY_RUN_RETENTION_DAYS: ${LAZYBOY_RUN_RETENTION_DAYS:-90}
|
||||
LAZYBOY_RECORDING_RETENTION_DAYS: ${LAZYBOY_RECORDING_RETENTION_DAYS:-30}
|
||||
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS: ${LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS:-90}
|
||||
LAZYBOY_DB_WARN_MB: ${LAZYBOY_DB_WARN_MB:-1024}
|
||||
LAZYBOY_WEB_DIR: /web
|
||||
LAZYBOY_SCREEN_UPSTREAM: host.docker.internal
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ WORKDIR /src/apps/web
|
|||
COPY apps/web/package.json apps/web/package-lock.json ./
|
||||
RUN --mount=type=cache,target=/root/.npm npm ci --no-audit --no-fund
|
||||
COPY apps/web/index.html apps/web/tsconfig.json apps/web/vite.config.ts ./
|
||||
COPY apps/web/public public
|
||||
COPY apps/web/src src
|
||||
RUN npm run build \
|
||||
&& apt-get update \
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ RUN printf '%s\n' \
|
|||
procps \
|
||||
python3 \
|
||||
python3-websocket \
|
||||
sudo \
|
||||
gosu \
|
||||
util-linux \
|
||||
websockify \
|
||||
wmctrl \
|
||||
|
|
@ -97,6 +99,7 @@ RUN useradd --create-home --uid 1000 --shell /bin/zsh lazyboy \
|
|||
&& chown -R 1000:1000 /home/lazyboy /tmp/lazyboy
|
||||
|
||||
COPY --from=controld --chmod=755 /lazyboy-controld /usr/local/bin/lazyboy-controld
|
||||
COPY --chmod=755 image/computer/rotate-logs.py /usr/local/bin/lazyboy-rotate-logs
|
||||
COPY --chmod=755 image/computer/lazyboy-screen /usr/local/bin/lazyboy-screen
|
||||
COPY --chmod=755 image/computer/lazyboy-browser /usr/local/bin/lazyboy-browser
|
||||
COPY --chmod=755 image/computer/lazyboy-terminal /usr/local/bin/lazyboy-terminal
|
||||
|
|
@ -132,11 +135,12 @@ COPY --chmod=644 image/computer/xfce/helper-browser.desktop /usr/share/xfce4/hel
|
|||
COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium
|
||||
RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true
|
||||
COPY --chmod=755 image/computer/start.sh /usr/local/bin/lazyboy-computer
|
||||
COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint
|
||||
|
||||
USER 1000:1000
|
||||
USER root
|
||||
ENV HOME=/home/lazyboy DISPLAY=:1 SHELL=/bin/zsh TERM=xterm-256color \
|
||||
LANG=zh_TW.UTF-8 LC_ALL=zh_TW.UTF-8 LANGUAGE=zh_TW:zh:en \
|
||||
GTK_MODULES=atk-bridge GTK_A11Y=atspi GNOME_ACCESSIBILITY=1 NO_AT_BRIDGE=0
|
||||
WORKDIR /home/lazyboy
|
||||
EXPOSE 6080 6081 6082 6083 6084 6085 6086 6087
|
||||
CMD ["/usr/local/bin/lazyboy-computer"]
|
||||
CMD ["/usr/local/bin/lazyboy-entrypoint"]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${LAZYBOY_COMPUTER_SUDO:-false}" =~ ^(1|true|yes)$ ]]; then
|
||||
printf 'lazyboy ALL=(ALL) NOPASSWD:ALL\n' > /etc/sudoers.d/lazyboy
|
||||
chmod 0440 /etc/sudoers.d/lazyboy
|
||||
else
|
||||
rm -f /etc/sudoers.d/lazyboy
|
||||
fi
|
||||
|
||||
exec gosu lazyboy /usr/local/bin/lazyboy-computer
|
||||
|
|
@ -103,7 +103,7 @@ start_xvfb() {
|
|||
return 0
|
||||
fi
|
||||
rm -f "/tmp/.X${number}-lock" "/tmp/.X11-unix/X${number}"
|
||||
Xvfb "$display" -screen 0 1280x800x24 -ac +extension RANDR +render -noreset >"${log}-xvfb.log" 2>&1 &
|
||||
Xvfb "$display" -screen 0 1280x800x24 -ac +extension RANDR +render -noreset >>"${log}-xvfb.log" 2>&1 &
|
||||
echo $! > "${log}-xvfb.pid"
|
||||
wait_display "$display"
|
||||
}
|
||||
|
|
@ -141,11 +141,11 @@ start_atspi() {
|
|||
fi
|
||||
done
|
||||
if [[ -n "$launcher" ]]; then
|
||||
DISPLAY="$display" "$launcher" --launch-immediately >"${log}-atspi-bus.log" 2>&1 &
|
||||
DISPLAY="$display" "$launcher" --launch-immediately >>"${log}-atspi-bus.log" 2>&1 &
|
||||
echo $! >"${log}-atspi-bus.pid"
|
||||
fi
|
||||
if [[ -n "$registry" ]]; then
|
||||
DISPLAY="$display" "$registry" >"${log}-atspi-registry.log" 2>&1 &
|
||||
DISPLAY="$display" "$registry" >>"${log}-atspi-registry.log" 2>&1 &
|
||||
echo $! >"${log}-atspi-registry.pid"
|
||||
fi
|
||||
sleep 0.2
|
||||
|
|
@ -175,12 +175,12 @@ start_desktop() {
|
|||
xfconfd --daemon >/dev/null 2>&1 || true
|
||||
fi
|
||||
if command -v xfwm4 >/dev/null 2>&1; then
|
||||
xfwm4 --compositor=off --display="$display" --sm-client-disable >"${log}-wm.log" 2>&1 &
|
||||
xfwm4 --compositor=off --display="$display" --sm-client-disable >>"${log}-wm.log" 2>&1 &
|
||||
echo $! >"${log}-wm.pid"
|
||||
sleep 0.3
|
||||
xfdesktop --disable-wm-check --sm-client-disable >"${log}-desktop.log" 2>&1 &
|
||||
xfdesktop --disable-wm-check --sm-client-disable >>"${log}-desktop.log" 2>&1 &
|
||||
echo $! >"${log}-desktop.pid"
|
||||
xfce4-panel --disable-wm-check --sm-client-disable >"${log}-panel.log" 2>&1 &
|
||||
xfce4-panel --disable-wm-check --sm-client-disable >>"${log}-panel.log" 2>&1 &
|
||||
echo $! >"${log}-panel.pid"
|
||||
else
|
||||
echo "XFCE is missing" >&2
|
||||
|
|
@ -196,7 +196,7 @@ start_vnc() {
|
|||
local log="$4"
|
||||
if ! port_open "$vnc_port"; then
|
||||
x11vnc -display "$display" -forever -shared -nopw -listen 127.0.0.1 -rfbport "$vnc_port" \
|
||||
-xkb -repeat -cursor arrow -noxdamage -ncache 0 >"${log}-x11vnc.log" 2>&1 &
|
||||
-xkb -repeat -cursor arrow -noxdamage -ncache 0 >>"${log}-x11vnc.log" 2>&1 &
|
||||
fi
|
||||
if ! port_open "$view_port"; then
|
||||
local novnc=/usr/share/novnc
|
||||
|
|
@ -205,7 +205,7 @@ start_vnc() {
|
|||
return 1
|
||||
fi
|
||||
websockify --heartbeat=30 --web="$novnc" "0.0.0.0:${view_port}" "127.0.0.1:${vnc_port}" \
|
||||
>"${log}-novnc.log" 2>&1 &
|
||||
>>"${log}-novnc.log" 2>&1 &
|
||||
fi
|
||||
wait_port "$vnc_port"
|
||||
wait_port "$view_port"
|
||||
|
|
@ -217,7 +217,7 @@ start_xterm() {
|
|||
if alive_pidfile "${log}-xterm.pid"; then
|
||||
return 0
|
||||
fi
|
||||
DISPLAY="$display" SHELL=/bin/zsh lazyboy-terminal >"${log}-xterm.log" 2>&1 &
|
||||
DISPLAY="$display" SHELL=/bin/zsh lazyboy-terminal >>"${log}-xterm.log" 2>&1 &
|
||||
echo $! > "${log}-xterm.pid"
|
||||
}
|
||||
|
||||
|
|
@ -235,7 +235,7 @@ start_browser() {
|
|||
fi
|
||||
rm -f "$profile/SingletonLock" "$profile/SingletonCookie" "$profile/SingletonSocket"
|
||||
DISPLAY="$display" HOME="$HOME" LAZYBOY_BROWSER_PROFILE="$profile" \
|
||||
lazyboy-browser https://duckduckgo.com >"${log}-browser.log" 2>&1 &
|
||||
lazyboy-browser https://duckduckgo.com >>"${log}-browser.log" 2>&1 &
|
||||
echo $! > "${log}-browser.pid"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Rotate only LazyBoy service logs; preserve open file descriptors (copytruncate)."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import time
|
||||
|
||||
MAX_BYTES = 5 * 1024 * 1024
|
||||
BACKUPS = 3
|
||||
|
||||
|
||||
def rotate(root: Path, limit: int = MAX_BYTES):
|
||||
for path in root.glob('*.log'):
|
||||
try:
|
||||
fd = os.open(path, os.O_RDWR | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, 'r+b') as source:
|
||||
info = os.fstat(source.fileno())
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size <= limit:
|
||||
continue
|
||||
# Keep a bounded tail even after an unusually large burst.
|
||||
source.seek(-limit, os.SEEK_END)
|
||||
tail = source.read(limit)
|
||||
for n in range(BACKUPS - 1, 0, -1):
|
||||
previous = Path(f'{path}.{n}')
|
||||
if previous.exists() or previous.is_symlink():
|
||||
os.replace(previous, Path(f'{path}.{n + 1}'))
|
||||
backup_fd = os.open(f'{path}.1', os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(backup_fd, 'wb') as backup:
|
||||
backup.write(tail)
|
||||
source.truncate(0)
|
||||
except (OSError, ValueError) as error:
|
||||
print(f'log rotation skipped {path.name}: {error}', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
root = Path('/tmp/lazyboy')
|
||||
root.mkdir(exist_ok=True)
|
||||
with (root / 'log-rotation.lock').open('w') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
while True:
|
||||
rotate(root)
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -19,8 +19,10 @@ rm -f /tmp/lazyboy/ready
|
|||
export PATH="$HOME/.local/bin:/usr/local/bin:$PATH"
|
||||
cd "$HOME"
|
||||
|
||||
python3 /usr/local/bin/lazyboy-rotate-logs &
|
||||
|
||||
if [[ -n "${LAZYBOY_CONTROL_TOKEN:-}" ]]; then
|
||||
lazyboy-controld >/tmp/lazyboy/control.log 2>&1 &
|
||||
lazyboy-controld >>/tmp/lazyboy/control.log 2>&1 &
|
||||
fi
|
||||
|
||||
if command -v dbus-launch >/dev/null 2>&1; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
ALTER TABLE spaces
|
||||
ADD COLUMN IF NOT EXISTS voice_provider TEXT,
|
||||
ADD COLUMN IF NOT EXISTS voice_model_id TEXT,
|
||||
ADD COLUMN IF NOT EXISTS voice_id TEXT,
|
||||
ADD COLUMN IF NOT EXISTS voice_api_key TEXT;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE spaces
|
||||
ADD COLUMN voice_enabled BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
-- Age scans use small batches rather than locking or scanning the full database.
|
||||
CREATE INDEX IF NOT EXISTS events_retention_idx ON events(created_at);
|
||||
CREATE INDEX IF NOT EXISTS runs_retention_idx ON runs((COALESCE(completed_at,updated_at)))
|
||||
WHERE status IN ('completed','failed','cancelled');
|
||||
CREATE INDEX IF NOT EXISTS recording_retention_idx ON taught_skills(updated_at)
|
||||
WHERE status IN ('saved','failed');
|
||||
CREATE INDEX IF NOT EXISTS memory_revisions_age_idx ON memory_revisions(created_at);
|
||||
CREATE INDEX IF NOT EXISTS memory_deleted_age_idx ON memory_items(deleted_at) WHERE deleted_at IS NOT NULL;
|
||||
-- Encourage normal background vacuum to reuse space after regular small deletes.
|
||||
ALTER TABLE events SET (autovacuum_vacuum_scale_factor=0.05, autovacuum_vacuum_threshold=1000);
|
||||
ALTER TABLE runs SET (autovacuum_vacuum_scale_factor=0.05, autovacuum_vacuum_threshold=1000);
|
||||
ALTER TABLE memory_revisions SET (autovacuum_vacuum_scale_factor=0.05, autovacuum_vacuum_threshold=1000);
|
||||
|
|
@ -43,6 +43,22 @@ const mdBox={exports:{},require:(name)=>{
|
|||
}};
|
||||
vm.runInNewContext(mdJs,mdBox);
|
||||
const {sanitizeMarkdownUrl}=mdBox.exports;
|
||||
const audioJs=ts.transpileModule(fs.readFileSync('apps/web/src/call-audio.ts','utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS}}).outputText;
|
||||
const audioBox={exports:{},require:()=>{throw new Error('unexpected import')}};
|
||||
vm.runInNewContext(audioJs,audioBox);
|
||||
const {floatToPcm16,pcm16ToFloat,resample,VOICE_SAMPLE_RATE}=audioBox.exports;
|
||||
test('pcm16 roundtrip keeps amplitude sign and resample identity at 24 kHz',()=>{
|
||||
const source=new Float32Array([0,0.5,-0.5,1,-1]);
|
||||
const bytes=floatToPcm16(source);
|
||||
const back=pcm16ToFloat(bytes.buffer);
|
||||
assert.equal(back.length,source.length);
|
||||
assert.ok(back[1]>0.49&&back[1]<0.51);
|
||||
assert.ok(back[2]< -0.49&&back[2]> -0.51);
|
||||
const same=resample(source,VOICE_SAMPLE_RATE,VOICE_SAMPLE_RATE);
|
||||
assert.equal(same,source);
|
||||
const down=resample(new Float32Array([0,1,0,1]),48000,24000);
|
||||
assert.equal(down.length,2);
|
||||
});
|
||||
test('markdown links only keep http(s), mailto, tel, and in-page hashes',()=>{
|
||||
assert.equal(sanitizeMarkdownUrl('https://example.com/docs'),'https://example.com/docs');
|
||||
assert.equal(sanitizeMarkdownUrl('mailto:hi@example.com'),'mailto:hi@example.com');
|
||||
|
|
@ -51,3 +67,15 @@ test('markdown links only keep http(s), mailto, tel, and in-page hashes',()=>{
|
|||
assert.equal(sanitizeMarkdownUrl('data:text/html,<script>alert(1)</script>'),undefined);
|
||||
assert.equal(sanitizeMarkdownUrl('/relative'),undefined);
|
||||
});
|
||||
|
||||
test('hanging up before microphone permission resolves releases the late stream',async()=>{
|
||||
let grant; let stopped=0;
|
||||
const context={exports:{},navigator:{mediaDevices:{getUserMedia:()=>new Promise(resolve=>grant=resolve)}}};
|
||||
vm.runInNewContext(audioJs,context);
|
||||
const audio=new context.exports.CallAudio({onCapture(){},onError(){}});
|
||||
const starting=audio.start();
|
||||
audio.stop();
|
||||
grant({getTracks:()=>[{stop(){stopped++;}}]});
|
||||
await starting;
|
||||
assert.equal(stopped,1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location('rotation', Path(__file__).resolve().parents[1] / 'image/computer/rotate-logs.py')
|
||||
rotation = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(rotation)
|
||||
|
||||
|
||||
class RotationTests(unittest.TestCase):
|
||||
def test_bounds_backups_and_keeps_live_writer(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
log = root / 'control.log'
|
||||
with log.open('ab', buffering=0) as writer:
|
||||
for n in range(6):
|
||||
writer.write(bytes([65+n]) * 40)
|
||||
rotation.rotate(root, 16)
|
||||
self.assertEqual(log.stat().st_size, 0)
|
||||
self.assertEqual(Path(f'{log}.1').read_bytes(), bytes([65+n])*16)
|
||||
writer.write(b'live')
|
||||
self.assertEqual(log.read_bytes(), b'live')
|
||||
self.assertEqual(len(list(root.glob('*.log.*'))), 3)
|
||||
rotation.rotate(root, 16)
|
||||
self.assertEqual(log.read_bytes(), b'live')
|
||||
|
||||
def test_does_not_follow_symlinks_or_touch_other_files(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
target = root / 'notes.txt'
|
||||
target.write_bytes(b'important' * 20)
|
||||
(root / 'control.log').symlink_to(target)
|
||||
rotation.rotate(root, 16)
|
||||
self.assertEqual(target.read_bytes(), b'important' * 20)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
"""Integration regression: run production cleanup SQL in an isolated disposable DB.
|
||||
Requires the project's Postgres Compose service; never modifies the app database.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DB = 'retention_test_' + uuid.uuid4().hex
|
||||
|
||||
|
||||
def sql(text, database=DB):
|
||||
result = subprocess.run(['docker','compose','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U','lazyboy','-d',database], input=text, text=True, capture_output=True, cwd=ROOT)
|
||||
if result.returncode:
|
||||
raise AssertionError(result.stderr)
|
||||
|
||||
|
||||
def check(condition):
|
||||
sql(f"DO $$ BEGIN IF NOT ({condition}) THEN RAISE EXCEPTION 'retention assertion failed'; END IF; END $$;")
|
||||
|
||||
|
||||
def clean(name, age, batch=1000):
|
||||
query = (ROOT/'crates/api/src/retention'/f'{name}.sql').read_text()
|
||||
sql(query.replace('$1',str(age)).replace('$2',str(batch))+';')
|
||||
|
||||
|
||||
sql(f'CREATE DATABASE {DB};','postgres')
|
||||
try:
|
||||
for migration in sorted((ROOT/'migrations').glob('*.sql')):
|
||||
sql(migration.read_text())
|
||||
sql("""
|
||||
INSERT INTO users(id,name) VALUES ('u','test');
|
||||
INSERT INTO spaces(id,user_id,name) VALUES ('s','u','test');
|
||||
INSERT INTO computers(id,space_id,user_id,scope,scope_key,home_key) VALUES ('c','s','u','bot','c','c');
|
||||
INSERT INTO bots(id,space_id,user_id,name) VALUES ('b','s','u','test');
|
||||
INSERT INTO threads(id,space_id,bot_id,user_id) VALUES ('t','s','b','u');
|
||||
INSERT INTO messages(id,thread_id,seq,role,body,created_at) VALUES ('m','t',1,'user','keep my conversation',now()-interval '1 year');
|
||||
INSERT INTO events(id,thread_id,seq,type,created_at)
|
||||
SELECT 'e'||n,'t',n,'test',now()-interval '31 days' FROM generate_series(1,5) n;
|
||||
INSERT INTO events(id,thread_id,seq,type) VALUES ('recent','t',6,'test');
|
||||
INSERT INTO runs(id,space_id,bot_id,thread_id,user_id,status,checkpoint,updated_at,completed_at)
|
||||
SELECT status,'s','b','t','u',status,'{"harnessHistory":["debug"]}',now()-interval '100 days',
|
||||
CASE WHEN status IN ('completed','failed','cancelled') THEN now()-interval '100 days' ELSE NULL END
|
||||
FROM unnest(ARRAY['completed','failed','cancelled','running','queued','leased','waiting_input','waiting_takeover']) status;
|
||||
INSERT INTO runs(id,space_id,bot_id,thread_id,user_id,status,checkpoint,updated_at,completed_at)
|
||||
VALUES ('recent-completed','s','b','t','u','completed','{"keep":true}',now(),now());
|
||||
INSERT INTO memory_items(id,space_id,user_id,bot_id,content,revision,source_run_id)
|
||||
VALUES ('00000000-0000-0000-0000-000000000001','s','u','b','keep current memory',15,'completed');
|
||||
INSERT INTO memory_revisions(memory_id,revision,space_id,user_id,bot_id,content,importance,action)
|
||||
SELECT '00000000-0000-0000-0000-000000000001',n,'s','u','b','version',.5,'update' FROM generate_series(1,15) n;
|
||||
UPDATE memory_revisions SET created_at=now()-interval '100 days' WHERE revision IN (6,15);
|
||||
INSERT INTO memory_items(id,space_id,user_id,bot_id,content,deleted_at)
|
||||
VALUES ('00000000-0000-0000-0000-000000000002','s','u','b','deleted old',now()-interval '100 days'),
|
||||
('00000000-0000-0000-0000-000000000003','s','u','b','deleted recent',now());
|
||||
INSERT INTO taught_skills(id,space_id,user_id,bot_id,goal,status,playbook,recording,updated_at)
|
||||
SELECT status,'s','u','b','goal',status,'{"keep":true}','{"frames":["raw"]}',now()-interval '100 days'
|
||||
FROM unnest(ARRAY['saved','failed','draft','drafting','recording']) status;
|
||||
INSERT INTO computer_profile_locks(computer_id,profile_key,bot_id,run_id,expires_at)
|
||||
VALUES ('c','old','b','completed',now()-interval '10 days'),('c','active','b','running',now()-interval '10 days');
|
||||
INSERT INTO computer_execution_leases(id,computer_id,bot_id,run_id,expires_at)
|
||||
VALUES ('old','c','b','completed',now()-interval '10 days'),('active','c','b2','running',now()-interval '10 days');
|
||||
""")
|
||||
clean('events',30,2)
|
||||
check('(SELECT count(*) FROM events)=4')
|
||||
clean('events',30)
|
||||
check("(SELECT count(*) FROM events)=1 AND EXISTS(SELECT 1 FROM events WHERE id='recent')")
|
||||
clean('checkpoints',7)
|
||||
check("(SELECT count(*) FROM runs WHERE checkpoint='{}')=3")
|
||||
clean('runs',90)
|
||||
check('(SELECT count(*) FROM runs)=6')
|
||||
check("EXISTS(SELECT 1 FROM memory_items WHERE content='keep current memory' AND source_run_id IS NULL)")
|
||||
clean('recordings',30)
|
||||
check("(SELECT count(*) FROM taught_skills WHERE recording='{}')=3 AND (SELECT count(*) FROM taught_skills WHERE playbook->>'keep'='true')=5")
|
||||
clean('revisions',90)
|
||||
check('(SELECT count(*) FROM memory_revisions)=9 AND EXISTS(SELECT 1 FROM memory_revisions WHERE revision=15)')
|
||||
clean('deleted_memories',90)
|
||||
check('(SELECT count(*) FROM memory_items)=2')
|
||||
clean('leases',7)
|
||||
clean('profile_locks',7)
|
||||
check("(SELECT count(*) FROM computer_execution_leases)=1 AND EXISTS(SELECT 1 FROM computer_execution_leases WHERE id='active')")
|
||||
check("(SELECT count(*) FROM computer_profile_locks)=1 AND EXISTS(SELECT 1 FROM computer_profile_locks WHERE profile_key='active')")
|
||||
check("EXISTS(SELECT 1 FROM messages WHERE body='keep my conversation')")
|
||||
for name,age in [('events',30),('checkpoints',7),('runs',90),('recordings',30),('revisions',90),('deleted_memories',90),('leases',7),('profile_locks',7)]:
|
||||
clean(name,age)
|
||||
check('(SELECT count(*) FROM runs)=6 AND (SELECT count(*) FROM memory_revisions)=9')
|
||||
print('PASS: all 8 retention rules; bounded batches; live work and user content preserved; repeat-safe')
|
||||
finally:
|
||||
sql(f'DROP DATABASE {DB};','postgres')
|
||||
Loading…
Reference in New Issue