diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index e53754b..484ddaa 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -26,7 +26,7 @@ import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, preset import { CallOverlay, PhoneIcon } from "./call"; import { VoiceSettingsDialog } from "./voice-settings"; -const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; +const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",sharedInput:true,controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; const SESSION_STORE="lazyboy.sessionByBot"; const PANE_STORE="lazyboy.rightPane"; const WORKSPACE_STORE="lazyboy.workspace"; @@ -184,6 +184,7 @@ export function App(){ const lastMessageId=messages[messages.length-1]?.id||""; // The bot is parked in waiting_takeover: nothing moves (including queued // messages) until the human releases the screen, so say so loudly. + const desktopInteractive=computer.screenAvailable&&!viewOnlyFor(computer.controlHolder,computer.sharedInput); const pausedForUser=computer.takeoverRequested&&workingMembers.length===0&&(!computer.waitingSessionId||computer.waitingSessionId===activeSessionId||Boolean(activeRoom)); // Only the newest assistant reply can still be "continued": an older pause // belongs to a run that has already moved on. @@ -260,12 +261,12 @@ export function App(){ document.addEventListener("visibilitychange",resume); return()=>{window.clearTimeout(timer);window.clearInterval(heartbeat);document.removeEventListener("visibilitychange",resume);feed.close();settle.cancel();refreshSeqRef.current+=1}; },[activeId,activeRoomId,activeSessionId,refresh]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&desktopInteractive)void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&desktopInteractive)setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]); - useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]); + useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running"&&!computer.sharedInput)setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state,computer.sharedInput]); useEffect(()=>{if(!handingOff)return;handoffAtRef.current=Date.now();const timer=setTimeout(()=>{expectedHolderRef.current=null;setHandingOff(false)},HANDOFF_MS);return()=>clearTimeout(timer)},[handingOff]); - useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:viewOnlyFor(computer.controlHolder)},location.origin)},[computer.controlHolder,screenUrl,desktopReady]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void setControl("user")};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); + useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:!desktopInteractive},location.origin)},[desktopInteractive,screenUrl,desktopReady]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;if(computer.sharedInput){pushViewOnly(!desktopInteractive);return}void setControl("user")};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId,computer.sharedInput,desktopInteractive]); useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{setWorkspaceName(name=>isDefaultWorkspaceName(name)?t("localWorkspace"):name)},[locale]); @@ -279,9 +280,8 @@ export function App(){ const id=paneBot?.id||active?.id;if(!id)return; setRightPart("computer");setRightCollapsed(false);setComputerOpen(true); await action(async()=>{try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */}}); - // The mouse is the point of the login screen, so it comes over on the - // optimistic handoff rather than a second round trip behind a spinner. - await setControl("user",id); + // Opening the shared screen never pauses a running task. + if(!computer.sharedInput)await setControl("user",id); } async function reloadSchedules(){if(!paneBotId)return;setSchedules(await api(`/api/bots/${paneBotId}/schedules`))} async function saveScheduleDraft(){ @@ -330,7 +330,7 @@ export function App(){ function pasteText(text:string){queueDesktopInput({kind:"clipboard",text})} function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string}){ const botId=paneBotId; - if(!botId||computer.controlHolder!=="user")return; + if(!botId||!desktopInteractive)return; pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{ if(currentPaneRef.current!==botId)return; await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify(input)}); @@ -390,8 +390,8 @@ export function App(){ controlBusyRef.current=true;setControlBusy(true); expectedHolderRef.current=holder; setComputer(current=>({...current,controlHolder:holder,takeoverRequested:holder==="user"?false:current.takeoverRequested})); - setHandingOff(true); - pushViewOnly(viewOnlyFor(holder)); + if(!computer.sharedInput)setHandingOff(true); + pushViewOnly(viewOnlyFor(holder,computer.sharedInput)); try{await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`,{method:"POST",body:"{}"})} catch(error){setError(localizeError(error instanceof Error?error.message:t("operationFailed")))} finally{ @@ -468,8 +468,8 @@ export function App(){
{activeRoom?<>member.id)}/>{activeRoom.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:active?<>{active.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:<>{t("chooseBot")}{topTools}}
- {callOpen&&voiceSettings?.enabled&&active&&activeSessionId&&!activeRoomId?setCallOpen(false)} onTakeOver={()=>{setRightPart("computer");setRightCollapsed(false);if(active)void setControl("user",active.id)}}/>:null} -
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:listJoin(activeRoom.members.map(member=>member.name))}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const chips=chipBlocks(message.blocks);const hideBody=isAutoAttachCaption(message.body,files)||isAutoScheduleCaption(message.body,chips);return
{spoken&&}{spoken&&{speakerName}}{files.length>0&&
{files.map(file=>)}
}{chips.map((chip,index)=>chip.kind==="error"?
{errorTitle(chip.code)}
{errorActions(chip.code).map(next=>next==="retry"?:next==="screen"?:)}
:chip.kind==="resume"?
{resumeTitle(chip.reason)}
{(chip.limit||0)>0?
{t("resumeProgress",{turns:chip.turns||0,limit:chip.limit||0})}
:null}{resumedChips[message.id]||(message.seq??0)
}
:chip.kind==="login"?
{t("loginNeedsYou")}
{chip.site||message.body}
{chip.why?
{t("loginWhy",{why:chip.why})}
:null}
:chip.kind==="schedule"?
{t("scheduleChip")}
{chip.name}{scheduleWhen(chip.cron,chip.human)}
:
{t("scheduleRunChip")}
{chip.name}{scheduleWhen(chip.cron,chip.human)}
)}{!hideBody&&(message.role==="assistant"?
{message.body}
{message.body.trim()?:null}
:{message.body})}{!hideBody&&message.body.trim()&&}
})}{pausedForUser&&paneBot&&
{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}}{computer.controlHolder==="user"?:}
}{teaching&&active&&
{t("teachingLive",{goal:teaching.goal})}{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}}
}{drafting&&
{t("distilling",{goal:drafting.goal})}
}{skillDraft&&active&&void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>} + {callOpen&&voiceSettings?.enabled&&active&&activeSessionId&&!activeRoomId?setCallOpen(false)} onTakeOver={()=>{setRightPart("computer");setRightCollapsed(false);setComputerOpen(true);if(active&&!computer.sharedInput)void setControl("user",active.id)}}/>:null} +
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:listJoin(activeRoom.members.map(member=>member.name))}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const chips=chipBlocks(message.blocks);const hideBody=isAutoAttachCaption(message.body,files)||isAutoScheduleCaption(message.body,chips);return
{spoken&&}{spoken&&{speakerName}}{files.length>0&&
{files.map(file=>)}
}{chips.map((chip,index)=>chip.kind==="error"?
{errorTitle(chip.code)}
{errorActions(chip.code).map(next=>next==="retry"?:next==="screen"?:)}
:chip.kind==="resume"?
{resumeTitle(chip.reason)}
{(chip.limit||0)>0?
{t("resumeProgress",{turns:chip.turns||0,limit:chip.limit||0})}
:null}{resumedChips[message.id]||(message.seq??0)
}
:chip.kind==="login"?
{t("loginNeedsYou")}
{chip.site||message.body}
{chip.why?
{t("loginWhy",{why:chip.why})}
:null}
:chip.kind==="schedule"?
{t("scheduleChip")}
{chip.name}{scheduleWhen(chip.cron,chip.human)}
:
{t("scheduleRunChip")}
{chip.name}{scheduleWhen(chip.cron,chip.human)}
)}{!hideBody&&(message.role==="assistant"?
{message.body}
{message.body.trim()?:null}
:{message.body})}{!hideBody&&message.body.trim()&&}
})}{pausedForUser&&paneBot&&
{computer.sharedInput?t("sharedNeedsUser",{name:paneBot.name}):computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}}{computer.sharedInput||computer.controlHolder==="user"?:}
}{teaching&&active&&
{t("teachingLive",{goal:teaching.goal})}{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}}
}{drafting&&
{t("distilling",{goal:drafting.goal})}
}{skillDraft&&active&&void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>}
{error&&
{error}
} {otherSessionBusy&&
{t("anotherConversationQueued")}
} @@ -499,7 +499,7 @@ export function App(){ } - {computerOpen&&paneBot&&
{modeLabel(computer.mode)}{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}
{teaching?
:void setControl("user")} onRelease={()=>void setControl("none")}/>}
{frame}{hud}
{error&&
{error}
}
} + {computerOpen&&paneBot&&
{modeLabel(computer.mode)}{teaching?t("teachingBadge"):computer.sharedInput?t("sharedDesktop"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}
{teaching?
:void setControl("user")} onRelease={()=>void setControl("none")}/>}
{frame}{hud}
{error&&
{error}
}
} {teachOpen&&active&&setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>} {editingSkill&&setEditingSkillId(null)} save={(name,playbook)=>void updateSkill(editingSkill,name,playbook)} test={(name,playbook)=>void testSkill(editingSkill,name,playbook)} remove={()=>void deleteSkill(editingSkill)} exportFile={(name,playbook)=>downloadSkill(name,editingSkill.goal,playbook)}/>} {clipboardOpen&&setClipboardOpen(false)} paste={text=>{pasteText(text);setClipboardOpen(false)}}/>} @@ -757,6 +757,7 @@ function ControlButtons({computer,busy,pending,action,active,sessionId,onBoot,on const run=(operation:"boot"|"restart"|"stop",handler?:()=>Promise)=>{setMenuOpen(false);void action(handler||(()=>api(`/api/computer/${active.id}/${operation}`,{method:"POST",body:"{}"})))}; return
{!running? + :computer.sharedInput?(computer.takeoverRequested||computer.controlHolder==="user"?:null) :computer.controlHolder==="user"? :} {running&&working&&} @@ -770,7 +771,7 @@ function ControlButtons({computer,busy,pending,action,active,sessionId,onBoot,on
}
; } -function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise)=>Promise;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise;onRestart?:()=>Promise;onStop?:()=>Promise;pending?:boolean;onTakeOver:()=>void;onRelease:()=>void}){const interactive=props.computer.controlHolder==="user";return
} +function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise)=>Promise;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise;onRestart?:()=>Promise;onStop?:()=>Promise;pending?:boolean;onTakeOver:()=>void;onRelease:()=>void}){const interactive=props.computer.screenAvailable&&!viewOnlyFor(props.computer.controlHolder,props.computer.sharedInput);return
} function TeachDialog({bot,busy,close,start}:{bot:Bot;busy:boolean;close:()=>void;start:(goal:string)=>void}){ const [goal,setGoal]=useState(""); return
e.stopPropagation()} onSubmit={e=>{e.preventDefault();if(goal.trim())start(goal.trim())}}> diff --git a/apps/web/src/call.tsx b/apps/web/src/call.tsx index e5007b5..930814d 100644 --- a/apps/web/src/call.tsx +++ b/apps/web/src/call.tsx @@ -175,7 +175,7 @@ export function CallOverlay({
{showTakeover ? ( - + ) : null}

{t("callShortcuts")}

diff --git a/apps/web/src/handoff.ts b/apps/web/src/handoff.ts index 06bcd13..1aed52b 100644 --- a/apps/web/src/handoff.ts +++ b/apps/web/src/handoff.ts @@ -50,8 +50,8 @@ export function nextVeil(label: string | null, current: Veil): Veil { return current.leaving ? current : { label: current.label, leaving: true }; } -/** The lease is advisory inside the container, so the browser gate is what - * actually moves the mouse: whoever holds control may send input right now. */ -export function viewOnlyFor(holder: "none" | "bot" | "user"): boolean { - return holder !== "user"; +/** Shared desktop input stays enabled while the agent works or waits for help. + * Older servers still use the exclusive holder gate. */ +export function viewOnlyFor(holder: "none" | "bot" | "user", sharedInput = false): boolean { + return !sharedInput && holder !== "user"; } diff --git a/apps/web/src/locales/en.ts b/apps/web/src/locales/en.ts index b1124c6..dd816a0 100644 --- a/apps/web/src/locales/en.ts +++ b/apps/web/src/locales/en.ts @@ -2,6 +2,9 @@ import type { zhTW } from "./zh-TW"; /** English UI copy. Keys must stay in lockstep with zh-TW. */ export const en: { [K in keyof typeof zhTW]: string } = { + sharedDesktop: "Shared control", + sharedNeedsUser: "{name} is waiting for you to finish the steps on screen. Then press “Done, continue”.", + doneContinue: "Done, continue", search: "Search", sharedComputer: "Shared computer", privateComputer: "Private computer", diff --git a/apps/web/src/locales/zh-TW.ts b/apps/web/src/locales/zh-TW.ts index aeab04d..1b209d1 100644 --- a/apps/web/src/locales/zh-TW.ts +++ b/apps/web/src/locales/zh-TW.ts @@ -1,5 +1,8 @@ /** Traditional Chinese UI copy. Keep keys stable when adding another locale. */ export const zhTW = { + sharedDesktop: "共同操作", + sharedNeedsUser: "{name} 已暫停等你完成畫面上的步驟。完成後按「完成，繼續」。", + doneContinue: "完成，繼續", search: "搜尋", sharedComputer: "共用電腦", privateComputer: "私人電腦", stopped: "已關閉", booting: "啟動中", running: "執行中", suspended: "休眠中", error: "發生錯誤", openComputer: "開啟電腦", stopTask: "停止任務", takeControl: "取得控制", takeOverNow: "接手操作", releaseControl: "釋放控制", done: "完成", skip: "略過", diff --git a/apps/web/src/types.ts b/apps/web/src/types.ts index f03918d..85499e9 100644 --- a/apps/web/src/types.ts +++ b/apps/web/src/types.ts @@ -9,7 +9,7 @@ export interface Message { id:string; sessionId?:string; seq?:number; role:strin export interface MessageFile { kind:"image"|"file"; name:string; mimeType?:string; size?:number } export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape } export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number } -export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean } +export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; sharedInput?:boolean; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean } /** One line of the live trail a run writes while it works. */ export type RunActivityKind = "run"|"model"|"tool"|"retry"|"notice"; export interface RunActivityEntry { id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null } diff --git a/apps/web/vnc.html b/apps/web/vnc.html index 55ec508..2660009 100644 --- a/apps/web/vnc.html +++ b/apps/web/vnc.html @@ -382,7 +382,7 @@ setStatus("Connecting to desktop…"); window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin); rfb = new RFB(document.getElementById("screen"), url); - rfb.viewOnly = flag("view_only", true); + rfb.viewOnly = wantedViewOnly ?? flag("view_only", true); rfb.scaleViewport = true; rfb.qualityLevel = 6; rfb.compressionLevel = 2; @@ -402,7 +402,11 @@ }); rfb.addEventListener("disconnect", (event) => { releaseDrag(); gesture = null; - applyViewOnly(true); + // Disable the disconnected instance without overwriting host intent. + rfb.viewOnly = true; + modifier = null; + if (shortcuts) shortcuts.hidden = true; + updatePointerControls(); keyboard.blur(); resetKeyboard(); window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin); const clean = event && event.detail && event.detail.clean; diff --git a/crates/api/src/computer.rs b/crates/api/src/computer.rs index d8cf3bc..90225a7 100644 --- a/crates/api/src/computer.rs +++ b/crates/api/src/computer.rs @@ -9,7 +9,7 @@ use lazyboy_contracts::{ use lazyboy_control::{ AdapterContext, CommandRequest, EnsureScreenRequest, ProvisionRequest, admit_gui, admit_new_screen, browser_profile_path, execution_blocks_user_takeover, profile_lock_key, - screen_layout, team_bot_workspace_directory, user_holds_control, + screen_layout, team_bot_workspace_directory, }; use uuid::Uuid; @@ -46,6 +46,7 @@ pub fn status_from( mode: parse_mode(&computer.scope), kind: parse_kind(&computer.kind), state: parse_state(&computer.state), + shared_input: true, control_holder, control_bot_id, takeover_requested: false, @@ -910,7 +911,7 @@ pub async fn takeover( &thread_id, &run_id, bot_id, - "你已接手操作。完成後釋放控制權，我會從目前畫面繼續。", + "我已暫停等你操作。完成後按「完成，繼續」，我會從目前畫面接著做。", ) .await; } @@ -998,27 +999,12 @@ pub async fn heartbeat(state: &AppState, actor: &Actor, bot_id: &str) -> Result< Ok(()) } -pub fn user_has_screen_control( - computer: &ComputerRow, - screen: Option<&ScreenRow>, - bot_id: &str, -) -> bool { - if let Some(screen) = screen { - return user_holds_control( - parse_holder(&screen.control_holder), - Some(screen.bot_id.as_str()), - bot_id, - screen.control_lease_expires_at, - Utc::now(), - ); - } - user_holds_control( - parse_holder(&computer.control_holder), - computer.control_bot_id.as_deref(), - bot_id, - computer.control_lease_expires_at, - Utc::now(), - ) +/// Viewing/input does not acquire the human pause lease. Agent execution and +/// explicit requests for human assistance retain their own pause/resume flow. +pub fn user_can_interact(computer: &ComputerRow, screen: Option<&ScreenRow>, bot_id: &str) -> bool { + computer.state == "running" + && computer.provider_ref.is_some() + && screen.is_some_and(|screen| screen.bot_id == bot_id && screen.computer_id == computer.id) } pub async fn idle_loop(state: AppState) { @@ -1227,3 +1213,78 @@ struct ActiveRunRow { thread_id: String, step: Option, } + +#[cfg(test)] +mod shared_input_tests { + use super::*; + + fn desktop() -> (ComputerRow, ScreenRow) { + let computer = ComputerRow { + id: "computer".into(), + space_id: "space".into(), + user_id: "user".into(), + scope: "team".into(), + scope_key: "team:space".into(), + home_key: "home".into(), + home_revision: "1".into(), + kind: "docker".into(), + provider_ref: Some("container".into()), + state: "running".into(), + control_holder: "bot".into(), + control_lease_id: None, + control_lease_expires_at: None, + control_bot_id: Some("bot".into()), + control_run_id: None, + execution_run_id: Some("run".into()), + execution_bot_id: Some("bot".into()), + execution_lease_expires_at: None, + execution_fence: 1, + browser_profile_mode: "per-bot".into(), + }; + let screen = ScreenRow { + id: "screen".into(), + computer_id: computer.id.clone(), + bot_id: "bot".into(), + slot: 1, + display: ":1".into(), + view_port: 6080, + profile_mode: "per-bot".into(), + profile_path: "/tmp/profile".into(), + control_holder: "bot".into(), + control_lease_id: None, + control_lease_expires_at: None, + execution_run_id: Some("run".into()), + execution_lease_expires_at: None, + execution_fence: 1, + }; + (computer, screen) + } + + #[test] + fn human_input_does_not_require_or_change_the_agent_lease() { + let (computer, mut screen) = desktop(); + for holder in ["bot", "none", "user"] { + screen.control_holder = holder.into(); + assert!(user_can_interact(&computer, Some(&screen), "bot")); + assert_eq!(screen.execution_run_id.as_deref(), Some("run")); + assert!(status_from("bot", &computer, Some(&screen), None).shared_input); + } + } + + #[test] + fn shared_input_requires_the_bots_own_live_screen() { + let (mut computer, mut screen) = desktop(); + assert!(!user_can_interact(&computer, None, "bot")); + assert!(!user_can_interact(&computer, Some(&screen), "other-bot")); + screen.computer_id = "other-computer".into(); + assert!(!user_can_interact(&computer, Some(&screen), "bot")); + screen.computer_id = computer.id.clone(); + for state in ["stopped", "booting", "suspended", "error"] { + computer.state = state.into(); + assert!(!user_can_interact(&computer, Some(&screen), "bot")); + } + computer.state = "running".into(); + computer.provider_ref = None; + assert!(!user_can_interact(&computer, Some(&screen), "bot")); + } +} diff --git a/crates/api/src/routes.rs b/crates/api/src/routes.rs index d6027ce..5ee3654 100644 --- a/crates/api/src/routes.rs +++ b/crates/api/src/routes.rs @@ -648,7 +648,7 @@ async fn screen_url( state.db.get_screen(&computer.id, &id).await.ok().flatten() } }; - let interactive = computer::user_has_screen_control(&computer, screen.as_ref(), &id); + let interactive = computer::user_can_interact(&computer, screen.as_ref(), &id); let _ = state .sandbox .connect_screen( @@ -728,7 +728,7 @@ async fn input( .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? .ok_or(StatusCode::NOT_FOUND)?; let screen = state.db.get_screen(&computer.id, &id).await.ok().flatten(); - if !computer::user_has_screen_control(&computer, screen.as_ref(), &id) { + if !computer::user_can_interact(&computer, screen.as_ref(), &id) { return Err(StatusCode::CONFLICT); } let computer_ref = computer::computer_ref(&computer).ok_or(StatusCode::BAD_REQUEST)?; diff --git a/crates/api/src/runs.rs b/crates/api/src/runs.rs index 29ec6b3..32bea5d 100644 --- a/crates/api/src/runs.rs +++ b/crates/api/src/runs.rs @@ -47,7 +47,7 @@ Use tools only when the user wants something done on the computer: open a site, The shell is one real terminal that stays open between calls: same directory, same exports, same background jobs. cd where the work is and stay there. A long-running job (server, build, download) comes back as status=running and keeps going — read it with log_lines, stop it with keys \"C-c\", and never type a second command into a terminal that is still busy. -When you ARE using the desktop: the human watches the same live screen. Only the latest screenshot you received is current; they may have interacted since. Call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead. +When you ARE using the desktop: the human can interact with the same live screen while you work; this does not pause your task. Prefer browser/native element actions over moving the shared pointer. If the screen changes unexpectedly, observe again and continue from the current state; do not undo human changes or replay an uncertain click. Request human assistance only when the task needs it. Only the latest screenshot you received is current; they may have interacted since. Call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead. When you use the browser tool: - snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate. diff --git a/crates/api/src/screen_proxy.rs b/crates/api/src/screen_proxy.rs index fd09359..9292c6b 100644 --- a/crates/api/src/screen_proxy.rs +++ b/crates/api/src/screen_proxy.rs @@ -128,7 +128,7 @@ async fn upstream_target( .sandbox .connect_screen( &computer_ref, - computer::user_has_screen_control(&computer, screen.as_ref(), bot_id), + computer::user_can_interact(&computer, screen.as_ref(), bot_id), &computer::adapter_context_for(&actor, bot_id, "view", screen.as_ref(), None), ) .await diff --git a/crates/api/src/tools.rs b/crates/api/src/tools.rs index 840c2cc..acefd45 100644 --- a/crates/api/src/tools.rs +++ b/crates/api/src/tools.rs @@ -186,7 +186,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec { }, ToolDefinition { name: "request_takeover".into(), - description: "Ask the user to take over the live screen for passwords, 2FA, CAPTCHA, or a login wall when no saved account fits. Never ask them to paste secrets in chat. Prefer use_saved_login when list_accounts has a matching site.".into(), + description: "Pause the task and ask the user to complete passwords, 2FA, CAPTCHA, or a login wall when no saved account fits. The shared screen already accepts their input; they press Done, continue when finished. Never ask them to paste secrets in chat. Prefer use_saved_login when list_accounts has a matching site.".into(), parameters: json!({ "type":"object", "properties":{ diff --git a/crates/contracts/src/computer.rs b/crates/contracts/src/computer.rs index 81bb9bf..326e2cb 100644 --- a/crates/contracts/src/computer.rs +++ b/crates/contracts/src/computer.rs @@ -172,6 +172,9 @@ pub struct ComputerStatus { pub mode: ComputerMode, pub kind: SandboxKind, pub state: ComputerState, + /// Human input is independent of the agent execution/pause lease. + #[serde(default)] + pub shared_input: bool, pub control_holder: ControlHolder, pub control_bot_id: Option, pub takeover_requested: bool, diff --git a/crates/control/src/actions.rs b/crates/control/src/actions.rs index 59b8299..fbfb596 100644 --- a/crates/control/src/actions.rs +++ b/crates/control/src/actions.rs @@ -406,7 +406,7 @@ fn ref_kind(action: &serde_json::Map) -> String { fn coordinate(value: Option<&Value>, name: &'static str) -> Result { let number = value.and_then(Value::as_f64).unwrap_or(f64::NAN).round(); - if !number.is_finite() || number < 0.0 || number > 100_000.0 { + if !number.is_finite() || !(0.0..=100_000.0).contains(&number) { return Err(ActionError::BadCoordinate(name)); } Ok(number as u32) diff --git a/crates/control/src/cua/client.rs b/crates/control/src/cua/client.rs index d5519c0..ec75dad 100644 --- a/crates/control/src/cua/client.rs +++ b/crates/control/src/cua/client.rs @@ -174,7 +174,8 @@ fn decode_stdout( let empty = Value::Null; let decoded = (!trimmed.is_empty()) .then(|| parse_jsonish(trimmed)) - .flatten(); + .flatten() + .filter(Value::is_object); let error = if !success || trimmed.starts_with('\u{274c}') || decoded.is_none() { Some(classify(&combined)) } else { @@ -383,6 +384,11 @@ mod tests { assert!(error.is_none()); assert_eq!(value["width"], 1280); + for malformed in ["null", "true", "42", "[]", r#""ok""#] { + let (_, error) = decode_stdout(malformed, "", true, classify); + assert!(error.is_some(), "non-object response accepted: {malformed}"); + } + // Prose with a zero exit code used to be reported as success. let (value, error) = decode_stdout("no window matched", "", true, classify); assert!(matches!(error, Some(ControlError::DriverUnhealthy))); diff --git a/crates/control/src/cua/mod.rs b/crates/control/src/cua/mod.rs index b039740..29ce55d 100644 --- a/crates/control/src/cua/mod.rs +++ b/crates/control/src/cua/mod.rs @@ -172,9 +172,10 @@ impl ComputerController for CuaController { } let key = normalize_display(&ctx.display).to_string(); let mut windows = self.windows(&ctx.display).await.unwrap_or_default(); - let mut cache = self.browser.lock().await; + // The screen lock serializes this display. Never hold the shared cache + // lock across driver calls or waits on behalf of other displays. + let mut bind = self.browser.lock().await.remove(&key); for attempt in 0..2 { - let mut bind = cache.remove(&key); let had_bind = bind.is_some(); match browser::run( &self.client, @@ -204,20 +205,13 @@ impl ComputerController for CuaController { | ControlError::TargetNotFound ) => { + bind = None; windows = self.windows(&ctx.display).await.unwrap_or_default(); } other => { if let Some(mut current) = bind { - current.page = match &other { - Ok(page) - if page.ok - && !matches!(request.action.as_str(), "probe" | "ensure") => - { - Some(page.clone()) - } - _ => None, - }; - cache.insert(key, current); + update_browser_page(&mut current, &request.action, &other); + self.browser.lock().await.insert(key, current); } return map_browser_unavailable(other); } @@ -295,6 +289,22 @@ impl ComputerController for CuaController { } } +fn update_browser_page( + bind: &mut browser::BrowserBind, + action: &str, + result: &Result, +) { + // These checks neither observe nor mutate the page. Keep the refs returned + // by the previous snapshot usable for the next click/type. + if matches!(action, "probe" | "ensure") && result.is_ok() { + return; + } + bind.page = match result { + Ok(page) if page.ok => Some(page.clone()), + _ => None, + }; +} + impl CuaController { async fn run_actions( &self, @@ -737,6 +747,44 @@ fn observe_png_path(display: &str) -> PathBuf { mod tests { use super::*; + #[test] + fn browser_checks_preserve_refs_but_failed_actions_invalidate_them() { + let mut bind = browser::BrowserBind { + pid: 1, + window_id: 2, + target_id: "target".into(), + tab_id: "tab".into(), + page: Some(CdpPage { + ok: true, + title: "original snapshot".into(), + ..CdpPage::default() + }), + }; + for action in ["probe", "ensure"] { + update_browser_page( + &mut bind, + action, + &Ok(CdpPage { + ok: true, + ..CdpPage::default() + }), + ); + assert_eq!(bind.page.as_ref().unwrap().title, "original snapshot"); + } + update_browser_page(&mut bind, "click", &Err(ControlError::StaleReference)); + assert!(bind.page.is_none()); + update_browser_page( + &mut bind, + "snapshot", + &Ok(CdpPage { + ok: true, + title: "fresh".into(), + ..CdpPage::default() + }), + ); + assert_eq!(bind.page.as_ref().unwrap().title, "fresh"); + } + #[test] fn driver_release_reads_the_pinned_minor_series() { assert_eq!(driver_release("cua-driver 0.23.2"), Some((0, 23))); diff --git a/crates/control/src/cua/translate.rs b/crates/control/src/cua/translate.rs index 7b330c4..125eb90 100644 --- a/crates/control/src/cua/translate.rs +++ b/crates/control/src/cua/translate.rs @@ -99,26 +99,28 @@ fn translate_pointer( } fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction { - let key = map_key(key); - match modifiers { - Some(items) if !items.is_empty() => { - let mut keys: Vec = items.iter().map(|item| map_key(item)).collect(); - keys.push(key); - TranslatedAction::Cua { - tool: "hotkey", - payload: json!({ - "keys": keys, - "scope": "desktop", - }), - } + // The public action DSL and mobile keyboard send chords as "ctrl+a". + // Cua requires separate keys passed to hotkey, not a literal press_key. + let mut keys: Vec = modifiers + .unwrap_or_default() + .iter() + .map(|item| map_key(item)) + .collect(); + if key.contains('+') && key.split('+').all(|part| !part.is_empty()) { + keys.extend(key.split('+').map(map_key)); + } else { + keys.push(map_key(key)); + } + if keys.len() > 1 { + TranslatedAction::Cua { + tool: "hotkey", + payload: json!({ "keys": keys, "scope": "desktop" }), } - _ => TranslatedAction::Cua { + } else { + TranslatedAction::Cua { tool: "press_key", - payload: json!({ - "key": key, - "scope": "desktop", - }), - }, + payload: json!({ "key": keys[0], "scope": "desktop" }), + } } } @@ -138,6 +140,26 @@ mod tests { use super::*; use lazyboy_contracts::RefVerb; + #[test] + fn inline_shortcuts_use_hotkey_and_literal_plus_stays_a_key() { + for (key, expected) in [ + ("ctrl+a", json!(["ctrl", "a"])), + ("alt+Left", json!(["alt", "left"])), + ("Control+Shift+Tab", json!(["ctrl", "shift", "tab"])), + ] { + let TranslatedAction::Cua { tool, payload } = translate_key(key, None) else { + panic!("expected Cua") + }; + assert_eq!(tool, "hotkey"); + assert_eq!(payload["keys"], expected); + } + let TranslatedAction::Cua { tool, payload } = translate_key("+", None) else { + panic!("expected Cua") + }; + assert_eq!(tool, "press_key"); + assert_eq!(payload["key"], "+"); + } + #[test] fn pixel_click_is_desktop_cua_click() { let action = ComputerAction::Pointer { diff --git a/tests/frontend.test.mjs b/tests/frontend.test.mjs index 5cccc62..7dec9c8 100644 --- a/tests/frontend.test.mjs +++ b/tests/frontend.test.mjs @@ -518,11 +518,15 @@ test('only a human moves the mouse, and the veil never holds it back',()=>{ for(const holder of ['none','bot'])assert.equal(viewOnlyFor(holder),true,holder); }); +test('shared input stays interactive across agent ownership and pause states',()=>{ + for(const holder of ['none','bot','user'])assert.equal(viewOnlyFor(holder,true),false,holder); +}); + test('taking the screen flips the mouse on the click, not on the reply',()=>{ const app=fs.readFileSync('apps/web/src/App.tsx','utf8'); const body=app.slice(app.indexOf('async function setControl(')); const flip=body.indexOf('controlHolder:holder'); - const gate=body.indexOf('pushViewOnly(viewOnlyFor(holder))'); + const gate=body.indexOf('pushViewOnly(viewOnlyFor(holder,computer.sharedInput))'); const post=body.indexOf('await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`'); const readBack=body.indexOf('finally{'); assert.ok(flip>0&&gate>0&&post>0&&readBack>0,'optimistic flip, gate, request, and read back all present'); @@ -530,15 +534,27 @@ test('taking the screen flips the mouse on the click, not on the reply',()=>{ assert.ok(readBack>post,'the truth is read back after the request'); assert.match(body.slice(readBack),/await refresh\(\)/); assert.match(body,/if\(!botId\|\|controlBusyRef\.current\)return/,'a double click cannot fight itself'); - assert.match(app,/const listener=\(event:MessageEvent\)=>\{[\s\S]*?lazyboy-request-control[\s\S]*?void setControl\("user"\)/); + assert.match(app,/if\(computer\.sharedInput\)\{pushViewOnly\(!desktopInteractive\);return\}void setControl\("user"\)/); assert.match(app,/if\(expectedHolderRef\.current&&status\.controlHolder===expectedHolderRef\.current\)/,'agreement, not a timer, ends the handoff'); assert.match(app,/onClick=\{onTakeOver\}/); assert.match(app,/onClick=\{onRelease\}/); assert.doesNotMatch(app,/action\(\(\)=>api\(`\/api\/computer\/[^`]*takeover/,'takeover no longer rides the global busy path'); assert.equal((app.match(/\/api\/computer\/\$\{[^}]*\}\/(takeover|release)/g)||[]).length,0,'every handoff goes through setControl'); assert.equal((app.match(/holder==="user"\?"takeover":"release"/g)||[]).length,1,'one request path, one owner of it'); - assert.match(app,/void setControl\("user",active\.id\)/,'the call overlay hands over the same way'); - assert.match(app,/await setControl\("user",id\)/,'the login screen boots, then takes the mouse without a second spinner'); +}); + +test('shared CUA desktops hide takeover and keep the mouse live',()=>{ + const app=fs.readFileSync('apps/web/src/App.tsx','utf8'); + assert.match(app,/sharedInput:true/,'the first paint already assumes shared input'); + assert.match(app,/if\(!computer\.sharedInput\)await setControl\("user",id\)/,'opening the login screen must not pause a shared run'); + assert.match(app,/if\(active&&!computer\.sharedInput\)void setControl\("user",active\.id\)/,'a call only takes exclusive control on older exclusive desktops'); + assert.match(app,/computer\.sharedInput\?\(computer\.takeoverRequested\|\|computer\.controlHolder==="user"\?