fix conversion css

This commit is contained in:
王性驊 2026-09-07 17:05:14 +08:00
parent aa19f1bccf
commit 807f549908
15 changed files with 336 additions and 60 deletions

View File

@ -19,7 +19,7 @@ import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR
import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n";
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings, WorkspaceSettings } from "./types";
import { ChatMarkdown, CopyMessageButton } from "./markdown";
import { RunProbe, errorActions, errorTitle } from "./run-monitor";
import { RunStatus, RunProbe, errorActions, errorTitle } from "./run-monitor";
import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, scheduleWhen, type CronPreset, type ScheduleItem } from "./schedule";
import { CallOverlay, PhoneIcon } from "./call";
import { VoiceSettingsDialog } from "./voice-settings";
@ -209,7 +209,7 @@ export function App(){
useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]);
useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]);
useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(localizeError(e.message)));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]);
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&["Return","BackSpace","Tab","Escape","Left","Right","Up","Down"].includes(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]);
useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]);
useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]);
@ -315,10 +315,23 @@ export function App(){
const startBoot=()=>changeComputer("boot");
const stopComputer=()=>changeComputer("stop");
const restartComputer=()=>changeComputer("restart");
useEffect(() => {
const viewport = window.visualViewport;
const update = () => {
document.documentElement.style.setProperty("--visible-height", `${viewport?.height || window.innerHeight}px`);
document.documentElement.style.setProperty("--visible-top", `${viewport?.offsetTop || 0}px`);
};
update(); viewport?.addEventListener("resize", update); viewport?.addEventListener("scroll", update);
window.addEventListener("resize", update);
return () => { viewport?.removeEventListener("resize", update); viewport?.removeEventListener("scroll", update); window.removeEventListener("resize", update); };
}, []);
const connecting=computer.state==="running"&&Boolean(screenUrl)&&!desktopReady;
const overlayLabel=hudLabel(computer,connecting,handingOff);
const frame=screenUrl?<iframe ref={desktopFrameRef} key={screenUrl} className="desktop-frame" src={screenUrl} title={t("agentComputer")} allow="fullscreen; clipboard-read; clipboard-write"/>:<EmptyComputer state={computer.state}/>;
const hud=paneBot&&overlayLabel?<ComputerHud bot={paneBot} label={overlayLabel}/>:null;
const liveRunId = (computer.busySessionId === activeSessionId ? computer.busyRunId : null)
|| (computer.waitingSessionId === activeSessionId ? computer.waitingRunId : null)
|| [...messages].reverse().find(message => message.runId)?.runId;
const statusMembers=workingMembers;
const topTools=<nav className="top-tools" aria-label={t("workTools")}>
{active&&!activeRoomId?<span className="call-entry" tabIndex={!voiceSettings?.enabled?0:undefined} title={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined} aria-label={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined}><button type="button" className={`top-tool-button ${callOpen?"call-active":""}`} title={!voiceSettings?.enabled?t("voiceDisabledHint"):voiceSettings.ready?t("call"):t("setUpVoiceToCall")} aria-label={t("call")} disabled={!activeSessionId||!voiceSettings?.enabled} onClick={()=>{if(!voiceSettings?.enabled)return;if(!voiceSettings.ready){setAccountDialog("voice");return}setCallOpen(true)}}><PhoneIcon/></button></span>:null}
@ -373,6 +386,7 @@ export function App(){
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
{otherSessionBusy&&<div className="queue-hint">{t("anotherConversationQueued")}</div>}
<div className={`composer-dock ${statusMembers.length?"has-status":""}`}>
{!computerOpen && <RunStatus runId={liveRunId}/>}
{statusMembers.map(member=>{const step=computer.busyStep&&member.id===computer.botId?computer.busyStep:null;const transition=isTransitionStep(step);const label=t("working",{name:member.name});return <div className="thinking-row" key={member.id}><RunProbe runId={member.id===computer.botId?computer.busyRunId:null}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/></RunProbe><span className="working-copy"><span className="working-label">{label}</span>{!transition&&step?<span className="working-step">{localizeStep(step)}</span>:null}</span></div>})}
<form className={`composer ${pendingFiles.length?"has-files":""}`} onSubmit={send} onDragOver={event=>{event.preventDefault()}} onDrop={event=>{event.preventDefault();if(event.dataTransfer.files.length)addPendingFiles(event.dataTransfer.files)}}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled={!activeSessionId||Boolean(teaching)} title={t("attachFileHint")} onClick={()=>{setPlusOpen(false);attachRef.current?.click()}}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button><button type="button" role="menuitem" disabled={!active} title={t("importSkillHint")} onClick={()=>{setPlusOpen(false);importRef.current?.click()}}><Upload/>{t("importSkill")}</button>{savedSkills.length>0&&<><hr/><div className="plus-menu-skills"><small className="plus-menu-label">{t("taughtSkills")}{savedSkills.length>5?` · ${savedSkills.length}`:""}</small>{savedSkills.length>=6&&<input className="plus-menu-search" value={skillQuery} onChange={e=>setSkillQuery(e.target.value)} placeholder={t("searchSkills")} aria-label={t("searchSkills")} onClick={e=>e.stopPropagation()}/>}<div className="plus-menu-skill-list">{listedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("exportSkillHint")} aria-label={t("exportSkill")} onClick={()=>downloadSkill(skill.name,skill.goal,skill.playbook)}><Download/></button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}{listedSkills.length===0&&<small className="plus-menu-empty">{t("noMatchingSkills")}</small>}</div></div></>}</div>}</div><input ref={importRef} className="skill-import-input" type="file" accept="application/json,.json" tabIndex={-1} aria-hidden="true" onChange={event=>{const file=event.target.files?.[0];event.currentTarget.value="";if(file)void importSkillFile(file)}}/><input ref={attachRef} className="skill-import-input attach-input" type="file" multiple accept={ATTACH_ACCEPT} tabIndex={-1} aria-hidden="true" onChange={event=>{const files=[...event.target.files||[]];event.currentTarget.value="";if(files.length)addPendingFiles(files)}}/>{pendingFiles.length>0&&<div className="composer-files">{pendingFiles.map(item=><FileCard key={item.id} file={{name:item.file.name,size:item.file.size}} preview={item.preview} onRemove={()=>removePendingFile(item.id)}/>)}</div>}{slashSuggestions.length>0&&<div className="slash-suggestions" role="listbox" aria-label={t("slashList")}>{slashSuggestions.map((skill,index)=><button type="button" role="option" aria-selected={index===slashIndex} key={skill.name} onClick={()=>{setDraft(`/${skill.name} `);requestAnimationFrame(()=>document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus())}}><strong>/{skill.name}</strong><small>{skill.kind}</small><span>{skill.description}</span></button>)}</div>}<textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} onPaste={event=>{const files=event.clipboardData?.files;if(files&&files.length){event.preventDefault();addPendingFiles(files)}}} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||(!draft.trim()&&pendingFiles.length===0)||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="var(--on-send)"/></button>}</form>
</div>
@ -388,7 +402,7 @@ export function App(){
<div className="side-card-body">
<div className={`side-part computer-part ${rightPart==="computer"?"":"hidden-part"}`}>
<div className="computer-status-row">{paneBot?<span>{t("botComputer",{name:paneBot.name})}</span>:<span>{t("computer")}</span>}{computer.state==="booting"?<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/>:<i className={`state-dot ${computer.state}`}/>}<small>{stateLabel(computer.state)}</small></div>
<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}{!computerOpen&&hud}</div>
{!computerOpen&&hud}<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}</div>
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<p className="computer-login-hint">{t("computerLoginHint")}</p><p className="clipboard-status" role="status">{clipboardStatus}</p>{scheduleDraft?<ScheduleEditor draft={scheduleDraft} timezone={scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei"} saving={scheduleSaving} error={scheduleError} onChange={next=>setScheduleDraft(current=>current?{...current,...next}:next)} onBack={()=>setScheduleDraft(null)} onSave={()=>void saveScheduleDraft()} onDelete={scheduleDraft.id?()=>void action(async()=>{await api(`/api/schedules/${scheduleDraft.id}`,{method:"DELETE"});setScheduleDraft(null);await reloadSchedules()}):undefined}/>:<ScheduleList items={schedules} runningId={runningScheduleId} onCreate={()=>setScheduleDraft({name:"",instructions:"",enabled:true,preset:defaultCronPreset()})} onOpen={item=>setScheduleDraft({id:item.id,timezone:item.timezone,threadId:item.threadId,name:item.name,instructions:item.instructions,enabled:item.enabled,preset:presetFromCron(item.cron)})} onRun={item=>void action(async()=>{setRunningScheduleId(item.id);try{await api(`/api/schedules/${item.id}/run`,{method:"POST",body:"{}"});await loadSessions()}finally{setRunningScheduleId(null)}})}/>}</>}
</div>
{rightPart==="accounts"&&paneBot&&<VaultPane bot={paneBot}/>}
@ -399,7 +413,7 @@ export function App(){
</>
</aside>}
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="var(--ink)"/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen"><div className="overlay-desktop">{frame}{hud}</div></div>{error&&<div className="overlay-error">{error}</div>}</div>}
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="var(--ink)"/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><RunStatus runId={liveRunId}/>{hud}<div className="overlay-screen"><div className="overlay-desktop">{frame}</div></div>{error&&<div className="overlay-error">{error}</div>}</div>}
{teachOpen&&active&&<TeachDialog bot={active} busy={busy} close={()=>setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>}
{editingSkill&&<SkillEditDialog key={editingSkill.id} skill={editingSkill} busy={busy} close={()=>setEditingSkillId(null)} save={(name,playbook)=>void updateSkill(editingSkill,name,playbook)} test={(name,playbook)=>void testSkill(editingSkill,name,playbook)} remove={()=>void deleteSkill(editingSkill)} exportFile={(name,playbook)=>downloadSkill(name,editingSkill.goal,playbook)}/>}
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{pasteText(text);setClipboardOpen(false)}}/>}

View File

@ -68,3 +68,22 @@
.computer-power-menu .computer-power-stop{color:var(--danger-soft)}
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line);color:var(--muted);font-size:11px;line-height:1.6}
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}
/* Status belongs outside the remote pixels, including during connection/handoff. */
.computer-overlay{top:var(--visible-top,0px);bottom:auto;height:var(--visible-height,100dvh)}
.computer-overlay>header{flex-shrink:0}
.computer-hud{position:static;inset:auto;display:block;padding:7px 12px;border-radius:0;background:var(--surface);flex:none}
.computer-hud .computer-signal{display:none}
.overlay-error{position:static;transform:none;flex:none;margin:0 8px 8px}
.run-status{flex:none;min-width:0;padding:8px 12px;border:1px solid var(--line);border-radius:10px;background:var(--surface);font-size:12px;max-height:26vh;overflow:auto;margin:6px 0}
.run-status-heading{display:flex;gap:8px;align-items:center}
.run-status-heading small{margin-left:auto;color:var(--muted)}
.run-status p{margin:5px 0;overflow-wrap:anywhere}
.run-status summary{cursor:pointer;color:var(--muted);padding:4px 0}
.run-status ol{padding-left:20px;max-height:180px;overflow:auto}
.run-status li{padding:4px 0;overflow-wrap:anywhere}
.run-status time{color:var(--muted)}
.run-status.has-error,.run-status-issue{color:var(--danger-soft)}
.run-status-wait{color:var(--muted)}
.computer-overlay>.run-status{margin:6px 8px;max-height:22vh}
@media(max-width:700px){.computer-overlay>.run-status{max-height:18vh}.overlay-screen{padding:4px}.computer-overlay>header{min-height:44px;gap:4px}.computer-overlay>header>div{gap:5px}.computer-overlay>header strong{font-size:12px}}

View File

@ -2,6 +2,17 @@ import type { zhTW } from "./zh-TW";
/** English UI copy. Keys must stay in lockstep with zh-TW. */
export const en: { [K in keyof typeof zhTW]: string } = {
liveCompleted: "Completed",
liveFailed: "Task incomplete: an error occurred",
liveCancelled: "Stopped: cancelled or interrupted",
liveInput: "Your reply is needed to continue",
liveTakeover: "Waiting for you; resumes after handoff",
liveRunning: "Working",
liveLoading: "Loading run status…",
liveDisconnected: "Status unavailable; reconnecting. This does not mean the AI has stopped.",
liveSlow: "This step has not returned yet. Timeouts, retries and any stop reason will be reported.",
liveHistory: "Actions, errors and timings",
search: "Search",
sharedComputer: "Shared computer",
privateComputer: "Private computer",

View File

@ -1,5 +1,16 @@
/** Traditional Chinese UI copy. Keep keys stable when adding another locale. */
export const zhTW = {
liveCompleted: "已完成",
liveFailed: "任務未完成:發生錯誤",
liveCancelled: "已停止:使用者取消或工作被中斷",
liveInput: "需要你的回覆才能繼續",
liveTakeover: "等待你操作,交回後繼續",
liveRunning: "正在執行",
liveLoading: "正在取得執行狀態…",
liveDisconnected: "暫時無法取得狀態,正在重新連線;這不代表 AI 已停止。",
liveSlow: "這一步仍未回報結果。系統會處理逾時並顯示重試或停止原因。",
liveHistory: "查看做了什麼、錯誤與耗時",
search: "搜尋", sharedComputer: "共用電腦", privateComputer: "私人電腦",
stopped: "已關閉", booting: "啟動中", running: "執行中", suspended: "休眠中", error: "發生錯誤",
openComputer: "開啟電腦", stopTask: "停止任務", takeControl: "取得控制", takeOverNow: "接手操作", releaseControl: "釋放控制", done: "完成", skip: "略過",

View File

@ -370,3 +370,48 @@ export function RunProbe({ runId, align = "start", label, children }: { runId?:
</span>
);
}
/** A persistent readout: errors and waiting reasons stay visible after the run stops. */
export function RunStatus({ runId }: { runId?: string | null }) {
const [snapshot, setSnapshot] = useState<RunActivity | null>(null);
const [stale, setStale] = useState(false);
const [now, setNow] = useState(Date.now());
useEffect(() => {
setSnapshot(null); setStale(false);
if (!runId) return;
let stopped = false;
let timer = 0;
const load = async () => {
try {
const next = await api<RunActivity>(`/api/runs/${runId}/activity?limit=30`, {signal: AbortSignal.timeout(10000)});
if (stopped) return;
setSnapshot(next); setStale(false); setNow(Date.now());
} catch { if (!stopped) setStale(true); }
// Poll sequentially; also notice a paused/failed run being resumed elsewhere.
if (!stopped) timer = window.setTimeout(() => void load(), POLL_MS);
};
void load();
return () => { stopped = true; window.clearTimeout(timer); };
}, [runId]);
if (!runId) return null;
const entries = snapshot?.activity || [];
const latest = entries.at(-1);
const active = snapshot && ["queued", "leased", "running"].includes(snapshot.status);
const age = snapshot?.stepAt ? Math.max(0, now - Date.parse(snapshot.stepAt)) : 0;
const state = snapshot?.status || "loading";
const title = t(({completed:"liveCompleted", failed:"liveFailed", cancelled:"liveCancelled", waiting_input:"liveInput", waiting_takeover:"liveTakeover", queued:"monitorQueued", running:"liveRunning", leased:"liveRunning"} as Record<string, MessageKey>)[state] || "liveLoading");
const issue = [...entries].reverse().find(entry => entry.kind === "retry" || entry.status === "error" || entry.status === "timed_out");
return <section className={`run-status ${snapshot?.error ? "has-error" : ""}`} aria-label={t("monitorTitle")}>
<div className="run-status-heading" role="status"><strong>{title}</strong>{active && age > 0 ? <small>{shortDuration(age)}</small> : null}</div>
{stale ? <p role="alert">{t("liveDisconnected")}</p> : null}
{snapshot?.error ? <p role="alert">{snapshot.error.headline} {snapshot.error.action}</p> : <p>{active ? snapshot?.step || t("liveLoading") : latest ? trailText(latest) : t("monitorEmpty")}</p>}
{active && age >= 20000 ? <p className="run-status-wait">{t("liveSlow")}</p> : null}
{active && latest?.kind === "model" && latest.text ? <p>{latest.text}</p> : null}
{issue && !snapshot?.error ? <p className="run-status-issue">{trailText(issue)}</p> : null}
<details><summary>{t("liveHistory")}</summary>
<ol>{entries.map(entry => <li key={entry.id}><time>{clockOf(entry.createdAt)}</time> {trailText(entry)}</li>)}</ol>
<button type="button" onClick={() => void copyLog(logDump(snapshot, entries))}>{t("monitorCopy")}</button>
</details>
</section>;
}

View File

@ -13,8 +13,8 @@ export interface ComputerStatus { botId:string; mode:ComputerMode; state:Compute
/** One line of the live trail a run writes while it works. */
export type RunActivityKind = "run"|"model"|"tool"|"retry"|"notice";
export interface RunActivityEntry { id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null }
export interface RunActivityError { code:string; headline:string; raw:string }
export interface RunActivity { runId:string; status:string; turn:number|null; turnLimit:number|null; step:string|null; elapsedMs:number|null; error:RunActivityError|null; activity:RunActivityEntry[] }
export interface RunActivityError { code:string; headline:string; action?:string; raw:string }
export interface RunActivity { runId:string; status:string; turn:number|null; turnLimit:number|null; step:string|null; stepAt?:string|null; elapsedMs:number|null; error:RunActivityError|null; activity:RunActivityEntry[] }
export interface PlaybookStep { do:string; expect?:string; note?:string }
export interface PlaybookInput { name:string; description?:string; example?:string }

View File

@ -12,12 +12,28 @@
background: #0f172a;
overflow: hidden;
}
#status {
display: none;
}
#status { flex:none; padding:6px 10px; color:#f8fafc; font:12px system-ui; }
#status[hidden] { display:none; }
body { display:flex; flex-direction:column; }
#screen { flex:1; min-height:0; height:auto; }
#screen canvas { cursor: default; }
#trackpad { height:clamp(64px,18vh,150px); margin-top:6px; border:1px solid #64748b;
border-radius:10px; background:#111b2e; touch-action:none; user-select:none;
display:grid; place-items:center; color:#94a3b8; }
#trackpad[hidden] { display:none; }
#mobile-controls .shortcuts { margin-top:5px; }
#mobile-controls .shortcuts[hidden] { display:none; }
@media(orientation:landscape) {
body.touch-ui {
display:grid;
grid-template:"status status" auto "screen controls" 1fr / minmax(0,1fr) 210px;
}
body.touch-ui #status { grid-area:status; }
body.touch-ui #screen { grid-area:screen; width:auto; height:auto; min-width:0; }
body.touch-ui #mobile-controls { grid-area:controls; width:auto; box-sizing:border-box; overflow:auto; }
body.touch-ui #mobile-controls .buttons { flex-wrap:wrap; }
body.touch-ui #trackpad { height:80px; }
}
#mobile-controls { display:none; flex:none; padding:6px 8px max(6px,env(safe-area-inset-bottom));
background:#172033; color:#f8fafc; font:12px system-ui; border-top:1px solid #334155; }
#mobile-controls .buttons { display:flex; gap:5px; overflow-x:auto; }
@ -32,6 +48,7 @@
#mobile-cursor::after { content:""; position:absolute; width:4px; height:4px; background:#fff;
border-radius:50%; left:7px; top:7px; }
@media(any-pointer:coarse) { #mobile-controls { display:block; } }
body.touch-ui #mobile-controls { display:block; }
/* Keep a real editable element on screen for iOS/Android keyboards. */
#mobile-keyboard { position:fixed; bottom:0; left:0; width:1px; height:1px;
padding:0; border:0; opacity:.01; font-size:16px; pointer-events:none; }
@ -56,6 +73,7 @@
const statusEl = document.getElementById("status");
const setStatus = (text) => {
statusEl.textContent = text;
statusEl.hidden = !text;
};
const prefix = window.location.pathname.replace(/[^/]+$/, "");
@ -78,7 +96,13 @@
const modeButton = document.getElementById("pointer-mode");
const dragButton = document.getElementById("pointer-drag");
const pointerHelp = document.getElementById("pointer-help");
let trackpad = false;
const touchUi = window.matchMedia?.("(any-pointer:coarse)");
document.body?.classList?.toggle("touch-ui", Boolean(touchUi?.matches));
touchUi?.addEventListener?.("change", event => document.body?.classList?.toggle("touch-ui", event.matches));
let trackpad = Boolean(touchUi?.matches);
const pad = document.getElementById("trackpad");
const shortcuts = document.getElementById("keyboard-shortcuts");
let modifier = null;
let dragging = false;
let gesture = null;
let pointer = { x: .5, y: .5 };
@ -126,11 +150,12 @@
buttons:0, preventDefault(){}, stopPropagation(){} });
}
function updatePointerControls() {
if (pad) pad.hidden = !trackpad;
modeButton?.setAttribute?.("aria-pressed", String(trackpad));
if (modeButton) modeButton.textContent = trackpad ? "觸控板" : "直接點選";
if (pointerHelp) pointerHelp.textContent = trackpad
? "滑動移游標・輕點左鍵・雙指捲動/輕點右鍵・拖曳按完再按一次放開"
: "點畫面定位並開鍵盤・長按右鍵・精準操作可切換觸控板";
: "點畫面定位・按「鍵盤」輸入・精準操作可切換觸控板";
for (const button of controls?.querySelectorAll?.("button[data-action]") || []) {
button.disabled = !rfb || rfb.viewOnly;
}
@ -139,6 +164,7 @@
function showKeyboard() {
if (!rfb || rfb.viewOnly) return;
rfb.focusOnClick = false;
if (shortcuts) shortcuts.hidden = false;
keyboard.focus({ preventScroll: true });
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
}
@ -152,6 +178,13 @@
}
if (!rfb || rfb.viewOnly) return;
if (action === "keyboard") { releaseDrag(); showKeyboard(); return; }
if (action === "hide-keyboard") { keyboard.blur(); if (shortcuts) shortcuts.hidden = true; return; }
if (action === "modifier") {
modifier = modifier === button.dataset.key ? null : button.dataset.key;
for (const item of controls.querySelectorAll('[data-action="modifier"]')) item.setAttribute("aria-pressed", String(item.dataset.key === modifier));
return;
}
if (action === "key") { mobileKey(button.dataset.key); return; }
if (action === "left") pointerClick(1);
if (action === "right") pointerClick(4);
if (action === "up") { releaseDrag(); scrollPointer(0, -100); }
@ -170,7 +203,7 @@
y:points.reduce((sum,p)=>sum+p.clientY,0)/points.length };
}
function startPointerTouch(event) {
if (!trackpad || !event.target.closest?.("#screen")) return false;
if (!trackpad || !event.target.closest?.("#trackpad")) return false;
stopTouch(event);
if (!rfb || rfb.viewOnly) {
window.parent.postMessage({ type:"lazyboy-request-control" }, parentOrigin);
@ -222,7 +255,9 @@
}
function mobileKey(key) {
if (!rfb || rfb.viewOnly) return;
window.parent.postMessage({ type: "lazyboy-mobile-key", key }, parentOrigin);
window.parent.postMessage({ type: "lazyboy-mobile-key", key: modifier ? `${modifier}+${key}` : key }, parentOrigin);
modifier = null;
for (const item of controls?.querySelectorAll?.('[data-action="modifier"]') || []) item.setAttribute("aria-pressed", "false");
}
function commitKeyboard() {
if (composing) return;
@ -255,9 +290,17 @@
});
// Focus synchronously in the user's tap, before iOS loses activation.
// VNC pixels cannot reveal whether the remote target is a text field.
function ignoreScreenTouch(event) {
if (!trackpad || !event.target.closest?.("#screen")) return false;
event.preventDefault();
event.stopImmediatePropagation();
touchStart = null;
return true;
}
window.addEventListener("touchstart", event => {
if (event.target === keyboard) return;
if (startPointerTouch(event)) { touchStart = null; return; }
if (ignoreScreenTouch(event)) return;
const point = event.touches[0];
const rect = pointerGeometry();
if (point && rect && event.target.closest?.("#screen")) {
@ -270,6 +313,7 @@
}, { passive: false, capture: true });
window.addEventListener("touchmove", event => {
if (movePointerTouch(event)) return;
if (ignoreScreenTouch(event)) return;
const point = event.touches[0];
if (touchStart && (!point || event.touches.length !== 1 ||
Math.hypot(point.clientX - touchStart.x, point.clientY - touchStart.y) > 10)) touchStart = null;
@ -277,11 +321,11 @@
window.addEventListener("touchcancel", () => { touchStart = null; gesture = null; releaseDrag(); }, true);
window.addEventListener("touchend", event => {
if (endPointerTouch(event)) return;
if (ignoreScreenTouch(event)) return;
const tapped = touchStart && event.touches.length === 0;
touchStart = null;
if (!tapped || !rfb || rfb.viewOnly) return;
keyboard.focus({ preventScroll: true });
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
// Direct taps position the pointer; the keyboard opens only on request.
}, { passive: false, capture: true });
function pasteIntoDesktop(text) {
@ -305,7 +349,7 @@
function applyViewOnly(value) {
if (!rfb) return;
if (value) { releaseDrag(); gesture = null; }
if (value) { releaseDrag(); gesture = null; modifier = null; if (shortcuts) shortcuts.hidden = true; }
rfb.viewOnly = Boolean(value);
updatePointerControls();
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
@ -317,11 +361,14 @@
rfb = new RFB(document.getElementById("screen"), url);
rfb.viewOnly = flag("view_only", true);
rfb.scaleViewport = true;
rfb.qualityLevel = 6;
rfb.compressionLevel = 2;
rfb.clipViewport = false;
rfb.background = "#0f172a";
pinTaskbar();
updatePointerControls();
rfb.addEventListener("connect", () => {
setStatus("");
pinTaskbar();
updatePointerControls();
try { rfb.focus(); } catch (_) {}
@ -416,6 +463,18 @@
<button data-action="down" aria-label="向下捲動"></button>
<button data-action="keyboard">鍵盤</button>
</div>
<div class="buttons shortcuts" id="keyboard-shortcuts" hidden>
<button data-action="modifier" data-key="ctrl" aria-pressed="false">Ctrl</button>
<button data-action="modifier" data-key="alt" aria-pressed="false">Alt</button>
<button data-action="key" data-key="a">A</button><button data-action="key" data-key="c">C</button>
<button data-action="key" data-key="v">V</button><button data-action="key" data-key="z">Z</button>
<button data-action="key" data-key="Tab">Tab</button><button data-action="key" data-key="Escape">Esc</button>
<button data-action="key" data-key="Left"></button><button data-action="key" data-key="Right"></button>
<button data-action="key" data-key="Up"></button><button data-action="key" data-key="Down"></button>
<button data-action="key" data-key="Return">Enter</button><button data-action="key" data-key="BackSpace"></button>
<button data-action="hide-keyboard">收起鍵盤</button>
</div>
<div id="trackpad" role="group" aria-label="獨立觸控板" hidden>在這裡滑動控制滑鼠</div>
<p id="pointer-help"></p>
</nav>
<textarea id="mobile-keyboard" aria-label="Remote desktop keyboard" tabindex="-1"

View File

@ -277,6 +277,7 @@ struct RunRow {
turn: Option<i64>,
turn_limit: Option<i64>,
error: Option<String>,
step_at: Option<DateTime<Utc>>,
started_at: Option<DateTime<Utc>>,
completed_at: Option<DateTime<Utc>>,
}
@ -301,7 +302,7 @@ async fn activity(
let actor = actor(&state).await?;
let run = sqlx::query_as::<_, RunRow>(
"SELECT status, checkpoint->>'step' AS step, (checkpoint->>'turn')::bigint AS turn,
(checkpoint->>'turnLimit')::bigint AS turn_limit, error, started_at, completed_at
(checkpoint->>'turnLimit')::bigint AS turn_limit, error, (checkpoint->>'stepAt')::timestamptz AS step_at, started_at, completed_at
FROM runs WHERE id=$1 AND space_id=$2 AND user_id=$3",
)
.bind(&id)
@ -336,7 +337,7 @@ async fn activity(
.filter(|raw| !raw.trim().is_empty())
.map(|raw| {
let failure = classify_run_error(raw);
json!({"code": failure.code, "headline": failure.headline, "raw": raw})
json!({"code": failure.code, "headline": failure.headline, "action": failure.action, "raw": raw})
});
let elapsed = run.started_at.map(|started| {
(run.completed_at.unwrap_or_else(Utc::now) - started)
@ -350,6 +351,7 @@ async fn activity(
"turnLimit": run.turn_limit,
"step": run.step,
"elapsedMs": elapsed,
"stepAt": run.step_at,
"error": error,
"activity": rows.into_iter().map(|(row_id, kind, payload, created_at)| {
let mut entry = payload;

View File

@ -55,6 +55,8 @@ When you use the browser tool:
When a click changes nothing: do not repeat it blindly. Take a fresh snapshot, read the [disabled] tags and the page text, then act. Pages that need patience (training videos, quizzes, slow forms) are normal: keep working through them step by step and report progress in one short sentence when done.
Explain your next concrete action briefly before tool calls, in the user's language. When a tool fails, explain what failed and how you will recover in your next update. Recover from temporary errors by observing current state and choosing a different action; never replay an uncertain mutation blindly. Before any necessary stop, state what is completed, what remains, the specific blocker, and the next action needed. Never silently stop or claim a failed tool succeeded.
Waiting is a tool call, never a reply. Ending your turn with \"waiting for X\" stops the whole run; nobody resumes it. If something must finish first, call wait (or click, which waits) and continue.
Multi-step tasks and taught skills: you are done only when the playbook's check passes (for example the course shows completed, the form shows a confirmation). Do not stop with a status sentence in the middle; keep calling tools until the check passes or you are truly blocked, then say exactly why. Never repeat an earlier reply word for word; describe the current screen.
@ -688,16 +690,13 @@ async fn execute_run(
)
.await;
}
// Taught skills run long (a 24-page course is 24 clicks); plain chats stay
// bounded tighter so a confused model cannot burn budget for as long.
let execution_mode = if goal_mode {
ExecutionMode::Goal
} else if skill_check.is_some() && file_skill.is_none() {
ExecutionMode::Bounded(80)
} else if chat_only {
// Computer work keeps going until it verifies or explains a blocker.
// Plain chat stays short so a confused model cannot burn budget.
let goal_mode = goal_mode || !chat_only;
let execution_mode = if chat_only {
ExecutionMode::Bounded(4)
} else {
ExecutionMode::Bounded(40)
ExecutionMode::Goal
};
let turn_limit = match execution_mode {
ExecutionMode::Goal => None,
@ -1375,6 +1374,7 @@ async fn execute_run(
"run",
json!({
"event": if needs_input { "waiting_input" } else { "completed" },
"reason": if needs_input { Some(final_text.as_str()) } else { None },
"turns": turns,
}),
)
@ -1490,6 +1490,13 @@ async fn complete_with_retry(
) -> Result<Vec<AssistantContent>, String> {
let mut last = String::new();
for attempt in 0..3 {
if attempt > 0 {
let failure = crate::monitor::classify_run_error(&last);
set_run_step(trace.state, trace.run_id, &format!(
"{} 正在自動重試模型(第 {}/3 次);保留已完成的操作。",
failure.headline, attempt + 1
)).await;
}
let started = std::time::Instant::now();
let result = tokio::time::timeout(
Duration::from_secs(165),
@ -1507,7 +1514,7 @@ async fn complete_with_retry(
"turn": trace.turn,
"attempt": attempt + 1,
"error": error,
"gaveUp": !retryable_run_error(&error),
"gaveUp": attempt == 2 || !retryable_run_error(&error),
}),
)
.await;
@ -1527,7 +1534,7 @@ async fn complete_with_retry(
trace.state,
trace.run_id,
"retry",
json!({"turn": trace.turn, "attempt": attempt + 1, "error": error}),
json!({"turn": trace.turn, "attempt": attempt + 1, "error": error, "gaveUp": attempt == 2}),
)
.await;
tracing::warn!(
@ -1928,16 +1935,26 @@ async fn pause_for_answer(
req: PauseRequest<'_>,
) -> Result<(), String> {
let draft = req.draft.replace(NEEDS_INPUT_MARKER, "").trim().to_string();
let stall = match req.reason {
StopReason::BudgetExhausted => "已達本輪執行上限",
StopReason::MidTaskText => {
"模型多次未能提供可執行的下一步,系統已要求它重新確認並繼續,但仍無法推進"
}
};
let draft = if draft.is_empty() {
match req.reason {
StopReason::BudgetExhausted => format!(
"我在這個任務上用了 {} 輪,還沒有做到可以幫你確認完成的地步,先停在目前的畫面。要我繼續嗎?",
req.turns
"我在這個任務上用了 {} 輪,還沒有做到可以幫你確認完成的地步,先停在目前的畫面。要我繼續嗎?\n\n任務尚未確認完成。停止原因:{}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。",
req.turns, stall
),
StopReason::MidTaskText => format!(
"模型多次未提供可執行的下一步,系統無法確認任務已完成。請補充下一步指示或接管確認現況後繼續。\n\n任務尚未確認完成。停止原因:{}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。",
stall
),
StopReason::MidTaskText => {
"我先到這裡,需要你的決定或資料才能繼續。要我接著做嗎?".to_string()
}
}
} else if !asks_for_input(req.draft) {
// A model's optimistic draft must not hide a harness-detected stall.
format!("{draft}\n\n任務尚未確認完成。停止原因:{stall}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。")
} else {
draft
};
@ -1978,7 +1995,7 @@ async fn pause_for_answer(
"run",
json!({
"event": "paused",
"reason": req.reason.as_str(),
"reason": draft,
"turns": req.turns,
"limit": req.limit,
}),

View File

@ -138,7 +138,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
},
ToolDefinition {
name: "browser".into(),
description: "Control Chromium through the page DOM when the user asked you to use the browser. Prefer this over computer_act for anything in the page. snapshot returns numbered elements and visible text (no screenshot). click/type/navigate by element id or CSS selector. click scrolls off-screen elements into view and, if the control is [disabled], waits up to 45s (waitMs to change) for it to enable before clicking. Ids are renumbered after every page change. The human still sees the live window.".into(),
description: "Control Chromium through the page DOM when the user asked you to use the browser. Prefer this over computer_act for anything in the page. Every action returns fresh numbered elements and visible text; screenshots are opt-in with observe:true for visual ambiguity. click/type/navigate by element id or CSS selector. click scrolls off-screen elements into view and, if the control is [disabled], waits up to 45s (waitMs to change) for it to enable before clicking. Ids are renumbered after every page change. The human still sees the live window.".into(),
parameters: json!({
"type":"object",
"properties":{
@ -149,6 +149,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
"key":{"type":"string"},
"url":{"type":"string"},
"ms":{"type":"number"},
"observe":{"type":"boolean","description":"Attach a screenshot only when visual verification is needed; default false"},
"waitMs":{"type":"number","description":"click only: how long to wait for a disabled control to enable (default 45000, max 120000)"}
},
"required":["action"]
@ -765,7 +766,7 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
));
}
let page = cdp_call(ctx, request).await;
if !page.elements.is_empty() {
if page.ok || !page.elements.is_empty() {
*ctx.elements.lock().unwrap() = page.elements.clone();
}
if !page.ok {
@ -789,7 +790,7 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
" (the control was disabled; waited {seconds:.0}s for it to enable before clicking)"
));
}
if action == "snapshot" {
if args.get("observe").and_then(Value::as_bool) != Some(true) {
return ToolOutcome {
text,
image: None,
@ -812,17 +813,10 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
}
fn browser_result_text(action: &str, page: &CdpPage) -> String {
if matches!(action, "snapshot" | "navigate") {
format!(
"browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}",
page.title,
page.url,
format_ui_elements(&page.elements),
page.text
)
} else {
format!("browser {action} ok")
}
format!(
"browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}",
page.title, page.url, format_ui_elements(&page.elements), page.text
)
}
async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome {

View File

@ -609,7 +609,8 @@ def main():
fail(val.get("error") or "click failed")
pointer(display, val.get("x") or 0, val.get("y") or 0)
wait_for_visual_update(ws)
out = {"ok": True, "action": "click", "selector": sel, "restarted": restarted}
out = snapshot(ws)
out.update({"action": "click", "selector": sel, "restarted": restarted})
if waited >= 1.0:
out["waitedSeconds"] = round(waited, 1)
print(json.dumps(out))
@ -636,23 +637,30 @@ def main():
text = req.get("text") or ""
if sel:
val = evaluate(ws, CLICK_JS, sel) or {}
if val.get("ok"):
pointer(display, val.get("x") or 0, val.get("y") or 0)
if not val.get("ok"):
fail(val.get("error") or "target field is unavailable; no text inserted")
pointer(display, val.get("x") or 0, val.get("y") or 0)
if text:
ws.call("Input.insertText", {"text": text})
wait_for_visual_update(ws)
print(json.dumps({"ok": True, "action": "type", "restarted": restarted}))
out = snapshot(ws)
out.update({"action": "type", "restarted": restarted})
print(json.dumps(out))
return
if action == "press":
key = req.get("key") or "Return"
press(ws, key)
wait_for_visual_update(ws)
print(json.dumps({"ok": True, "action": "press", "key": key, "restarted": restarted}))
out = snapshot(ws)
out.update({"action": "press", "key": key, "restarted": restarted})
print(json.dumps(out))
return
if action == "wait":
ms = min(max(int(req.get("ms") or 400), 0), 5000)
time.sleep(ms / 1000.0)
print(json.dumps({"ok": True, "action": "wait", "ms": ms}))
out = snapshot(ws)
out.update({"action": "wait", "ms": ms})
print(json.dumps(out))
return
fail("unsupported action")
finally:

View File

@ -28,7 +28,12 @@ impl ExecutionMode {
/// Only a standalone terminal marker is an outcome, not a quoted mention.
pub fn goal_outcome(reply: &str) -> GoalOutcome {
match reply.trim().lines().last().map(str::trim) {
let reply = reply.trim();
// A bare marker provides neither verification nor a reason to the human.
if reply.lines().count() < 2 {
return GoalOutcome::Continue;
}
match reply.lines().last().map(str::trim) {
Some("[GOAL_COMPLETE]") => GoalOutcome::Complete,
Some("[GOAL_BLOCKED]") => GoalOutcome::NeedsInput,
_ => GoalOutcome::Continue,
@ -165,6 +170,8 @@ mod tests {
#[test]
fn progress_and_quoted_markers_do_not_complete_a_goal() {
assert_eq!(goal_outcome("[GOAL_COMPLETE]"), GoalOutcome::Continue);
assert_eq!(goal_outcome("\n[GOAL_BLOCKED]"), GoalOutcome::Continue);
assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue);
assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete);
assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput);

View File

@ -1,3 +1,6 @@
import contextlib
import io
import json
import importlib.util
import subprocess
import unittest
@ -40,4 +43,33 @@ class ClipboardTest(unittest.TestCase):
ws=cdp.Ws.__new__(cdp.Ws);ws.sock=Socket();ws.n=0
self.assertEqual(ws.call('Runtime.test'),{'ok':True})
class BrowserActionTest(unittest.TestCase):
def run_action(self, request, evaluation=None):
calls=[]
class Socket:
def call(self, method, params=None): calls.append((method, params)); return {}
def close(self): pass
output=io.StringIO()
def evaluate(ws, expression, arg=None):
if expression == cdp.SNAP_JS:
return {"url":"https://example.test/next", "title":"Next", "text":"Saved", "elements":[{"id":1,"title":"Continue"}]}
return evaluation if evaluation is not None else {"ok":True,"x":10,"y":20}
with patch.object(cdp.sys,'argv',['cdp',json.dumps(request)]), patch.object(cdp,'probe',return_value=True), patch.object(cdp,'connect',return_value=Socket()), patch.object(cdp,'evaluate',side_effect=evaluate), patch.object(cdp,'pointer'), patch.object(cdp,'wait_for_visual_update'), patch.object(cdp,'wait_until_enabled',return_value=0), patch.object(cdp.time,'sleep'), contextlib.redirect_stdout(output):
try: cdp.main()
except SystemExit: pass
return json.loads(output.getvalue()), calls
def test_every_browser_action_returns_current_elements_and_text(self):
for action in ['click','type','press','wait','navigate']:
with self.subTest(action=action):
result,_=self.run_action({"action":action,"selector":"#field","text":"hello","url":"https://example.test"})
self.assertTrue(result['ok'])
self.assertEqual(result['text'],'Saved')
self.assertEqual(result['elements'][0]['title'],'Continue')
def test_missing_field_never_types_into_previous_focus(self):
result,calls=self.run_action({"action":"type","selector":"#gone","text":"private"},{"ok":False,"error":"element gone"})
self.assertFalse(result['ok'])
self.assertFalse(any(method=='Input.insertText' for method,_ in calls))
if __name__=='__main__':unittest.main()

View File

@ -159,16 +159,18 @@ function mobileViewer(){
_handleMouseButton(x,y,mask){pointerEvents.push({type:'button',x,y,mask})}
_handleWheel(event){pointerEvents.push({type:'wheel',dx:event.deltaX,dy:event.deltaY})}
}
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent:{postMessage:x=>sent.push(x)},addEventListener:(n,f,o)=>{handlers[n]=f;options[n]=o}};
vm.runInNewContext(source,{window,document:{location:{href:'http://localhost/vnc.html?view_only=false'},getElementById:id=>id==='mobile-keyboard'?keyboard:element(id),querySelector:selector=>selector==='#screen canvas'?canvas:null},navigator:{},RFB,setTimeout(){},clearTimeout(){}});
const body={className:'',classList:{tokens:new Set(),toggle(name,force){if(force===undefined)force=!this.tokens.has(name);if(force)this.tokens.add(name);else this.tokens.delete(name);body.className=[...this.tokens].join(' ')}}};
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent:{postMessage:x=>sent.push(x)},addEventListener:(n,f,o)=>{handlers[n]=f;options[n]=o},matchMedia:()=>({matches:false,addEventListener(){}})};
vm.runInNewContext(source,{window,document:{body,location:{href:'http://localhost/vnc.html?view_only=false'},getElementById:id=>id==='mobile-keyboard'?keyboard:element(id),querySelector:selector=>selector==='#screen canvas'?canvas:null},navigator:{},RFB,setTimeout(){},clearTimeout(){}});
return {handlers,options,sent,keyboard,canvas,rfb,window,pointerEvents,elements,body};
return {handlers,options,sent,keyboard,canvas,rfb,window,pointerEvents,elements};
}
test('mobile tap focuses editable input before noVNC stops touch propagation; drag and multi-touch do not',()=>{
test('direct mobile taps position without unexpectedly opening the keyboard',()=>{
const {handlers:h,options,keyboard,canvas,rfb}=mobileViewer();
for(const name of ['touchstart','touchmove','touchend'])assert.equal(options[name].capture,true);
const point={clientX:20,clientY:30};
h.touchstart({target:canvas,touches:[point]});h.touchend({touches:[]});
assert.equal(keyboard.focused,true);assert.equal(rfb.focusOnClick,false);
assert.equal(keyboard.focused,false);assert.equal(rfb.focusOnClick,false);
keyboard.blur();h.touchstart({target:canvas,touches:[point]});h.touchmove({touches:[{clientX:50,clientY:30}]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
h.touchstart({target:canvas,touches:[point,point]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
rfb.viewOnly=true;h.touchstart({target:canvas,touches:[point]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
@ -200,7 +202,7 @@ function pointerButton(viewer,action){
}
function pointerTouch(viewer,name,points){
let prevented=false,stopped=false;
viewer.handlers[name]({target:viewer.canvas,touches:points.map(([clientX,clientY])=>({clientX,clientY})),preventDefault(){prevented=true},stopImmediatePropagation(){stopped=true}});
viewer.handlers[name]({target:{closest:selector=>selector==='#trackpad'?{}:null},touches:points.map(([clientX,clientY])=>({clientX,clientY})),preventDefault(){prevented=true},stopImmediatePropagation(){stopped=true}});
return prevented&&stopped;
}
test('trackpad moves relative to cursor, clamps edges and taps without opening keyboard',()=>{
@ -241,6 +243,53 @@ test('view-only pointer controls never send mouse input and trackpad asks for co
assert.equal(v.pointerEvents.length,0);
assert.equal(v.sent.at(-1).type,'lazyboy-request-control');
});
test('trackpad mode swallows screen taps so fingers on the picture do not click',()=>{
const v=mobileViewer();pointerButton(v,'mode');
let prevented=false,stopped=false;
v.handlers.touchstart({target:v.canvas,touches:[{clientX:20,clientY:30}],preventDefault(){prevented=true},stopImmediatePropagation(){stopped=true}});
assert.equal(prevented&&stopped,true);
assert.equal(v.pointerEvents.length,0);
assert.equal(v.keyboard.focused,false);
});
test('coarse pointers start in trackpad mode with a body class for independent controls',()=>{
const source=fs.readFileSync('apps/web/vnc.html','utf8').match(/<script type="module">([\s\S]*?)<\/script>/)[1].replace(/import RFB[^;]+;/,'');
const body={className:'',classList:{tokens:new Set(),toggle(name,force){if(force)this.tokens.add(name);else this.tokens.delete(name);body.className=[...this.tokens].join(' ')}}};
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent:{postMessage(){}},addEventListener(){},matchMedia:()=>({matches:true,addEventListener(){}})};
const elements={}; const element=id=>elements[id]||(elements[id]={style:{},attributes:{},hidden:true,setAttribute(k,v){this.attributes[k]=v},querySelectorAll(){return []}});
class RFB{constructor(){} addEventListener(){} focus(){}}
vm.runInNewContext(source,{window,document:{body,location:{href:'http://localhost/vnc.html?view_only=false'},getElementById:id=>element(id),querySelector:()=>null},navigator:{},RFB,setTimeout(){},clearTimeout(){}});
assert.equal(body.className,'touch-ui');
assert.equal(elements.trackpad.hidden,false);
});
test('mobile shortcut row sends modifiers and hides with the keyboard',()=>{
const v=mobileViewer();
pointerButton(v,'keyboard');
assert.equal(v.keyboard.focused,true);
assert.equal(v.elements['keyboard-shortcuts'].hidden,false);
v.handlers.click({target:{closest:()=>({dataset:{action:'modifier',key:'ctrl'}})}});
v.handlers.click({target:{closest:()=>({dataset:{action:'key',key:'c'}})}});
assert.equal(v.sent.at(-1).type,'lazyboy-mobile-key');
assert.equal(v.sent.at(-1).key,'ctrl+c');
v.handlers.click({target:{closest:()=>({dataset:{action:'hide-keyboard'}})}});
assert.equal(v.keyboard.focused,false);
assert.equal(v.elements['keyboard-shortcuts'].hidden,true);
});
test('status and HUD sit outside the remote pixels; screenshots stay opt-in',()=>{
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
assert.match(app,/\{!computerOpen && <RunStatus runId=\{liveRunId\}\/>\}/);
assert.match(app,/<RunStatus runId=\{liveRunId\}\/>\{hud\}/);
assert.match(app,/\{!computerOpen&&hud\}<div className="preview">/);
assert.match(app,/--visible-height/);
const css=fs.readFileSync('apps/web/src/computer.css','utf8');
assert.match(css,/\.computer-overlay>\.run-status/);
assert.match(css,/\.computer-hud\{position:static/);
const tools=fs.readFileSync('crates/api/src/tools.rs','utf8');
assert.match(tools,/screenshots are opt-in with observe:true/);
assert.match(tools,/if args\.get\("observe"\)\.and_then\(Value::as_bool\) != Some\(true\)/);
const runs=fs.readFileSync('crates/api/src/runs.rs','utf8');
assert.match(runs,/Never silently stop/);
assert.match(runs,/let goal_mode = goal_mode \|\| !chat_only/);
});
const monitorJs=ts.transpileModule(fs.readFileSync('apps/web/src/run-monitor.tsx','utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX}}).outputText;
const monitorBox={exports:{},require:name=>{
@ -309,6 +358,7 @@ test('trail lines read as sentences with the detail a stuck run needs',()=>{
assert.match(trailText({id:4,kind:'run',createdAt:'',event:'started',task:'整理下載資料'}),/整理下載資料/);
assert.match(trailText({id:5,kind:'run',createdAt:'',event:'completed',turns:9}),/Done in 9 turns/);
assert.match(trailText({id:6,kind:'run',createdAt:'',event:'waiting_input',reason:'登入'}),/Waiting for you: 登入/);
assert.match(trailText({id:10,kind:'run',createdAt:'',event:'paused',reason:'已達本輪執行上限'}),/Paused: 已達本輪執行上限/);
assert.match(trailText({id:7,kind:'run',createdAt:'',event:'retry'}),/Re-queued/);
assert.match(trailText({id:8,kind:'retry',createdAt:'',attempt:2,gaveUp:true,error:'429 rate limit'}),/attempt 2 failed, retrying · gave up — 429 rate limit/);
assert.match(trailText({id:9,kind:'notice',createdAt:'',text:'這輪不需要電腦'}),/這輪不需要電腦/);
@ -326,5 +376,7 @@ test('the bubble reads the run activity endpoint the API actually mounts',()=>{
assert.match(app,/<RunProbe runId=\{member\.id===computer\.botId\?computer\.busyRunId:null\}><Avatar/);
const probe=fs.readFileSync('apps/web/src/run-monitor.tsx','utf8');
assert.match(probe,/`\/api\/runs\/\$\{runId\}\/activity\$\{after\}`/);
assert.match(probe,/export function RunStatus/);
assert.match(probe,/`\/api\/runs\/\$\{runId\}\/activity\?limit=30`/);
assert.match(fs.readFileSync('apps/web/src/main.tsx','utf8'),/import "\.\/monitor\.css";/);
});

View File

@ -75,6 +75,8 @@ for fragment in [
'SELECT id, kind, payload, created_at FROM run_activity WHERE run_id=$1 AND ($2::bigint IS NULL OR id>$2) ORDER BY id DESC LIMIT $3',
"(checkpoint->>'turn')::bigint AS turn",
"(checkpoint->>'turnLimit')::bigint AS turn_limit",
"(checkpoint->>'stepAt')::timestamptz AS step_at",
'"action": failure.action',
"AND status IN ('failed','cancelled')",
'a.bot_id=runs.bot_id AND a.id<>runs.id',
"a.status IN ('leased','running','waiting_input','waiting_takeover')",
@ -90,6 +92,9 @@ for fragment in [
"'step', $2::text, 'stepAt', now(), 'turn', $3::bigint, 'turnLimit', $4::bigint",
'"event": "started"',
'"event": "failed"',
'"event": "paused"',
'"reason": draft',
'Never silently stop',
'"status": tool_status(',
'"elapsedMs": model_elapsed',
]: