feat/test #2
|
|
@ -1,5 +1,5 @@
|
|||
import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Square, Users, X } from "./animated-icons";
|
||||
import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons";
|
||||
import UseAnimations from "react-useanimations";
|
||||
import loading from "react-useanimations/lib/loading";
|
||||
import loading2 from "react-useanimations/lib/loading2";
|
||||
|
|
@ -18,9 +18,9 @@ import searchToX from "react-useanimations/lib/searchToX";
|
|||
import { api, ApiError } from "./api";
|
||||
import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar";
|
||||
import { t, type MessageKey } from "./i18n";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Room, RoomMember, Session, WorkspaceSettings } from "./types";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types";
|
||||
|
||||
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,display:null,profileMode:"per-bot",screenAvailable:false};
|
||||
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false};
|
||||
const SESSION_STORE="lazyboy.sessionByBot";
|
||||
const PANE_STORE="lazyboy.rightPane";
|
||||
const WORKSPACE_STORE="lazyboy.workspace";
|
||||
|
|
@ -67,6 +67,7 @@ export function App(){
|
|||
const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null);
|
||||
const [roomToDelete,setRoomToDelete]=useState<Room|null>(null); const [mcpServers,setMcpServers]=useState<McpServer[]>([]);
|
||||
const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState<AccountDialog>(null);
|
||||
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false);
|
||||
const [workspaceName,setWorkspaceName]=useState(workspaceStart.name);
|
||||
const [looks,setLooks]=useState(readAvatarLooks);
|
||||
const sendingRef=useRef(false); const refreshSeqRef=useRef(0);
|
||||
|
|
@ -81,6 +82,15 @@ export function App(){
|
|||
const paneBot=bots.find(bot=>bot.id===paneBotId)||active;
|
||||
const workingMembers=activeRoom?busyMembers:active&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[];
|
||||
const lastMessageId=messages[messages.length-1]?.id||"";
|
||||
// The bot is parked in waiting_takeover: nothing moves (including queued
|
||||
// messages) until the human releases the screen, so say so loudly.
|
||||
const pausedForUser=computer.takeoverRequested&&workingMembers.length===0&&(!computer.waitingSessionId||computer.waitingSessionId===activeSessionId||Boolean(activeRoom));
|
||||
// Teaching by demonstration: one recording at a time per bot, then a draft
|
||||
// playbook the human names and saves before it becomes a real skill.
|
||||
const teaching=skills.find(skill=>skill.status==="recording")||null;
|
||||
const drafting=skills.find(skill=>skill.status==="drafting")||null;
|
||||
const skillDraft=teaching||drafting?null:skills.find(skill=>skill.status==="draft")||null;
|
||||
const savedSkills=skills.filter(skill=>skill.status==="saved");
|
||||
|
||||
const loadMcp=useCallback(async()=>{setMcpServers(await api<McpServer[]>("/api/mcp-servers").catch(()=>[] as McpServer[]))},[]);
|
||||
const loadBots=useCallback(async()=>{const [next,nextRooms]=await Promise.all([api<Bot[]>("/api/bots"),api<Room[]>("/api/rooms").catch(()=>[] as Room[])]);setBots(next);setRooms(nextRooms);setActiveRoomId(id=>id&&nextRooms.some(room=>room.id===id)?id:null);setActiveId(id=>id&&next.some(b=>b.id===id)?id:null);await loadMcp()},[loadMcp]);
|
||||
|
|
@ -93,22 +103,23 @@ export function App(){
|
|||
try{const roomStatus=await api<{busy:RoomMember[]}>(`/api/rooms/${activeRoomId}/status`);if(refreshSeq!==refreshSeqRef.current)return;nextBusy=roomStatus.busy;computerBot=roomStatus.busy[0]?.id||roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null}catch{if(refreshSeq!==refreshSeqRef.current)return;computerBot=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null}
|
||||
}
|
||||
const messagesJob=api<Message[]>(`/api/sessions/${activeSessionId}/messages`);
|
||||
if(!computerBot){const nextMessages=await messagesJob;if(refreshSeq===refreshSeqRef.current){setBusyMembers(nextBusy);setMessages(nextMessages);setComputer(blankComputer);setScreenUrl(null)}return}
|
||||
if(!computerBot){const nextMessages=await messagesJob;if(refreshSeq===refreshSeqRef.current){setBusyMembers(nextBusy);setMessages(nextMessages);setComputer(blankComputer);setSkills([]);setScreenUrl(null)}return}
|
||||
const skillsJob=activeRoomId?Promise.resolve([] as TaughtSkill[]):api<TaughtSkill[]>(`/api/bots/${computerBot}/skills`).catch(()=>[] as TaughtSkill[]);
|
||||
const status=await api<ComputerStatus>(`/api/computer/${computerBot}/status`);if(refreshSeq!==refreshSeqRef.current)return;
|
||||
const [nextMessages,screen]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null})]);
|
||||
const [nextMessages,screen,nextSkills]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null}),skillsJob]);
|
||||
if(refreshSeq!==refreshSeqRef.current)return;
|
||||
setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setScreenUrl(status.botId===computerBot?screen.url:null)
|
||||
setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null)
|
||||
},[activeId,activeRoomId,activeSessionId]);
|
||||
useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e.message)})},[loadBots]);
|
||||
useEffect(()=>{if(activeId||activeRoomId||bots.length===0)return;setActiveId(bots[0].id)},[bots,activeId,activeRoomId]);
|
||||
useEffect(()=>{setMessages([]);loadSessions().catch(e=>setError(e.message))},[loadSessions]);
|
||||
useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current=""},[activeSessionId]);
|
||||
useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length]);
|
||||
useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]);
|
||||
useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(e.message));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]);
|
||||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
|
||||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]);
|
||||
useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]);
|
||||
useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]);
|
||||
useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]);
|
||||
useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]);
|
||||
useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]);
|
||||
useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]);
|
||||
useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]);
|
||||
|
|
@ -120,6 +131,13 @@ export function App(){
|
|||
|
||||
async function action(work:()=>Promise<unknown>){setBusy(true);setError(null);try{await work();await refresh()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
|
||||
async function stopChat(){if(activeSessionId)await api(`/api/sessions/${activeSessionId}/stop`,{method:"POST",body:"{}"});else if(active)await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});}
|
||||
async function startTeaching(goal:string){if(!active)return;setTeachOpen(false);await action(async()=>{await api(`/api/bots/${active.id}/skills/start`,{method:"POST",body:JSON.stringify({goal})});setComputerOpen(true)})}
|
||||
async function stopTeaching(){if(!active)return;await action(()=>api(`/api/bots/${active.id}/skills/stop`,{method:"POST",body:"{}"}));setComputerOpen(false)}
|
||||
async function cancelTeaching(){if(!active)return;await action(()=>api(`/api/bots/${active.id}/skills/cancel`,{method:"POST",body:"{}"}));setComputerOpen(false)}
|
||||
async function saveSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}))}
|
||||
async function testSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(async()=>{await api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook})});await api(`/api/skills/${skill.id}/test`,{method:"POST",body:"{}"})})}
|
||||
async function discardSkill(skill:TaughtSkill){await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}))}
|
||||
function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}執行「${skill.name}」`);document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus()}
|
||||
async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();if((!active&&!activeRoom)||!activeSessionId||!text)return;sendingRef.current=true;setDraft("");historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});sentHistoryRef.current.push(text);if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();await loadSessions()})}finally{sendingRef.current=false}}
|
||||
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
|
||||
function selectSession(id:string){setActiveSessionId(id);setSessionMenuOpen(false);if(sessionStoreKey)writeSessionStore(sessionStoreKey,id)}
|
||||
|
|
@ -184,10 +202,10 @@ export function App(){
|
|||
|
||||
<main className="chat-panel">
|
||||
<header className="topbar"><button className="icon-button mobile-menu" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="#dfdfe2"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}<span className="message-body">{message.body}</span>{message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{workingMembers.map(member=><div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-label">{t("working",{name:member.name})}</span></div>)}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}<span className="message-body">{message.body}</span>{message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{workingMembers.map(member=><div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-label">{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}</span></div>)}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
|
||||
{otherSessionBusy&&<div className="queue-hint">{t("anotherConversationQueued")}</div>}
|
||||
<form className="composer" onSubmit={send}><button type="button" className="composer-plus" disabled title={t("attachmentsUnavailable")} aria-label={t("attachmentsUnavailable")}><Plus/></button><textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} placeholder={activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||!draft.trim()||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
<form className="composer" onSubmit={send}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled title={t("attachmentsUnavailable")}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button>{savedSkills.length>0&&<><hr/><small className="plus-menu-label">{t("taughtSkills")}</small>{savedSkills.map(skill=><button type="button" role="menuitem" key={skill.id} title={skill.playbook.whenToUse||skill.goal} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button>)}</>}</div>}</div><textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||!draft.trim()||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
</main>
|
||||
|
||||
{!rightCollapsed&&<div className="side-card-backdrop" onClick={()=>setRightCollapsed(true)}/>}
|
||||
|
|
@ -201,7 +219,7 @@ export function App(){
|
|||
<div className={`side-part computer-part ${rightPart==="computer"?"":"hidden-part"}`}>
|
||||
<div className="computer-status-row">{paneBot?<span>{t("botComputer",{name:paneBot.name})}</span>:<span>{t("computer")}</span>}{computer.state==="booting"?<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/>:<i className={`state-dot ${computer.state}`}/>}<small>{stateLabel(computer.state)}</small></div>
|
||||
<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}</div>
|
||||
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div><ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/></>}
|
||||
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}</>}
|
||||
</div>
|
||||
{rightPart==="memory"&&paneBot&&<MemoryPane bot={paneBot} changed={loadBots}/>}
|
||||
{rightPart==="plugins"&&<McpPane servers={mcpServers} reload={loadMcp}/>}
|
||||
|
|
@ -210,7 +228,8 @@ export function App(){
|
|||
</>
|
||||
</aside>}
|
||||
|
||||
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className="control-badge">{computer.controlHolder==="user"?t("userControlling"):computer.busyBotName?t("aiReadOnly"):t("readOnly")}</span></div><div><ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/><button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" title={t("moreActions")}><Ellipsis/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen">{frame}</div>{error&&<div className="overlay-error">{error}</div>}</div>}
|
||||
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.busyBotName?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" title={t("moreActions")}><Ellipsis/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen">{frame}</div>{error&&<div className="overlay-error">{error}</div>}</div>}
|
||||
{teachOpen&&active&&<TeachDialog bot={active} busy={busy} close={()=>setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>}
|
||||
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{document.querySelectorAll<HTMLIFrameElement>(".desktop-frame").forEach(frame=>frame.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin));setClipboardOpen(false)}}/>}
|
||||
|
||||
{createOpen&&<CreateDialog close={()=>setCreateOpen(false)} created={async bot=>{setCreateOpen(false);await loadBots();setActiveId(bot.id)}}/>}
|
||||
|
|
@ -422,6 +441,35 @@ function McpCustomForm({busy,error,onBack,onSubmit}:{busy:boolean;error:string;o
|
|||
function EmptyComputer({state}:{state:ComputerStatus["state"]}){const loading=state==="booting";return <div className="empty-computer">{loading?<UseAnimations animation={loading2} size={38} wrapperStyle={{display:"block"}}/>:<Computer/>}<strong>{stateLabel(state)}</strong><span>{loading?t("preparingDesktop"):t("computerPreviewHint")}</span></div>}
|
||||
function ControlButtons({computer,busy,action,active,sessionId,onBoot,onRestart}:{computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;active:Bot;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const working=Boolean(computer.busyBotName);const restart=<button className="outline restart-computer" disabled={busy} title={t("restartDocker")} onClick={()=>void action(onRestart||(()=>api(`/api/computer/${active.id}/restart`,{method:"POST",body:"{}"})))}><RefreshCw/> {t("restartDocker")}</button>;if(computer.state!=="running")return <div className="computer-actions"><button className="primary" disabled={busy||computer.state==="booting"} onClick={()=>void action(onBoot||(()=>api(`/api/computer/${active.id}/boot`,{method:"POST",body:"{}"})))}>{(busy||computer.state==="booting")&&<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle",marginRight:7}}/>}{computer.state==="booting"?t("bootingProgress"):t("openComputer")}</button>{(computer.state==="booting"||computer.state==="error")&&restart}</div>;if(working)return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button><button className="outline" disabled={busy} onClick={()=>action(async()=>{if(sessionId)await api(`/api/sessions/${sessionId}/stop`,{method:"POST",body:"{}"});else await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});})}><Square/>{t("stopTask")}</button>{restart}</div>;if(computer.controlHolder==="user")return <div className="computer-actions"><button className="outline" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseControl")}</button>{restart}</div>;return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeControl")}</button>{restart}</div>}
|
||||
function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const interactive=props.computer.controlHolder==="user";return <div className="control-bar"><ControlButtons {...props}/><button className="icon-button" disabled={!interactive} onClick={props.paste}><ClipboardPaste/></button><button className="icon-button" disabled={!interactive} onClick={props.copy}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button></div>}
|
||||
function TeachDialog({bot,busy,close,start}:{bot:Bot;busy:boolean;close:()=>void;start:(goal:string)=>void}){
|
||||
const [goal,setGoal]=useState("");
|
||||
return <div className="modal-backdrop" onClick={close}><form className="dialog teach-dialog" onClick={e=>e.stopPropagation()} onSubmit={e=>{e.preventDefault();if(goal.trim())start(goal.trim())}}>
|
||||
<div className="dialog-title"><h2><i className="record-dot"/>{t("teachTask")}</h2><button type="button" onClick={close}><X/></button></div>
|
||||
<p>{t("teachDialogLead",{name:bot.name})}</p>
|
||||
<label>{t("teachGoalLabel")}<textarea className="teach-goal" autoFocus rows={3} value={goal} onChange={e=>setGoal(e.target.value)} placeholder={t("teachGoalPlaceholder")} onKeyDown={e=>{if(e.key==="Enter"&&!e.shiftKey&&!e.nativeEvent.isComposing){e.preventDefault();e.currentTarget.form?.requestSubmit()}}}/></label>
|
||||
<ul className="teach-tips"><li>{t("teachTip1")}</li><li>{t("teachTip2")}</li><li>{t("teachTip3")}</li></ul>
|
||||
<div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!goal.trim()}>{t("startDemo")}</button></div>
|
||||
</form></div>;
|
||||
}
|
||||
function stepText(step:unknown):string{if(typeof step==="string")return step;if(step&&typeof step==="object"&&"do" in step)return String((step as {do:unknown}).do||"");return ""}
|
||||
function SkillDraftCard({skill,busy,onSave,onTest,onDiscard}:{skill:TaughtSkill;busy:boolean;onSave:(name:string,playbook:Playbook)=>void;onTest:(name:string,playbook:Playbook)=>void;onDiscard:()=>void}){
|
||||
const [name,setName]=useState(skill.name||skill.playbook.name||"");
|
||||
const [stepsText,setStepsText]=useState((skill.playbook.steps||[]).map(step=>stepText(step)).join("\n"));
|
||||
const [expanded,setExpanded]=useState(false);
|
||||
const steps=skill.playbook.steps||[];
|
||||
const build=():Playbook=>{const lines=stepsText.split("\n").map(line=>line.replace(/^\s*\d+[.、)]\s*/,"").trim()).filter(Boolean);const next=lines.map((line,index)=>{const prev=steps[index];return typeof prev==="object"&&prev&&stepText(prev)===line?prev:{do:line,expect:typeof prev==="object"&&prev&&stepText(prev)===line?prev.expect:"",note:""}});return {...skill.playbook,name:name.trim(),steps:next}};
|
||||
const inputs=(skill.playbook.inputs||[]).map(input=>input.name).filter(Boolean);
|
||||
return <section className="skill-draft" aria-label={t("skillDraftTitle")}>
|
||||
<header><Sparkle/><strong>{t("skillDraftTitle")}</strong><small>{t("skillDraftFrom",{count:skill.eventCount})}</small></header>
|
||||
<label className="skill-name">{t("skillName")}<input value={name} maxLength={40} onChange={e=>setName(e.target.value)} placeholder={t("skillNamePlaceholder")}/></label>
|
||||
{skill.playbook.intent&&<p className="skill-intent">{skill.playbook.intent}</p>}
|
||||
{inputs.length>0&&<p className="skill-inputs">{t("skillInputs")}：{inputs.map(input=><code key={input}>{input}</code>)}</p>}
|
||||
<label className="skill-steps">{t("skillSteps")}{expanded?<textarea rows={Math.min(12,Math.max(4,stepsText.split("\n").length+1))} value={stepsText} onChange={e=>setStepsText(e.target.value)}/>:<ol onClick={()=>setExpanded(true)}>{stepsText.split("\n").filter(Boolean).slice(0,6).map((line,index)=><li key={index}>{line}</li>)}{stepsText.split("\n").filter(Boolean).length>6&&<li className="more">{t("moreSteps",{count:stepsText.split("\n").filter(Boolean).length-6})}</li>}</ol>}{!expanded&&<button type="button" className="link" onClick={()=>setExpanded(true)}>{t("editSteps")}</button>}</label>
|
||||
{skill.playbook.howToCheck&&<p className="skill-check">{t("skillCheck")}：{skill.playbook.howToCheck}</p>}
|
||||
{skill.error&&<p className="skill-error">{t("skillDistillFailed")}</p>}
|
||||
<div className="skill-actions"><button type="button" className="outline danger-ghost" disabled={busy} onClick={onDiscard}>{t("discard")}</button><span className="grow"/><button type="button" className="outline" disabled={busy||!name.trim()} onClick={()=>onTest(name.trim(),build())}>{t("testRun")}</button><button type="button" className="primary" disabled={busy||!name.trim()} onClick={()=>onSave(name.trim(),build())}>{t("saveSkill")}</button></div>
|
||||
</section>;
|
||||
}
|
||||
function ClipboardDialog({close,paste}:{close:()=>void;paste:(text:string)=>void}){const[text,setText]=useState("");return <div className="modal-backdrop"><div className="dialog compact"><div className="dialog-title"><h2>{t("pasteToRemoteComputer")}</h2><button onClick={close}><X/></button></div><p>{t("pasteRemoteHelp")}</p><textarea className="clipboard-text" autoFocus value={text} onChange={e=>setText(e.target.value)} placeholder={t("pasteTextPlaceholder")}/><div className="dialog-actions"><button className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={!text} onClick={()=>paste(text)}>{t("pasteIntoVnc")}</button></div></div></div>}
|
||||
function CreateDialog({close,created}:{close:()=>void;created:(bot:Bot)=>void}){const[name,setName]=useState("");const[mode,setMode]=useState<ComputerMode>("team");const[busy,setBusy]=useState(false);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();if(!name.trim())return;setBusy(true);try{created(await api<Bot>("/api/bots",{method:"POST",body:JSON.stringify({name:name.trim(),computerMode:mode})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addBot")}</h2><button type="button" onClick={close}><X/></button></div><label>{t("name")}<input autoFocus value={name} onChange={e=>setName(e.target.value)} placeholder={t("botNamePlaceholder")}/></label><div className="mode-grid"><button type="button" className={mode==="team"?"picked":""} onClick={()=>setMode("team")}><BotIcon/><strong>{t("sharedComputer")}</strong><small>{t("sharedComputerHint")}</small></button><button type="button" className={mode==="dedicated"?"picked":""} onClick={()=>setMode("dedicated")}><Computer/><strong>{t("privateComputer")}</strong><small>{t("privateComputerHint")}</small></button></div><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()}>{t("create")}</button></div></form></div>}
|
||||
function CreateGroupDialog({bots,close,created}:{bots:Bot[];close:()=>void;created:(room:Room)=>void}){const[name,setName]=useState("");const[selected,setSelected]=useState<string[]>([]);const[busy,setBusy]=useState(false);const visible=bots.filter(bot=>!bot.hidden);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();const groupName=name.trim();if(!groupName||selected.length<2)return;setBusy(true);try{created(await api<Room>("/api/rooms",{method:"POST",body:JSON.stringify({name:groupName,memberIds:selected})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addGroup")}</h2><button type="button" onClick={close}><X/></button></div><p className="dialog-lead">{t("groupDescription")}</p><label>{t("groupName")}<input autoFocus value={name} maxLength={30} onChange={e=>setName(e.target.value)} placeholder={t("groupNamePlaceholder")}/></label><fieldset className="group-picker"><legend>{t("chooseBots")}</legend>{visible.map(bot=><label key={bot.id}><input type="checkbox" checked={selected.includes(bot.id)} onChange={()=>setSelected(ids=>ids.includes(bot.id)?ids.filter(id=>id!==bot.id):[...ids,bot.id])}/><Avatar lookId={bot.id} name={bot.name} color={bot.avatarColor} shape={bot.avatarShape} online/><span>{bot.name}</span></label>)}</fieldset><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()||selected.length<2}>{busy?t("creating"):t("createGroup")}</button></div></form></div>}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import UseAnimations from "react-useanimations";
|
||||
import type { Animation } from "react-useanimations/utils";
|
||||
import activity from "react-useanimations/lib/activity";
|
||||
import archive from "react-useanimations/lib/archive";
|
||||
import star from "react-useanimations/lib/star";
|
||||
import airplay from "react-useanimations/lib/airplay";
|
||||
import arrowDown from "react-useanimations/lib/arrowDown";
|
||||
import arrowRightCircle from "react-useanimations/lib/arrowRightCircle";
|
||||
|
|
@ -45,7 +47,9 @@ export const Ellipsis=animatedIcon(menu3,18);
|
|||
export const Info=animatedIcon(info,16);
|
||||
export const LogOut=animatedIcon(arrowRightCircle,16);
|
||||
export const Megaphone=animatedIcon(notification,16);
|
||||
export const Paperclip=animatedIcon(archive,16);
|
||||
export const Pin=animatedIcon(pocket,14);
|
||||
export const Sparkle=animatedIcon(star,16);
|
||||
export const Plug=animatedIcon(toggle,16);
|
||||
export const Plus=animatedIcon(plusToX,17);
|
||||
export const RefreshCw=animatedIcon(skipBack,14);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,21 @@ export const zhTW = {
|
|||
openOnPhone: "在手機開啟", settings: "設定", about: "關於", helpCenter: "說明中心", sendFeedback: "傳送意見回饋", logout: "登出", workspaceMenu: "工作區選單",
|
||||
chooseBot: "選擇一個機器人", startRoomDiscussion: "和 {name} 開始討論", startBotWork: "和 {name} 開始工作", roomWillReply: "{names} 會一起回覆。",
|
||||
botWelcome: "傳送訊息，讓它在自己的電腦上完成任務。", remembered: "已記住", remember: "記住", working: "{name} 正在工作…",
|
||||
anotherConversationQueued: "另一則對話正在執行，這則會排隊。", attachmentsUnavailable: "附件功能尚未開放", messageTo: "傳訊息給 {name}", chooseConversationFirst: "先選擇對話", stopConversation: "停止對話",
|
||||
anotherConversationQueued: "另一則對話正在執行，這則會排隊。",
|
||||
workingStep: "{name} 正在工作… {step}", queuedMessages: "還有 {count} 則訊息排隊中",
|
||||
pausedUserControl: "你正在操控畫面，{name} 已暫停。完成後按「釋放控制」，它會從目前畫面接著做（排隊中的訊息也會一起處理）。",
|
||||
pausedNeedsUser: "{name} 需要你接手畫面（例如登入或驗證）。按「接手操作」處理，完成後再「釋放控制」。",
|
||||
releaseAndContinue: "釋放控制並繼續", attachmentsUnavailable: "附件功能尚未開放", attachFile: "附加檔案",
|
||||
teachTask: "教它一項任務", teachTaskHint: "你示範一次，它學成技能", teachNeedsBot: "先選擇一個機器人", taughtSkills: "已學會的技能",
|
||||
teachDialogLead: "接下來畫面交給你操作，{name} 會在旁邊看：記錄你點了哪些控制項、輸入了什麼、去了哪些頁面，之後整理成一個「知道目的與流程」的技能，而不是死記座標。",
|
||||
teachGoalLabel: "你要示範什麼？（一句話說目標）", teachGoalPlaceholder: "例如：到 STAR 訓練系統，把指定課程的影片看完並完成測驗",
|
||||
teachTip1: "用平常的方式操作即可，不用刻意放慢；多餘的點擊會被忽略。", teachTip2: "會變動的值（搜尋字、名稱、日期）之後可以當作參數，做的時候先用實際例子。", teachTip3: "密碼欄位不會被記錄；遇到登入請先登好再開始示範。",
|
||||
startDemo: "開始示範", finishDemo: "完成示範", teachingBadge: "示範中", teachingComposerHint: "示範中：做完按「完成示範」再送訊息",
|
||||
teachingLive: "正在學：{goal}", teachingHint: "畫面已交給你，直接在右側（或放大）操作；做完按「完成示範」。", teachingCaptured: "已記錄 {count} 個動作",
|
||||
distilling: "正在把「{goal}」的示範整理成技能…",
|
||||
skillDraftTitle: "新技能草稿", skillDraftFrom: "從 {count} 個動作整理而來", skillName: "技能名稱", skillNamePlaceholder: "例如：完成 STAR 訓練課程",
|
||||
skillInputs: "可變輸入", skillSteps: "步驟（依意圖描述，可修改）", moreSteps: "…還有 {count} 步", editSteps: "編輯步驟", skillCheck: "怎麼確認完成",
|
||||
skillDistillFailed: "模型整理失敗，這是依動作直接列出的版本，建議修改後再儲存。", discard: "捨棄", testRun: "試跑", saveSkill: "儲存技能", messageTo: "傳訊息給 {name}", chooseConversationFirst: "先選擇對話", stopConversation: "停止對話",
|
||||
collapseSidebar: "收合側欄", botComputer: "{name} 的電腦", dedicatedScreen: "獨立螢幕", enlarge: "放大",
|
||||
userControlling: "你正在控制", aiReadOnly: "AI 操作中（唯讀）", readOnly: "唯讀", pasteClipboard: "貼上剪貼簿", copyDesktopClipboard: "複製桌面剪貼簿", moreActions: "更多操作",
|
||||
unpin: "取消釘選", pin: "釘選", markUnread: "標示為未讀", enterGroupName: "輸入分組名稱", changeGroup: "變更分組", createOrMoveGroup: "建立／移入分組", removeFromGroup: "移出分組", unhide: "取消隱藏", hide: "隱藏", delete: "刪除",
|
||||
|
|
|
|||
|
|
@ -130,6 +130,9 @@
|
|||
.session-item .icon-button{flex:0 0 32px;width:32px;height:32px;color:var(--muted)}
|
||||
.session-menu .danger-item{color:#ff7777}
|
||||
.queue-hint{position:absolute;bottom:92px;left:50%;transform:translateX(-50%);color:var(--muted);font-size:12px}
|
||||
.pause-banner{display:flex;align-items:center;gap:14px;width:min(760px,100%);margin:4px auto 18px;padding:12px 14px;border:1px solid #5a4a1f;border-radius:12px;background:#221c0e;color:#fcd68a;font-size:13px;line-height:1.5}
|
||||
.pause-banner span{flex:1}
|
||||
.pause-banner .primary{flex:0 0 auto;white-space:nowrap}
|
||||
.message{position:relative;padding-right:28px}
|
||||
.remember-msg{position:absolute;top:8px;right:4px;width:26px;height:26px;display:grid;place-items:center;border:0;border-radius:8px;background:transparent;color:var(--muted);cursor:pointer}
|
||||
.messages .remember-msg{display:none}
|
||||
|
|
@ -262,3 +265,43 @@
|
|||
.provider-grid button.picked{border-color:var(--accent);color:var(--ink);background:rgba(62,197,168,.08)}
|
||||
.dialog.settings-dialog{width:min(520px,calc(100vw - 28px));max-height:min(850px,calc(100dvh - 28px));overflow:auto}
|
||||
.dialog.settings-dialog select{width:100%;height:42px;padding:0 12px;border:1px solid var(--border);border-radius:10px;background:#0c0c0d;color:var(--ink)}
|
||||
/* Teach-by-demonstration: composer menu, live banner, draft card */
|
||||
.plus-menu-wrap{position:relative;align-self:center}
|
||||
.composer-plus.open{color:var(--ink);background:rgba(255,255,255,.08)}
|
||||
.plus-menu{position:absolute;z-index:40;bottom:50px;left:0;display:grid;width:240px;padding:6px;border:1px solid var(--border);border-radius:13px;background:#18181b;box-shadow:0 18px 60px rgba(0,0,0,.55)}
|
||||
.plus-menu button{display:flex;align-items:center;gap:10px;width:100%;height:38px;padding:0 10px;border:0;border-radius:8px;background:transparent;color:var(--ink);font:inherit;text-align:left;cursor:pointer;overflow:hidden;white-space:nowrap;text-overflow:ellipsis}
|
||||
.plus-menu button:hover:not(:disabled){background:rgba(255,255,255,.07)}
|
||||
.plus-menu button:disabled{opacity:.4;cursor:not-allowed}
|
||||
.plus-menu svg{width:16px;height:16px;flex:0 0 16px}
|
||||
.plus-menu hr{width:100%;margin:5px 0;border:0;border-top:1px solid var(--border)}
|
||||
.plus-menu-label{padding:4px 10px 2px;color:var(--faint);font-size:11px}
|
||||
.record-dot{display:inline-block;flex:0 0 12px;width:12px;height:12px;border-radius:50%;background:#ef5555;box-shadow:inset 0 0 0 2px #18181b,0 0 0 1.5px #ef5555}
|
||||
.record-dot.live{animation:pulse 1.2s infinite;margin-right:8px;vertical-align:-1px}
|
||||
.teach-dialog h2{display:flex;align-items:center;gap:10px}
|
||||
.teach-goal{min-height:78px;resize:vertical;border:1px solid var(--border);border-radius:10px;background:#0c0c0d;color:var(--ink);padding:10px 12px;outline:0;line-height:1.5}
|
||||
.teach-goal:focus{border-color:#4b4b50}
|
||||
.teach-tips{margin:0;padding-left:18px;color:var(--muted);font-size:12.5px;line-height:1.6}
|
||||
.teach-banner{display:flex;align-items:center;gap:12px;width:min(760px,100%);margin:4px auto 18px;padding:12px 14px;border:1px solid #3d2a2a;border-radius:12px;background:#1d1212;color:#f3c5c5;font-size:13px;line-height:1.5}
|
||||
.teach-banner.recording{border-color:#6a2a2a;background:#241313}
|
||||
.teach-banner>span{flex:1;display:grid;gap:2px}
|
||||
.teach-banner small{color:var(--muted)}
|
||||
.teach-banner .primary,.teach-banner .outline{flex:0 0 auto;white-space:nowrap}
|
||||
.control-badge.teaching{background:rgba(239,85,85,.16);color:#ff8a8a}
|
||||
.skill-draft{display:grid;gap:12px;width:min(760px,100%);margin:4px auto 18px;padding:16px 18px;border:1px solid #2f4a42;border-radius:14px;background:#0f1815;font-size:13.5px;line-height:1.55}
|
||||
.skill-draft header{display:flex;align-items:center;gap:9px;color:var(--accent)}
|
||||
.skill-draft header svg{width:16px;height:16px}
|
||||
.skill-draft header small{margin-left:auto;color:var(--muted);font-weight:400}
|
||||
.skill-draft label{display:grid;gap:6px;color:var(--muted);font-size:12.5px}
|
||||
.skill-draft input,.skill-draft textarea{border:1px solid var(--border);border-radius:10px;background:#0c0c0d;color:var(--ink);padding:9px 12px;outline:0;font:inherit;line-height:1.5}
|
||||
.skill-draft input:focus,.skill-draft textarea:focus{border-color:#4b4b50}
|
||||
.skill-draft textarea{resize:vertical}
|
||||
.skill-draft p{margin:0;color:var(--ink)}
|
||||
.skill-intent{color:#d8d8dc}
|
||||
.skill-inputs code{margin-right:6px;padding:1px 7px;border-radius:6px;background:rgba(62,197,168,.14);color:var(--accent);font-size:12px}
|
||||
.skill-steps ol{margin:0;padding-left:22px;color:var(--ink);cursor:text}
|
||||
.skill-steps li{margin:2px 0}
|
||||
.skill-steps li.more,.skill-check{color:var(--muted)}
|
||||
.skill-draft .link{width:fit-content;padding:0;border:0;background:none;color:var(--accent);font-size:12px;cursor:pointer}
|
||||
.skill-error{color:#fca5a5;font-size:12.5px}
|
||||
.skill-actions{display:flex;align-items:center;gap:8px}
|
||||
.skill-actions .grow{flex:1}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,12 @@ export interface Session { id:string; botId:string; title:string; status:"active
|
|||
export interface Message { id:string; sessionId?:string; seq?:number; role:string; body:string; blocks?:unknown[]; runId?:string|null; clientNonce?:string|null; createdAt:string; speakerBotId?:string|null; speakerName?:string|null; speakerColor?:string|null; speakerShape?:AvatarShape|null }
|
||||
export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape }
|
||||
export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number }
|
||||
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; display:string|null; profileMode:string; screenAvailable:boolean }
|
||||
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
|
||||
export interface PlaybookStep { do:string; expect?:string; note?:string }
|
||||
export interface PlaybookInput { name:string; description?:string; example?:string }
|
||||
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
|
||||
export type TaughtSkillStatus = "recording"|"drafting"|"draft"|"saved"|"failed"|"cancelled";
|
||||
export interface TaughtSkill { id:string; botId:string; threadId:string|null; name:string; goal:string; status:TaughtSkillStatus; playbook:Playbook; error:string|null; startedAt:string|null; expiresAt:string|null; stoppedAt:string|null; createdAt:string; updatedAt:string; eventCount:number; frameCount:number }
|
||||
export interface MemoryItem { id:string; sessionId:string|null; sourceRunId:string|null; sourceMessageId:string|null; content:string; importance:number; revision:number; createdAt:string; updatedAt:string }
|
||||
export type McpTransport = "stdio" | "http" | "sse";
|
||||
export interface McpTool { name:string; exposedName:string; description:string }
|
||||
|
|
|
|||
|
|
@ -54,6 +54,10 @@ pub fn status_from(
|
|||
busy_bot_name,
|
||||
busy_session_id: None,
|
||||
busy_run_id: None,
|
||||
busy_step: None,
|
||||
waiting_run_id: None,
|
||||
waiting_session_id: None,
|
||||
queued_runs: 0,
|
||||
multi_screen: true,
|
||||
screen_id: screen.map(|row| row.id.clone()),
|
||||
display: screen.map(|row| row.display.clone()),
|
||||
|
|
@ -1024,26 +1028,64 @@ pub async fn current_status(
|
|||
.get_screen(&computer.id, bot_id)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let active = state.db.active_run(bot_id).await.ok().flatten();
|
||||
let run_status = active
|
||||
.as_ref()
|
||||
.and_then(|(_, run_status, _)| parse_run_status(run_status));
|
||||
let waiting_for_takeover = run_status == Some(lazyboy_contracts::RunStatus::WaitingTakeover);
|
||||
let busy = run_status.filter(|status| {
|
||||
status.is_active() && *status != lazyboy_contracts::RunStatus::WaitingTakeover
|
||||
// Newest-first is wrong here: a message queued behind a paused run must not
|
||||
// hide the pause. Rank by what the user needs to know about.
|
||||
let active: Vec<ActiveRunRow> = sqlx::query_as(
|
||||
"SELECT id, status, thread_id, checkpoint->>'step' AS step FROM runs
|
||||
WHERE bot_id = $1
|
||||
AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
ORDER BY CASE status
|
||||
WHEN 'running' THEN 0 WHEN 'leased' THEN 1
|
||||
WHEN 'waiting_takeover' THEN 2 WHEN 'waiting_input' THEN 3
|
||||
ELSE 4 END,
|
||||
created_at ASC",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.fetch_all(state.pool())
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let waiting = active
|
||||
.iter()
|
||||
.find(|run| run.status == "waiting_takeover");
|
||||
let busy = active.iter().find(|run| {
|
||||
parse_run_status(&run.status).is_some_and(|status| {
|
||||
status.is_active() && status != lazyboy_contracts::RunStatus::WaitingTakeover
|
||||
})
|
||||
});
|
||||
// While the bot is paused for the human, later messages just queue up; the
|
||||
// spinner would lie, so report them as queued instead of busy.
|
||||
let busy = if waiting.is_some() {
|
||||
busy.filter(|run| run.status != "queued")
|
||||
} else {
|
||||
busy
|
||||
};
|
||||
let busy_bot_name = busy.map(|_| bot.name.clone());
|
||||
let mut status = status_from(bot_id, &computer, screen.as_ref(), busy_bot_name);
|
||||
status.takeover_requested = waiting_for_takeover;
|
||||
if busy.is_some() {
|
||||
if let Some((run_id, _, thread_id)) = active {
|
||||
status.busy_run_id = Some(run_id);
|
||||
status.busy_session_id = Some(thread_id);
|
||||
}
|
||||
status.takeover_requested = waiting.is_some();
|
||||
if let Some(run) = busy {
|
||||
status.busy_run_id = Some(run.id.clone());
|
||||
status.busy_session_id = Some(run.thread_id.clone());
|
||||
status.busy_step = run.step.clone();
|
||||
}
|
||||
if let Some(run) = waiting {
|
||||
status.waiting_run_id = Some(run.id.clone());
|
||||
status.waiting_session_id = Some(run.thread_id.clone());
|
||||
}
|
||||
status.queued_runs = active
|
||||
.iter()
|
||||
.filter(|run| run.status == "queued" && Some(run.id.as_str()) != busy.map(|b| b.id.as_str()))
|
||||
.count() as u32;
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
struct ActiveRunRow {
|
||||
id: String,
|
||||
status: String,
|
||||
thread_id: String,
|
||||
step: Option<String>,
|
||||
}
|
||||
|
||||
pub fn _keep_state(state: ComputerState, holder: ControlHolder, mode: BrowserProfileMode) {
|
||||
let _ = (state, holder, mode);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ mod routes;
|
|||
mod runs;
|
||||
mod screen_proxy;
|
||||
mod sessions;
|
||||
mod skills;
|
||||
mod state;
|
||||
mod tools;
|
||||
mod workspace;
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ pub fn router(state: AppState) -> Router {
|
|||
.merge(crate::rooms::router())
|
||||
.merge(crate::mcp::router())
|
||||
.merge(crate::workspace::router())
|
||||
.merge(crate::skills::router())
|
||||
.route("/api/bots", get(list_bots).post(create_bot))
|
||||
.route(
|
||||
"/api/bots/{id}",
|
||||
|
|
@ -544,29 +545,9 @@ async fn stop_task(
|
|||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.ok_or(StatusCode::NOT_FOUND)?;
|
||||
let run_ids: Vec<String> = sqlx::query_scalar(
|
||||
"UPDATE runs SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE bot_id = $1 AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(&id)
|
||||
.fetch_all(state.pool())
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
for run_id in &run_ids {
|
||||
computer::release_screen_execution(&state, run_id)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
}
|
||||
sqlx::query(
|
||||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL,
|
||||
execution_lease_expires_at = NULL, updated_at = now()
|
||||
WHERE execution_bot_id = $1",
|
||||
)
|
||||
.bind(&id)
|
||||
.execute(state.pool())
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
crate::runs::cancel_active_runs(&state, &id)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(json!({ "ok": true })))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use crate::tools::{ToolCtx, dispatch, tool_definitions};
|
|||
|
||||
const SCREENSHOT_CAPTION: &str = "Desktop screenshot (1280x800) with yellow numbered marks. Click by those element ids. The live VNC view has no marks.";
|
||||
|
||||
const SYSTEM: &str = "You operate this bot's Linux desktop. The human always sees the live screen. You do not need a screenshot for every step.
|
||||
const SYSTEM: &str = "You operate this bot's Linux desktop. The human watches the same live screen you act on. Only the latest screenshot you received is current; the human may have interacted with the screen since, so call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead.
|
||||
|
||||
Prefer the fast path, in this order:
|
||||
1) shell, list_files, read_file, write_file
|
||||
|
|
@ -32,7 +32,11 @@ Prefer the fast path, in this order:
|
|||
When you use the browser tool:
|
||||
- snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate.
|
||||
- Yellow numbered marks on the screenshot match the element list. Click the number, not guessed pixels.
|
||||
- If the control is not in the element list, login/2FA/CAPTCHA, or clicks do nothing, call request_takeover and stop. Do not guess-click.
|
||||
- If the control is not in the element list, take a fresh snapshot or scroll; it may be off-screen or not rendered yet.
|
||||
|
||||
When a click changes nothing: do not repeat it. A button is often disabled until a video, timer or page load finishes; call wait (up to 60s) and re-observe, then click by element id. Pages that need patience (training videos, quizzes, slow forms) are normal: keep working through them step by step and report progress in one short sentence when done.
|
||||
|
||||
Call request_takeover only for passwords, 2FA, CAPTCHA, payment, or a decision only the human can make. Never call it just because a click missed. When you do call it, say exactly what the human must do.
|
||||
|
||||
computer_act examples (native windows only):
|
||||
- {\"kind\":\"click\",\"element\":1}
|
||||
|
|
@ -40,6 +44,7 @@ computer_act examples (native windows only):
|
|||
- {\"kind\":\"type\",\"text\":\"...\"}
|
||||
- {\"kind\":\"key\",\"key\":\"Return\"}
|
||||
- {\"kind\":\"focus\",\"title\":\"Open File\"}
|
||||
- wait: {\"seconds\":30,\"reason\":\"video playing\"}
|
||||
|
||||
On a Team Computer, relative files live in your bot folder; use shared/ for shared work. Finish the user's task.";
|
||||
|
||||
|
|
@ -52,6 +57,12 @@ pub async fn send(
|
|||
client_nonce: Option<&str>,
|
||||
blocks: &[Value],
|
||||
) -> Result<Value, String> {
|
||||
if crate::skills::recording_skill(state.pool(), bot_id)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
return Err("示範進行中：先按「完成示範」或「取消」，再送訊息。".into());
|
||||
}
|
||||
let mut tx = state
|
||||
.pool()
|
||||
.begin()
|
||||
|
|
@ -365,36 +376,8 @@ async fn execute_run(
|
|||
None
|
||||
};
|
||||
|
||||
let space = state
|
||||
.db
|
||||
.get_space(actor)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.ok_or_else(|| "workspace not found".to_string())?;
|
||||
let provider = bot
|
||||
.model_provider
|
||||
.as_deref()
|
||||
.or(Some(space.default_model_provider.as_str()))
|
||||
.unwrap_or("xai")
|
||||
.parse::<ModelProvider>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model_id = bot
|
||||
.model_id
|
||||
.clone()
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| Some(space.default_model_id.clone()).filter(|value| !value.is_empty()));
|
||||
let backend = resolve_backend(ResolveModelRequest {
|
||||
provider,
|
||||
model_id,
|
||||
base_url: space.default_model_base_url.clone(),
|
||||
credentials: CredentialChain {
|
||||
bot: None,
|
||||
space: space.default_model_api_key.clone(),
|
||||
env: lazyboy_harness::credential_from_env(provider),
|
||||
},
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model = connect_model(&backend).map_err(|error| error.to_string())?;
|
||||
let (model, vision) = bot_model(state, actor, &bot).await?;
|
||||
let skills = crate::skills::saved_skills(state.pool(), bot_id).await;
|
||||
|
||||
let ctx = Arc::new(ToolCtx {
|
||||
sandbox: state.sandbox.clone(),
|
||||
|
|
@ -402,7 +385,7 @@ async fn execute_run(
|
|||
context: adapter_context_for(actor, bot_id, "run", screen.as_ref(), Some(run_id)),
|
||||
mode: parse_mode(&computer.scope),
|
||||
bot_id: bot_id.to_string(),
|
||||
vision: backend.capabilities.vision,
|
||||
vision,
|
||||
gui_block,
|
||||
previous_frame: std::sync::Mutex::new(None),
|
||||
elements: std::sync::Mutex::new(Vec::new()),
|
||||
|
|
@ -507,6 +490,13 @@ async fn execute_run(
|
|||
} else {
|
||||
vec![UserContent::text(prompt)]
|
||||
};
|
||||
if !resume_after_takeover {
|
||||
// The user named a taught skill: hand the model the full playbook up
|
||||
// front so it does not have to guess or call use_skill first.
|
||||
if let Some(skill) = crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await {
|
||||
first.push(UserContent::text(crate::skills::format_playbook_for_run(&skill)));
|
||||
}
|
||||
}
|
||||
let mut screenshots: u32 = 0;
|
||||
let mut screenshot_bytes: u64 = 0;
|
||||
if resume_after_takeover && ctx.gui_block.is_none() {
|
||||
|
|
@ -579,8 +569,12 @@ async fn execute_run(
|
|||
preamble.push_str("\n\n");
|
||||
preamble.push_str(&memory);
|
||||
}
|
||||
if let Some(index) = crate::skills::skills_preamble(&skills) {
|
||||
preamble.push_str("\n\n");
|
||||
preamble.push_str(&index);
|
||||
}
|
||||
|
||||
for _ in 0..24 {
|
||||
for _ in 0..40 {
|
||||
turns += 1;
|
||||
if let Some(halt) = renew_or_halt(state, run_id, lease_owner).await? {
|
||||
return finish_halt(
|
||||
|
|
@ -596,7 +590,9 @@ async fn execute_run(
|
|||
)
|
||||
.await;
|
||||
}
|
||||
drop_history_screenshots(&mut history);
|
||||
drop_history_screenshots(&mut history, &pending);
|
||||
set_run_step(state, run_id, MODEL_STEP).await;
|
||||
let model_started = std::time::Instant::now();
|
||||
let content = tokio::select! {
|
||||
halt = wait_for_halt(state, run_id) => {
|
||||
return finish_halt(
|
||||
|
|
@ -629,6 +625,13 @@ async fn execute_run(
|
|||
_ => {}
|
||||
}
|
||||
}
|
||||
tracing::info!(
|
||||
run_id,
|
||||
turn = turns,
|
||||
elapsed_ms = model_started.elapsed().as_millis() as u64,
|
||||
tool_calls = calls.len(),
|
||||
"model turn"
|
||||
);
|
||||
if calls.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
|
@ -653,8 +656,11 @@ async fn execute_run(
|
|||
let name = call.function.name.clone();
|
||||
used_gui |= matches!(
|
||||
name.as_str(),
|
||||
"computer_observe" | "computer_act" | "open_path" | "launch_app" | "browser"
|
||||
"computer_observe" | "computer_act" | "open_path" | "launch_app" | "browser" | "wait"
|
||||
);
|
||||
let step = describe_step(&name, &call.function.arguments);
|
||||
set_run_step(state, run_id, &step).await;
|
||||
let tool_started = std::time::Instant::now();
|
||||
let outcome = tokio::select! {
|
||||
halt = wait_for_halt(state, run_id) => {
|
||||
return finish_halt(
|
||||
|
|
@ -682,6 +688,17 @@ async fn execute_run(
|
|||
},
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
run_id,
|
||||
turn = turns,
|
||||
step = %step,
|
||||
elapsed_ms = tool_started.elapsed().as_millis() as u64,
|
||||
result_chars = outcome.text.chars().count(),
|
||||
screenshot = outcome.image.is_some(),
|
||||
pause = outcome.pause,
|
||||
result = %outcome.text.chars().take(160).collect::<String>().replace('\n', " "),
|
||||
"tool call"
|
||||
);
|
||||
// xAI rejects images inside tool results. Attach a changed
|
||||
// screenshot as a following user image instead.
|
||||
if let Some(image) = outcome.image {
|
||||
|
|
@ -792,7 +809,47 @@ async fn execute_run(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete_once(
|
||||
/// Resolve the model a bot runs on (bot override → workspace default → env
|
||||
/// credentials). Returns the connected model and whether it accepts images.
|
||||
pub(crate) async fn bot_model(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot: &crate::db::BotRow,
|
||||
) -> Result<(DynModel, bool), String> {
|
||||
let space = state
|
||||
.db
|
||||
.get_space(actor)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.ok_or_else(|| "workspace not found".to_string())?;
|
||||
let provider = bot
|
||||
.model_provider
|
||||
.as_deref()
|
||||
.or(Some(space.default_model_provider.as_str()))
|
||||
.unwrap_or("xai")
|
||||
.parse::<ModelProvider>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model_id = bot
|
||||
.model_id
|
||||
.clone()
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| Some(space.default_model_id.clone()).filter(|value| !value.is_empty()));
|
||||
let backend = resolve_backend(ResolveModelRequest {
|
||||
provider,
|
||||
model_id,
|
||||
base_url: space.default_model_base_url.clone(),
|
||||
credentials: CredentialChain {
|
||||
bot: None,
|
||||
space: space.default_model_api_key.clone(),
|
||||
env: lazyboy_harness::credential_from_env(provider),
|
||||
},
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model = connect_model(&backend).map_err(|error| error.to_string())?;
|
||||
Ok((model, backend.capabilities.vision))
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_once(
|
||||
model: &DynModel,
|
||||
pending: Message,
|
||||
preamble: &str,
|
||||
|
|
@ -848,14 +905,38 @@ fn screenshot_parts(image: Vec<u8>) -> Vec<UserContent> {
|
|||
ImageMediaType::PNG
|
||||
};
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(image);
|
||||
// `Low` makes OpenAI-compatible backends downscale the 1280x800 frame to
|
||||
// ~512px before the model sees it: small text vanishes and coordinate
|
||||
// clicks land 2-3x off. Coordinates only work at native resolution.
|
||||
vec![
|
||||
UserContent::text(SCREENSHOT_CAPTION),
|
||||
UserContent::image_base64(encoded, Some(media), Some(ImageDetail::Low)),
|
||||
UserContent::image_base64(encoded, Some(media), Some(ImageDetail::High)),
|
||||
]
|
||||
}
|
||||
|
||||
fn drop_history_screenshots(history: &mut [Message]) {
|
||||
for message in history.iter_mut() {
|
||||
fn has_screenshot(message: &Message) -> bool {
|
||||
match message {
|
||||
Message::User { content } => content
|
||||
.iter()
|
||||
.any(|part| matches!(part, UserContent::Image(_))),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep exactly one screenshot in the model's context: the one in `pending`
|
||||
/// if it carries a fresh frame, otherwise the most recent one already in
|
||||
/// history. Without the fallback a "(screen unchanged)" turn would leave the
|
||||
/// model with no picture of the desktop at all.
|
||||
fn drop_history_screenshots(history: &mut [Message], pending: &Message) {
|
||||
let keep = if has_screenshot(pending) {
|
||||
None
|
||||
} else {
|
||||
history.iter().rposition(has_screenshot)
|
||||
};
|
||||
for (index, message) in history.iter_mut().enumerate() {
|
||||
if keep == Some(index) {
|
||||
continue;
|
||||
}
|
||||
let Message::User { content } = message else {
|
||||
continue;
|
||||
};
|
||||
|
|
@ -867,6 +948,33 @@ fn drop_history_screenshots(history: &mut [Message]) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Cancel every unfinished run of a bot and free the screen/execution leases
|
||||
/// it held, so the desktop is available to a human immediately.
|
||||
pub(crate) async fn cancel_active_runs(state: &AppState, bot_id: &str) -> Result<Vec<String>, String> {
|
||||
let run_ids: Vec<String> = sqlx::query_scalar(
|
||||
"UPDATE runs SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE bot_id = $1 AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.fetch_all(state.pool())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
for run_id in &run_ids {
|
||||
computer::release_screen_execution(state, run_id).await?;
|
||||
}
|
||||
sqlx::query(
|
||||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL,
|
||||
execution_lease_expires_at = NULL, updated_at = now()
|
||||
WHERE execution_bot_id = $1",
|
||||
)
|
||||
.bind(bot_id)
|
||||
.execute(state.pool())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(run_ids)
|
||||
}
|
||||
|
||||
pub(crate) async fn append_bot_message(
|
||||
state: &AppState,
|
||||
thread_id: &str,
|
||||
|
|
@ -1025,6 +1133,94 @@ fn history_window_start(summary_seq: i32, current_seq: i32) -> i32 {
|
|||
summary_seq.min(current_seq.saturating_sub(1)).max(0)
|
||||
}
|
||||
|
||||
const MODEL_STEP: &str = "思考中";
|
||||
|
||||
/// Human-readable label for what the run is doing right now. Surfaced through
|
||||
/// `computer.status` so the chat can show "working: browser click" instead of a
|
||||
/// bare spinner while a tool runs.
|
||||
fn describe_step(name: &str, args: &Value) -> String {
|
||||
fn short(value: Option<&str>, max: usize) -> String {
|
||||
let text = value.unwrap_or("").replace('\n', " ");
|
||||
if text.chars().count() > max {
|
||||
format!("{}…", text.chars().take(max).collect::<String>())
|
||||
} else {
|
||||
text
|
||||
}
|
||||
}
|
||||
let get = |key: &str| args.get(key).and_then(Value::as_str);
|
||||
let detail = match name {
|
||||
"computer_observe" => "看畫面".to_string(),
|
||||
"computer_act" => args
|
||||
.get("actions")
|
||||
.and_then(Value::as_array)
|
||||
.map(|actions| {
|
||||
actions
|
||||
.iter()
|
||||
.take(4)
|
||||
.map(|action| {
|
||||
let field = |key: &str| action.get(key).and_then(Value::as_str);
|
||||
let kind = field("kind").or(field("type")).unwrap_or("?");
|
||||
let target = if let Some(text) = field("text").or(field("keys")) {
|
||||
short(Some(text), 24)
|
||||
} else if let Some(id) = lazyboy_control::element_id(action.get("element")) {
|
||||
format!("#{id}")
|
||||
} else if let (Some(x), Some(y)) = (
|
||||
action.get("x").and_then(Value::as_i64),
|
||||
action.get("y").and_then(Value::as_i64),
|
||||
) {
|
||||
format!("({x},{y})")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
format!("{kind} {target}").trim().to_string()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
"browser" => {
|
||||
let target = get("url").or(get("text")).or(get("selector")).map(|s| short(Some(s), 40));
|
||||
let target = target.or_else(|| lazyboy_control::element_id(args.get("element")).map(|id| format!("#{id}")));
|
||||
format!("{} {}", get("action").unwrap_or("snapshot"), target.unwrap_or_default())
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
"shell" => short(get("command").or(get("cmd")), 60),
|
||||
"wait" => format!(
|
||||
"{}s {}",
|
||||
args.get("seconds").and_then(Value::as_f64).unwrap_or(0.0).round(),
|
||||
short(get("reason"), 30)
|
||||
)
|
||||
.trim()
|
||||
.to_string(),
|
||||
"launch_app" | "open_path" => short(get("app").or(get("path")), 40),
|
||||
"read_file" | "write_file" | "list_dir" => short(get("path"), 40),
|
||||
"use_skill" => format!("讀取技能 {}", short(get("name"), 30)),
|
||||
_ => String::new(),
|
||||
};
|
||||
if detail.is_empty() {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{name}: {detail}")
|
||||
}
|
||||
}
|
||||
|
||||
async fn set_run_step(state: &AppState, run_id: &str, step: &str) {
|
||||
let result = sqlx::query(
|
||||
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||||
|| jsonb_build_object('step', $2::text, 'stepAt', now()),
|
||||
updated_at = now()
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(step)
|
||||
.execute(state.pool())
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(run_id, "failed to record run step: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_run_metrics(
|
||||
state: &AppState,
|
||||
thread_id: &str,
|
||||
|
|
@ -1062,9 +1258,32 @@ async fn record_run_metrics(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
RunHalt, halt_from_status, history_window_start, retryable_run_error, screenshot_parts,
|
||||
RunHalt, SCREENSHOT_CAPTION, describe_step, drop_history_screenshots, halt_from_status,
|
||||
history_window_start, retryable_run_error, screenshot_parts,
|
||||
};
|
||||
use rig_core::completion::message::UserContent;
|
||||
use rig_core::completion::message::{Message, UserContent};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn step_labels_summarize_tool_arguments() {
|
||||
assert_eq!(describe_step("computer_observe", &json!({})), "computer_observe: 看畫面");
|
||||
assert_eq!(
|
||||
describe_step(
|
||||
"computer_act",
|
||||
&json!({"actions":[{"kind":"click","x":10,"y":20},{"kind":"type","text":"hello world"}]})
|
||||
),
|
||||
"computer_act: click (10,20), type hello world"
|
||||
);
|
||||
assert_eq!(
|
||||
describe_step("browser", &json!({"action":"click","element":12})),
|
||||
"browser: click #12"
|
||||
);
|
||||
assert_eq!(
|
||||
describe_step("shell", &json!({"command":"ls\n-la"})),
|
||||
"shell: ls -la"
|
||||
);
|
||||
assert_eq!(describe_step("mcp_search", &json!({"query":"x"})), "mcp_search");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_never_reads_past_the_current_prompt() {
|
||||
|
|
@ -1081,6 +1300,60 @@ mod tests {
|
|||
assert!(matches!(parts[1], UserContent::Image(_)));
|
||||
}
|
||||
|
||||
fn user_with_shot(label: &str, byte: u8) -> Message {
|
||||
let mut content = vec![UserContent::text(label)];
|
||||
content.extend(screenshot_parts(vec![0xFF, 0xD8, 0xFF, byte]));
|
||||
Message::User { content }
|
||||
}
|
||||
|
||||
fn image_count(history: &[Message]) -> usize {
|
||||
history
|
||||
.iter()
|
||||
.filter_map(|message| match message {
|
||||
Message::User { content } => Some(content),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.filter(|part| matches!(part, UserContent::Image(_)))
|
||||
.count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_keeps_latest_screenshot_when_new_turn_has_none() {
|
||||
let mut history = vec![
|
||||
user_with_shot("a", 1),
|
||||
Message::Assistant {
|
||||
id: None,
|
||||
content: vec![],
|
||||
},
|
||||
user_with_shot("b", 2),
|
||||
];
|
||||
let pending = Message::User {
|
||||
content: vec![UserContent::text("(screen unchanged)")],
|
||||
};
|
||||
drop_history_screenshots(&mut history, &pending);
|
||||
assert_eq!(image_count(&history), 1);
|
||||
let Message::User { content } = &history[2] else {
|
||||
panic!("expected user message");
|
||||
};
|
||||
assert!(content.iter().any(|part| matches!(part, UserContent::Image(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_drops_every_screenshot_when_new_turn_has_one() {
|
||||
let mut history = vec![user_with_shot("a", 1), user_with_shot("b", 2)];
|
||||
let pending = user_with_shot("c", 3);
|
||||
drop_history_screenshots(&mut history, &pending);
|
||||
assert_eq!(image_count(&history), 0);
|
||||
assert!(history.iter().all(|message| match message {
|
||||
Message::User { content } => !content.iter().any(|part| matches!(
|
||||
part,
|
||||
UserContent::Text(text) if text.text == SCREENSHOT_CAPTION
|
||||
)),
|
||||
_ => true,
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_maps_paused_and_cancelled_runs() {
|
||||
assert_eq!(
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -5,7 +5,8 @@ use lazyboy_contracts::{
|
|||
};
|
||||
use lazyboy_control::{
|
||||
ActionError, ActionRequest, AdapterContext, CdpPage, CommandRequest, ComputerRef,
|
||||
SandboxProvider, apply_element_targets, cdp_command_on, format_ui_elements, frames_match,
|
||||
SandboxProvider, apply_element_targets, cdp_command_on, element_id, format_ui_elements,
|
||||
frames_match,
|
||||
merge_page_elements, overlay_elements, parse_cdp_page, parse_computer_actions,
|
||||
resolve_bot_workspace_cwd, resolve_bot_workspace_path,
|
||||
};
|
||||
|
|
@ -145,11 +146,25 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|||
"required":["application"]
|
||||
}),
|
||||
},
|
||||
ToolDefinition {
|
||||
name: "wait".into(),
|
||||
description: "Wait for the screen to change on its own (video playing, page loading, a button that enables later), then return a fresh observation. seconds: 1-60.".into(),
|
||||
parameters: json!({
|
||||
"type":"object",
|
||||
"properties":{"seconds":{"type":"number","minimum":1,"maximum":60},"reason":{"type":"string"}},
|
||||
"required":["seconds"]
|
||||
}),
|
||||
},
|
||||
ToolDefinition {
|
||||
name: "request_takeover".into(),
|
||||
description: "Ask the user to take over for passwords, 2FA, CAPTCHA, login walls, or when the right on-screen control cannot be found. Never ask them to paste secrets in chat.".into(),
|
||||
parameters: json!({"type":"object","properties":{"reason":{"type":"string"}},"required":["reason"]}),
|
||||
},
|
||||
ToolDefinition {
|
||||
name: "use_skill".into(),
|
||||
description: "Load the playbook of a skill the human taught this bot by demonstration (see 'Taught skills' in your instructions). Returns intent, inputs and semantic steps to follow with the normal tools.".into(),
|
||||
parameters: json!({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}),
|
||||
},
|
||||
];
|
||||
if memory_enabled {
|
||||
definitions.extend([
|
||||
|
|
@ -198,6 +213,7 @@ pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome {
|
|||
match name {
|
||||
"computer_observe" => observe(ctx).await,
|
||||
"computer_act" => act(ctx, args).await,
|
||||
"wait" => wait_then_observe(ctx, args).await,
|
||||
"browser" => browser(ctx, args).await,
|
||||
"shell" => shell(ctx, args).await,
|
||||
"list_files" => list_files(ctx, args).await,
|
||||
|
|
@ -220,6 +236,25 @@ pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome {
|
|||
pause: true,
|
||||
}
|
||||
}
|
||||
"use_skill" => {
|
||||
let name = args.get("name").and_then(Value::as_str).unwrap_or("");
|
||||
let skills = crate::skills::saved_skills(&ctx.pool, &ctx.bot_id).await;
|
||||
match crate::skills::find_skill(&skills, name) {
|
||||
Some(skill) => text_outcome(crate::skills::format_playbook_for_run(skill)),
|
||||
None => text_outcome(format!(
|
||||
"no taught skill named {name:?}. Available: {}",
|
||||
if skills.is_empty() {
|
||||
"none".to_string()
|
||||
} else {
|
||||
skills
|
||||
.iter()
|
||||
.map(|skill| skill.name.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
)),
|
||||
}
|
||||
}
|
||||
other if other.starts_with("mcp_") => match ctx.mcp.call(other, args).await {
|
||||
Ok(text) => text_outcome(text),
|
||||
Err(error) => text_outcome(format!("MCP 工具失敗：{error}")),
|
||||
|
|
@ -368,6 +403,29 @@ async fn observe(ctx: &ToolCtx) -> ToolOutcome {
|
|||
}
|
||||
}
|
||||
|
||||
/// Bounded so it always fits inside the 90s per-tool budget with an observe.
|
||||
const MAX_WAIT_SECS: f64 = 60.0;
|
||||
|
||||
async fn wait_then_observe(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
||||
let seconds = args
|
||||
.get("seconds")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(5.0)
|
||||
.clamp(1.0, MAX_WAIT_SECS);
|
||||
tokio::time::sleep(std::time::Duration::from_secs_f64(seconds)).await;
|
||||
if vision_guard(ctx).is_some() {
|
||||
return text_outcome(format!("waited {seconds:.0}s"));
|
||||
}
|
||||
match ctx.sandbox.observe(&ctx.computer, &ctx.context).await {
|
||||
Ok(observation) => {
|
||||
let note = format!("waited {seconds:.0}s");
|
||||
let (observation, note) = attach_page_elements(ctx, observation, ¬e).await;
|
||||
pack_observation(ctx, ¬e, observation)
|
||||
}
|
||||
Err(error) => text_outcome(format!("waited {seconds:.0}s; observe failed: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn attach_page_elements(
|
||||
ctx: &ToolCtx,
|
||||
mut observation: ComputerObservation,
|
||||
|
|
@ -443,7 +501,7 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
}
|
||||
}
|
||||
if request.get("selector").and_then(Value::as_str).is_none() {
|
||||
if let Some(id) = args.get("element").and_then(Value::as_u64) {
|
||||
if let Some(id) = element_id(args.get("element").or_else(|| args.get("id"))) {
|
||||
let elements = ctx.elements.lock().unwrap().clone();
|
||||
match elements
|
||||
.iter()
|
||||
|
|
@ -459,9 +517,24 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
}
|
||||
}
|
||||
if matches!(action, "click") && request.get("selector").and_then(Value::as_str).is_none() {
|
||||
return text_outcome(
|
||||
"browser click needs element id or selector. Call browser snapshot first.",
|
||||
);
|
||||
let known: Vec<String> = ctx
|
||||
.elements
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|element| element.selector.is_some())
|
||||
.map(|element| format!("[{}] {}", element.id, element.title))
|
||||
.take(12)
|
||||
.collect();
|
||||
return text_outcome(format!(
|
||||
"browser click needs {{\"action\":\"click\",\"element\":N}} with a number from the last snapshot, or a CSS selector. You sent: {}. Known elements: {}",
|
||||
serde_json::to_string(args).unwrap_or_default(),
|
||||
if known.is_empty() {
|
||||
"none yet, call browser snapshot first".to_string()
|
||||
} else {
|
||||
known.join(", ")
|
||||
}
|
||||
));
|
||||
}
|
||||
let page = cdp_call(ctx, request).await;
|
||||
if !page.ok {
|
||||
|
|
@ -526,7 +599,7 @@ async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
if kind != "click" {
|
||||
continue;
|
||||
}
|
||||
let Some(id) = item.get("element").and_then(Value::as_u64) else {
|
||||
let Some(id) = element_id(item.get("element")) else {
|
||||
continue;
|
||||
};
|
||||
let Some(selector) = elements
|
||||
|
|
@ -568,7 +641,7 @@ async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
ActionRequest {
|
||||
actions,
|
||||
observe: args.get("observe").and_then(Value::as_bool) != Some(false),
|
||||
settle_ms: args.get("settle_ms").and_then(Value::as_u64).unwrap_or(120) as u32,
|
||||
settle_ms: args.get("settle_ms").and_then(Value::as_u64).unwrap_or(350) as u32,
|
||||
display: ctx.context.display.clone(),
|
||||
profile_path: ctx.context.profile_path.clone(),
|
||||
},
|
||||
|
|
@ -613,18 +686,22 @@ fn action_is_click(action: &ComputerAction) -> bool {
|
|||
)
|
||||
}
|
||||
|
||||
fn pause_unknown_element(ctx: &ToolCtx, id: u32, elements: &[UiElement]) -> ToolOutcome {
|
||||
*ctx.takeover_requested.lock().unwrap() = true;
|
||||
/// A stale element id is the model's mistake, not a reason to park the run
|
||||
/// on the human: tell it what is visible now and let it re-observe.
|
||||
fn pause_unknown_element(_ctx: &ToolCtx, id: u32, elements: &[UiElement]) -> ToolOutcome {
|
||||
ToolOutcome {
|
||||
text: format!(
|
||||
"找不到畫面上的元素 {id}。目前可見：{}。請接手操作，完成後釋放控制權，我會從目前畫面繼續。",
|
||||
"element {id} is not on screen any more. Visible now: {}. Call browser snapshot or computer_observe to get fresh ids, then retry.",
|
||||
format_ui_elements(elements)
|
||||
),
|
||||
image: None,
|
||||
pause: true,
|
||||
pause: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clicks that change nothing are common and usually recoverable (disabled
|
||||
/// button, video still playing, slightly off target). Coach the model instead
|
||||
/// of pausing; it can still call request_takeover when it is truly stuck.
|
||||
fn note_click_result(ctx: &ToolCtx, had_click: bool, unchanged: bool, outcome: &mut ToolOutcome) {
|
||||
if !had_click {
|
||||
return;
|
||||
|
|
@ -634,10 +711,8 @@ fn note_click_result(ctx: &ToolCtx, had_click: bool, unchanged: bool, outcome: &
|
|||
*streak += 1;
|
||||
*ctx.click_misses.lock().unwrap() += 1;
|
||||
if *streak >= 2 {
|
||||
*ctx.takeover_requested.lock().unwrap() = true;
|
||||
outcome.pause = true;
|
||||
outcome.text.push_str(
|
||||
"\n連續兩次點擊後畫面沒有變化。請接手確認，完成後釋放控制權，我會從目前畫面繼續。",
|
||||
"\nThe last clicks changed nothing. Do not repeat the same click. Options: the control may be disabled until a video/loading finishes (use wait, then re-observe); the target may be off (use browser snapshot and click by element id, or pick coordinates from a fresh computer_observe); if the page needs login, CAPTCHA or a human decision, call request_takeover.",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -126,15 +126,26 @@ async fn update_settings(
|
|||
if provider.requires_base_url() && base_url.is_none() {
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
let api_key = if input.clear_api_key {
|
||||
let current = state
|
||||
.db
|
||||
.get_space(&actor)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let provider_changed = current
|
||||
.as_ref()
|
||||
.is_some_and(|space| space.default_model_provider != provider.as_str());
|
||||
let supplied = input
|
||||
.api_key
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty());
|
||||
// A stored key belongs to the provider it was entered for. Carrying it
|
||||
// over to a new provider shadows the env key and every run fails with
|
||||
// "Incorrect API key", so drop it unless a new one is supplied.
|
||||
let api_key = if input.clear_api_key || (provider_changed && supplied.is_none()) {
|
||||
Some(None)
|
||||
} else {
|
||||
input
|
||||
.api_key
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(Some)
|
||||
supplied.map(Some)
|
||||
};
|
||||
state
|
||||
.db
|
||||
|
|
|
|||
|
|
@ -186,6 +186,17 @@ pub struct ComputerStatus {
|
|||
pub busy_bot_name: Option<String>,
|
||||
pub busy_session_id: Option<String>,
|
||||
pub busy_run_id: Option<String>,
|
||||
/// What the active run is doing right now ("思考中", "browser: click #12"…).
|
||||
#[serde(default)]
|
||||
pub busy_step: Option<String>,
|
||||
/// Run paused in `waiting_takeover` (bot asked for, or user forced, control).
|
||||
#[serde(default)]
|
||||
pub waiting_run_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub waiting_session_id: Option<String>,
|
||||
/// Messages queued behind the active/paused run for this bot.
|
||||
#[serde(default)]
|
||||
pub queued_runs: u32,
|
||||
pub multi_screen: bool,
|
||||
pub screen_id: Option<String>,
|
||||
pub display: Option<String>,
|
||||
|
|
|
|||
|
|
@ -140,12 +140,32 @@ pub fn model_capabilities(provider: ModelProvider, model_id: &str) -> ModelCapab
|
|||
|| id.starts_with("grok-3")
|
||||
}
|
||||
ModelProvider::OpencodeGo => id.contains("vision") || id.contains("omni"),
|
||||
// OpenAI-compatible is also how OpenRouter / LiteLLM / vLLM gateways
|
||||
// are reached, so the id may be any vendor's multimodal model.
|
||||
ModelProvider::OpenaiCompatible | ModelProvider::Openai => {
|
||||
id.contains("gpt-4o")
|
||||
|| id.contains("gpt-4.1")
|
||||
|| id.contains("gpt-5")
|
||||
|| id.starts_with("o3")
|
||||
|| id.starts_with("o4")
|
||||
|| id.contains("vision")
|
||||
|| id.contains("llava")
|
||||
|| id.contains("omni")
|
||||
|| id.contains("claude")
|
||||
|| id.contains("gemini")
|
||||
|| id.contains("grok-4")
|
||||
|| (id.contains("qwen") && (id.contains("vl") || id.contains("qwen3")))
|
||||
|| id.contains("glm-4v")
|
||||
|| id.contains("glm-4.5v")
|
||||
|| id.contains("glm-5v")
|
||||
|| id.contains("ui-tars")
|
||||
|| id.contains("pixtral")
|
||||
|| id.contains("internvl")
|
||||
|| id.contains("llama-4")
|
||||
|| id.contains("kimi-k2.5")
|
||||
|| id.contains("kimi-k2.6")
|
||||
|| id.contains("kimi-k2.7")
|
||||
|| id.contains("computer-use")
|
||||
}
|
||||
ModelProvider::Anthropic => id.contains("claude"),
|
||||
ModelProvider::Openrouter => {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,23 @@ pub enum ActionError {
|
|||
UnknownElement(u32),
|
||||
}
|
||||
|
||||
/// Models write element ids as `3`, `3.0`, `"3"` or `"[3]"`; accept them all
|
||||
/// instead of failing the click.
|
||||
pub fn element_id(value: Option<&Value>) -> Option<u64> {
|
||||
match value? {
|
||||
Value::Number(number) => number
|
||||
.as_u64()
|
||||
.or_else(|| number.as_f64().filter(|f| *f >= 0.0).map(|f| f.round() as u64)),
|
||||
Value::String(text) => text
|
||||
.trim()
|
||||
.trim_start_matches(['#', '['])
|
||||
.trim_end_matches(']')
|
||||
.parse()
|
||||
.ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fill x/y from a numbered on-screen element so the model can click by id.
|
||||
pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Result<(), ActionError> {
|
||||
let Some(items) = value.as_array_mut() else {
|
||||
|
|
@ -31,7 +48,7 @@ pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Resul
|
|||
let Some(action) = raw.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
let Some(id) = action.get("element").and_then(Value::as_u64) else {
|
||||
let Some(id) = element_id(action.get("element")) else {
|
||||
continue;
|
||||
};
|
||||
let Some(element) = elements.iter().find(|element| u64::from(element.id) == id) else {
|
||||
|
|
@ -246,6 +263,18 @@ mod tests {
|
|||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn element_ids_accept_common_model_spellings() {
|
||||
assert_eq!(element_id(Some(&json!(3))), Some(3));
|
||||
assert_eq!(element_id(Some(&json!(3.0))), Some(3));
|
||||
assert_eq!(element_id(Some(&json!("3"))), Some(3));
|
||||
assert_eq!(element_id(Some(&json!("#12"))), Some(12));
|
||||
assert_eq!(element_id(Some(&json!("[7]"))), Some(7));
|
||||
assert_eq!(element_id(Some(&json!("Learn more"))), None);
|
||||
assert_eq!(element_id(Some(&json!(-1))), None);
|
||||
assert_eq!(element_id(None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_click_and_double_click() {
|
||||
let actions = parse_computer_actions(&json!([
|
||||
|
|
|
|||
|
|
@ -111,20 +111,28 @@ class Ws:
|
|||
def probe(port):
|
||||
return http_json("http://127.0.0.1:%s/json/version" % port) is not None
|
||||
|
||||
def kill_profile(profile):
|
||||
def profile_alive(profile):
|
||||
if not profile:
|
||||
return
|
||||
return False
|
||||
try:
|
||||
out = subprocess.check_output(["pgrep", "-af", "chromium"], text=True, stderr=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return
|
||||
return False
|
||||
for line in out.splitlines():
|
||||
if profile not in line or "--type=" in line:
|
||||
continue
|
||||
try:
|
||||
os.kill(int(line.split()[0]), 15)
|
||||
except Exception:
|
||||
pass
|
||||
if ("--user-data-dir=%s" % profile) in line and "--type=" not in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
def active_port(profile):
|
||||
# Chromium writes the DevTools port it actually bound here. Trust it over
|
||||
# our expected port so we attach to the window the human already sees.
|
||||
if not profile:
|
||||
return None
|
||||
try:
|
||||
with open(os.path.join(profile, "DevToolsActivePort")) as f:
|
||||
return int(f.readline().strip())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def spawn_browser(display, profile, port):
|
||||
env = os.environ.copy()
|
||||
|
|
@ -144,8 +152,14 @@ def connect(port):
|
|||
pages = [t for t in tabs if t.get("type") == "page" and t.get("webSocketDebuggerUrl")]
|
||||
if not pages:
|
||||
fail("no browser tab")
|
||||
# /json/list is ordered by last activity, so pages[0] is the tab the human
|
||||
# is looking at. Bring it to the front anyway so what the model reads and
|
||||
# clicks is always the tab shown on the live screen.
|
||||
pages.sort(key=lambda t: (t.get("url") or "").startswith("chrome://"), reverse=False)
|
||||
ws = Ws(pages[0]["webSocketDebuggerUrl"])
|
||||
page = pages[0]
|
||||
if page.get("id"):
|
||||
http_json("http://127.0.0.1:%s/json/activate/%s" % (port, page["id"]))
|
||||
ws = Ws(page["webSocketDebuggerUrl"])
|
||||
ws.call("Runtime.enable")
|
||||
ws.call("Page.enable")
|
||||
return ws
|
||||
|
|
@ -279,6 +293,187 @@ def press(ws, key):
|
|||
ws.call("Input.dispatchKeyEvent", {"type": "keyDown", "text": key[:1]})
|
||||
ws.call("Input.dispatchKeyEvent", {"type": "keyUp", "text": key[:1]})
|
||||
|
||||
# Injected into every page while a human demonstrates a task. It reports what
|
||||
# the person did in terms of page semantics (which control, what text, which
|
||||
# URL) rather than pixels, so the distilled skill can generalise. Secrets are
|
||||
# masked before they leave the page.
|
||||
RECORD_JS = r"""
|
||||
(() => {
|
||||
if (window.__lbTeachInstalled) return;
|
||||
window.__lbTeachInstalled = true;
|
||||
const send = (ev) => { try { ev.at = Date.now(); ev.url = location.href; window.__lbTeach(JSON.stringify(ev)); } catch (e) {} };
|
||||
const clean = (s) => (s || "").replace(/\s+/g, " ").trim().slice(0, 120);
|
||||
const secretRe = /pass|pwd|secret|token|otp|cvv|card|pin\b/i;
|
||||
const isSecret = (el) => !el ? false : (el.type === "password" || secretRe.test(el.name || "") || secretRe.test(el.id || "") || secretRe.test(el.autocomplete || "") || secretRe.test(el.getAttribute && el.getAttribute("aria-label") || ""));
|
||||
const labelFor = (el) => {
|
||||
if (!el) return "";
|
||||
if (el.labels && el.labels.length) return clean(el.labels[0].innerText);
|
||||
const id = el.id && document.querySelector('label[for="' + el.id + '"]');
|
||||
if (id) return clean(id.innerText);
|
||||
return clean(el.getAttribute("aria-label") || el.placeholder || el.title || el.name || "");
|
||||
};
|
||||
const describe = (el) => {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const d = { tag, role: el.getAttribute("role") || "", text: clean(el.innerText || el.value || el.alt || el.getAttribute("aria-label") || el.title || el.placeholder || ""), label: labelFor(el) };
|
||||
if (el.id) d.id = el.id;
|
||||
if (el.name) d.name = el.name;
|
||||
if (tag === "a" && el.href) d.href = el.href.slice(0, 200);
|
||||
if (tag === "input") d.type = el.type || "text";
|
||||
return d;
|
||||
};
|
||||
const actionable = (node) => {
|
||||
let el = node;
|
||||
for (let i = 0; el && i < 6; i++) {
|
||||
if (el.nodeType === 1) {
|
||||
const t = el.tagName.toLowerCase();
|
||||
if (["a","button","input","select","textarea","summary","label","option"].includes(t) || el.getAttribute("role") || el.onclick || el.getAttribute("tabindex") !== null || el.isContentEditable) return el;
|
||||
}
|
||||
el = el.parentNode;
|
||||
}
|
||||
return node && node.nodeType === 1 ? node : null;
|
||||
};
|
||||
send({ t: "page", title: document.title });
|
||||
document.addEventListener("click", (e) => {
|
||||
const el = actionable(e.target);
|
||||
const d = describe(el);
|
||||
if (d) send({ t: "click", el: d, x: Math.round(e.clientX), y: Math.round(e.clientY) });
|
||||
}, true);
|
||||
const pending = new Map();
|
||||
const flush = (el) => {
|
||||
pending.delete(el);
|
||||
const d = describe(el);
|
||||
if (!d) return;
|
||||
let value = el.isContentEditable ? el.innerText : (el.value || "");
|
||||
if (el.tagName === "SELECT" && el.selectedOptions && el.selectedOptions[0]) value = el.selectedOptions[0].text;
|
||||
if (el.type === "checkbox" || el.type === "radio") value = el.checked ? "checked" : "unchecked";
|
||||
send({ t: "input", el: d, value: isSecret(el) ? "[redacted]" : clean(value) });
|
||||
};
|
||||
document.addEventListener("input", (e) => {
|
||||
const el = e.target;
|
||||
if (!el || !(el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) return;
|
||||
clearTimeout(pending.get(el));
|
||||
pending.set(el, setTimeout(() => flush(el), 900));
|
||||
}, true);
|
||||
document.addEventListener("change", (e) => { const el = e.target; if (el && el.nodeType === 1) { clearTimeout(pending.get(el)); flush(el); } }, true);
|
||||
document.addEventListener("keydown", (e) => {
|
||||
const special = ["Enter","Escape","Tab"].includes(e.key) || e.ctrlKey || e.metaKey || e.altKey;
|
||||
if (!special || e.key === "Control" || e.key === "Meta" || e.key === "Alt" || e.key === "Shift") return;
|
||||
const el = document.activeElement;
|
||||
if (el && pending.has(el)) { clearTimeout(pending.get(el)); flush(el); }
|
||||
const combo = [e.ctrlKey ? "Ctrl" : "", e.metaKey ? "Meta" : "", e.altKey ? "Alt" : "", e.shiftKey ? "Shift" : "", e.key].filter(Boolean).join("+");
|
||||
send({ t: "key", key: combo, el: describe(el) });
|
||||
}, true);
|
||||
document.addEventListener("submit", (e) => { const f = e.target; send({ t: "submit", form: { action: (f && f.action || "").slice(0, 200), name: f && (f.name || f.id) || "" } }); }, true);
|
||||
let lastScroll = 0;
|
||||
window.addEventListener("scroll", () => { const now = Date.now(); if (now - lastScroll > 2000) { lastScroll = now; send({ t: "scroll", y: Math.round(window.scrollY) }); } }, true);
|
||||
})()
|
||||
"""
|
||||
|
||||
class Recorder:
|
||||
"""Browser-level CDP session with flattened page sessions. Events from
|
||||
every tab are appended to a JSONL file until the process is killed."""
|
||||
|
||||
def __init__(self, port, out):
|
||||
info = http_json("http://127.0.0.1:%s/json/version" % port) or {}
|
||||
url = info.get("webSocketDebuggerUrl")
|
||||
if not url:
|
||||
raise RuntimeError("browser has no DevTools endpoint")
|
||||
self.ws = Ws(url)
|
||||
self.out = open(out, "a", buffering=1)
|
||||
self.sessions = {}
|
||||
self.pending = []
|
||||
self.last = (None, 0)
|
||||
|
||||
def emit(self, ev):
|
||||
ev.setdefault("at", int(time.time() * 1000))
|
||||
# Two sessions on one page (auto-attach + explicit) deliver the same
|
||||
# binding call twice; a key repeat is never that fast either.
|
||||
key = json.dumps({k: v for k, v in ev.items() if k != "at"}, sort_keys=True)
|
||||
if key == self.last[0] and ev["at"] - self.last[1] < 800:
|
||||
return
|
||||
self.last = (key, ev["at"])
|
||||
self.out.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
|
||||
def call(self, method, params=None, session=None):
|
||||
self.ws.n += 1
|
||||
msg = {"id": self.ws.n, "method": method}
|
||||
if params:
|
||||
msg["params"] = params
|
||||
if session:
|
||||
msg["sessionId"] = session
|
||||
self.ws.sock.sendall(self.ws._frame(json.dumps(msg).encode()))
|
||||
while True:
|
||||
obj = self.ws.recv_json()
|
||||
if obj.get("id") == self.ws.n:
|
||||
if "error" in obj:
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
return obj.get("result") or {}
|
||||
self.pending.append(obj)
|
||||
|
||||
def attach(self, session, target):
|
||||
if target.get("type") != "page" or not session:
|
||||
return
|
||||
target_id = target.get("targetId")
|
||||
if session in self.sessions:
|
||||
return
|
||||
if target_id in self.sessions.values():
|
||||
try:
|
||||
self.call("Target.detachFromTarget", {"sessionId": session})
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
self.sessions[session] = target_id
|
||||
for method, params in (
|
||||
("Runtime.enable", None),
|
||||
("Page.enable", None),
|
||||
("Runtime.addBinding", {"name": "__lbTeach"}),
|
||||
("Page.addScriptToEvaluateOnNewDocument", {"source": RECORD_JS}),
|
||||
("Runtime.evaluate", {"expression": RECORD_JS}),
|
||||
):
|
||||
try:
|
||||
self.call(method, params, session)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def handle(self, obj):
|
||||
method = obj.get("method")
|
||||
params = obj.get("params") or {}
|
||||
if method == "Target.attachedToTarget":
|
||||
self.attach(params.get("sessionId"), params.get("targetInfo") or {})
|
||||
elif method == "Target.detachedFromTarget":
|
||||
self.sessions.pop(params.get("sessionId"), None)
|
||||
elif method == "Runtime.bindingCalled" and params.get("name") == "__lbTeach":
|
||||
try:
|
||||
self.emit(json.loads(params.get("payload") or "{}"))
|
||||
except Exception:
|
||||
pass
|
||||
elif method == "Page.frameNavigated":
|
||||
frame = params.get("frame") or {}
|
||||
if not frame.get("parentId"):
|
||||
self.emit({"t": "navigate", "url": frame.get("url") or ""})
|
||||
elif method == "Target.targetInfoChanged":
|
||||
info = params.get("targetInfo") or {}
|
||||
if info.get("type") == "page" and info.get("title"):
|
||||
self.emit({"t": "title", "url": info.get("url") or "", "title": info.get("title")})
|
||||
|
||||
def run(self):
|
||||
self.ws.sock.settimeout(None)
|
||||
self.call("Target.setDiscoverTargets", {"discover": True})
|
||||
self.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True})
|
||||
for target in (self.call("Target.getTargets") or {}).get("targetInfos", []):
|
||||
if target.get("type") == "page":
|
||||
try:
|
||||
result = self.call("Target.attachToTarget", {"targetId": target["targetId"], "flatten": True})
|
||||
self.attach(result.get("sessionId"), target)
|
||||
except Exception:
|
||||
pass
|
||||
self.emit({"t": "recorder", "state": "started"})
|
||||
while True:
|
||||
while self.pending:
|
||||
self.handle(self.pending.pop(0))
|
||||
self.handle(self.ws.recv_json())
|
||||
|
||||
def main():
|
||||
req = json.loads(sys.argv[1])
|
||||
action = req.get("action") or "snapshot"
|
||||
|
|
@ -291,11 +486,32 @@ def main():
|
|||
print(json.dumps({"ok": probe(port)}))
|
||||
return
|
||||
|
||||
if not probe(port):
|
||||
def bound_port():
|
||||
if probe(port):
|
||||
return port
|
||||
bound = active_port(profile)
|
||||
if bound and bound != port and probe(bound):
|
||||
return bound
|
||||
return None
|
||||
|
||||
restarted = False
|
||||
ready_port = bound_port()
|
||||
if ready_port is None and profile_alive(profile):
|
||||
# The window may still be booting (lazyboy-screen just spawned it).
|
||||
for _ in range(12):
|
||||
time.sleep(0.25)
|
||||
ready_port = bound_port()
|
||||
if ready_port is not None:
|
||||
break
|
||||
if ready_port is not None:
|
||||
port = ready_port
|
||||
else:
|
||||
if profile_alive(profile):
|
||||
# Never kill the window the human is watching. Fall back to the
|
||||
# screenshot tools, which see exactly what the live screen shows.
|
||||
fail("browser is open but has no DevTools; use computer_observe/computer_act on it instead. Do not restart the browser.")
|
||||
if not ensure:
|
||||
fail("cdp unavailable")
|
||||
kill_profile(profile)
|
||||
time.sleep(0.4)
|
||||
spawn_browser(display, profile, port)
|
||||
ready = False
|
||||
for _ in range(24):
|
||||
|
|
@ -306,13 +522,17 @@ def main():
|
|||
if not ready:
|
||||
fail("cdp unavailable")
|
||||
restarted = True
|
||||
else:
|
||||
restarted = False
|
||||
|
||||
if action == "ensure":
|
||||
print(json.dumps({"ok": True, "restarted": restarted}))
|
||||
return
|
||||
|
||||
if action == "record":
|
||||
# Long-running: the API starts this detached and kills it on stop.
|
||||
out = req.get("out") or "/tmp/lazyboy-teach.jsonl"
|
||||
Recorder(port, out).run()
|
||||
return
|
||||
|
||||
ws = connect(port)
|
||||
try:
|
||||
if action == "snapshot":
|
||||
|
|
|
|||
|
|
@ -42,6 +42,44 @@ pub fn cdp_command(request: &Value) -> Vec<String> {
|
|||
cdp_command_on(PRIMARY_DISPLAY, None, request)
|
||||
}
|
||||
|
||||
/// Marker embedded in the recorder's argv so `pkill -f` can find exactly one
|
||||
/// teaching session without touching other python processes.
|
||||
pub fn teach_recorder_tag(skill_id: &str) -> String {
|
||||
format!("lazyboy-teach-{skill_id}")
|
||||
}
|
||||
|
||||
pub fn teach_recorder_output(skill_id: &str) -> String {
|
||||
format!("/tmp/{}.jsonl", teach_recorder_tag(skill_id))
|
||||
}
|
||||
|
||||
/// Detached, long-running CDP recorder for a human demonstration. The script
|
||||
/// is handed to `sh` as a positional argument so no shell quoting touches it.
|
||||
pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &str) -> Vec<String> {
|
||||
let mut request = json!({
|
||||
"action": "record",
|
||||
"ensure": true,
|
||||
"out": teach_recorder_output(skill_id),
|
||||
"tag": teach_recorder_tag(skill_id),
|
||||
"display": normalize_display(display),
|
||||
"port": devtools_port(display),
|
||||
});
|
||||
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
|
||||
request["profile"] = json!(profile);
|
||||
}
|
||||
vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
"setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 </dev/null &".into(),
|
||||
normalize_display(display).to_string(),
|
||||
CDP_PY.into(),
|
||||
request.to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn cdp_record_stop_command(skill_id: &str) -> Vec<String> {
|
||||
vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)]
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct CdpPage {
|
||||
pub ok: bool,
|
||||
|
|
|
|||
|
|
@ -53,6 +53,33 @@ pub fn frames_match(previous_frame_id: Option<&str>, observation: &ComputerObser
|
|||
previous_frame_id == Some(observation.frame_id.as_str())
|
||||
}
|
||||
|
||||
const SIGNATURE_W: u32 = 32;
|
||||
const SIGNATURE_H: u32 = 18;
|
||||
|
||||
/// Coarse grayscale thumbnail of a screenshot. Two frames whose signatures
|
||||
/// are `similar` differ only in small details (panel clock, caret blink),
|
||||
/// which a byte-level `frame_id` comparison would treat as a new frame.
|
||||
pub fn frame_signature(image: &[u8]) -> Option<Vec<u8>> {
|
||||
let dynamic = image::load_from_memory(image).ok()?;
|
||||
let thumb = dynamic
|
||||
.resize_exact(SIGNATURE_W, SIGNATURE_H, image::imageops::FilterType::Triangle)
|
||||
.to_luma8();
|
||||
Some(thumb.into_raw())
|
||||
}
|
||||
|
||||
/// True when fewer than 2% of the coarse cells moved by a visible amount.
|
||||
pub fn signatures_similar(a: &[u8], b: &[u8]) -> bool {
|
||||
if a.len() != b.len() || a.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let changed = a
|
||||
.iter()
|
||||
.zip(b)
|
||||
.filter(|(x, y)| x.abs_diff(**y) > 24)
|
||||
.count();
|
||||
changed * 50 < a.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -67,6 +94,33 @@ mod tests {
|
|||
assert!(!frames_match(Some(&a.frame_id), &c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signature_ignores_tiny_changes_but_sees_big_ones() {
|
||||
use image::{ImageEncoder, Rgb, RgbImage};
|
||||
fn png(paint: impl Fn(u32, u32) -> Rgb<u8>) -> Vec<u8> {
|
||||
let img = RgbImage::from_fn(320, 180, paint);
|
||||
let mut out = Vec::new();
|
||||
image::codecs::png::PngEncoder::new(&mut out)
|
||||
.write_image(img.as_raw(), 320, 180, image::ExtendedColorType::Rgb8)
|
||||
.unwrap();
|
||||
out
|
||||
}
|
||||
let base = frame_signature(&png(|_, _| Rgb([240, 240, 240]))).unwrap();
|
||||
// A panel clock flipping digits touches a couple of pixels only.
|
||||
let clock = frame_signature(&png(|x, y| {
|
||||
if x < 6 && y < 6 { Rgb([0, 0, 0]) } else { Rgb([240, 240, 240]) }
|
||||
}))
|
||||
.unwrap();
|
||||
// A dialog covering a quarter of the screen.
|
||||
let dialog = frame_signature(&png(|x, y| {
|
||||
if x < 160 && y < 90 { Rgb([20, 20, 20]) } else { Rgb([240, 240, 240]) }
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(signatures_similar(&base, &clock));
|
||||
assert!(!signatures_similar(&base, &dialog));
|
||||
assert!(frame_signature(b"not an image").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_magic_sets_mime() {
|
||||
let observation = observation_from_png(vec![0xFF, 0xD8, 0xFF, 0x00], 1, 1, None, None);
|
||||
|
|
|
|||
|
|
@ -15,20 +15,36 @@ else
|
|||
PROFILE="$HOME/.browser-profiles/displays/${display}"
|
||||
fi
|
||||
mkdir -p "$PROFILE"
|
||||
# The bot drives this same window over DevTools (port 9221 + display number,
|
||||
# see crates/control/src/cdp.rs). Every launch path (boot, panel launcher,
|
||||
# xdg-open, launch_app) must open it, otherwise the model cannot attach to the
|
||||
# browser the human is looking at and would have to restart it.
|
||||
DEVTOOLS_PORT=$((9221 + display))
|
||||
live=$(pgrep -f "chromium.*--user-data-dir=${PROFILE}" 2>/dev/null | head -1)
|
||||
lock="$(readlink "$PROFILE/SingletonLock" 2>/dev/null || true)"
|
||||
lock_pid="${lock##*-}"
|
||||
if [ -n "$live" ]; then
|
||||
# A second Chromium on the same profile is the "Profile error occurred" dialog.
|
||||
# Only hand the URL to the existing process when its singleton is still valid.
|
||||
if [ -n "$lock_pid" ] && kill -0 "$lock_pid" 2>/dev/null; then
|
||||
exec /usr/bin/chromium --no-sandbox --user-data-dir="$PROFILE" "$@"
|
||||
# Only hand URLs to the existing process (flags are meaningless to it and an
|
||||
# empty argument list would open a stray New Tab in the user's window).
|
||||
urls=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
-*) ;;
|
||||
*) urls="$urls $arg" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$urls" ] && [ -n "$lock_pid" ] && kill -0 "$lock_pid" 2>/dev/null; then
|
||||
# shellcheck disable=SC2086
|
||||
exec /usr/bin/chromium --no-sandbox --user-data-dir="$PROFILE" $urls
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
rm -f "$PROFILE/SingletonLock" "$PROFILE/SingletonCookie" "$PROFILE/SingletonSocket"
|
||||
exec /usr/bin/chromium \
|
||||
--no-sandbox \
|
||||
--remote-debugging-port="$DEVTOOLS_PORT" \
|
||||
--remote-allow-origins='*' \
|
||||
--test-type \
|
||||
--disable-gpu \
|
||||
--disable-dev-shm-usage \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
-- Skills taught by demonstration: the human drives the bot's desktop while we
|
||||
-- record semantic events (DOM clicks, typed text, navigations) and keyframes,
|
||||
-- then a model distils that into an intent-level playbook the bot can follow.
|
||||
CREATE TABLE taught_skills (
|
||||
id TEXT PRIMARY KEY,
|
||||
space_id TEXT NOT NULL REFERENCES spaces (id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL,
|
||||
bot_id TEXT NOT NULL REFERENCES bots (id) ON DELETE CASCADE,
|
||||
thread_id TEXT,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
goal TEXT NOT NULL,
|
||||
-- recording | drafting | draft | saved | failed
|
||||
status TEXT NOT NULL,
|
||||
playbook JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
recording JSONB NOT NULL DEFAULT '{"events":[],"frames":[]}'::jsonb,
|
||||
error TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
expires_at TIMESTAMPTZ,
|
||||
stopped_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX taught_skills_one_recording_idx
|
||||
ON taught_skills (bot_id) WHERE status = 'recording';
|
||||
CREATE INDEX taught_skills_bot_status_idx ON taught_skills (bot_id, status);
|
||||
Loading…
Reference in New Issue