feat/test #2
|
|
@ -0,0 +1,51 @@
|
|||
我這邊體感滑，是因為多數工作走 Shell／API／瀏覽器 DOM，很少靠截圖猜 XY。你 repo 裡 瀏覽器這條其實已經對了；卡的是 桌面原生 App。
|
||||
|
||||
你已經有的（別拆）
|
||||
能力 在哪
|
||||
Docker 真實 Linux 桌面 + Screen／lease／takeover supervisor / sandbox / control
|
||||
shell 工具 crates/api/src/tools.rs
|
||||
browser（CDP DOM snapshot／click by id／selector） control/cdp.rs + cdp.py；工具描述已寫「網頁優先用 browser」
|
||||
編號元素、apply_element_targets contracts/action.rs、control/actions.rs
|
||||
畫面沒變就不重傳圖（frame signature） control/observe.rs
|
||||
原生動作執行：xdotool / wmctrl control/x11.rs、image/computer/Dockerfile
|
||||
這已經比純「截圖 computer-use」產品完整一截。
|
||||
|
||||
現在實際主路徑（為什麼不順）
|
||||
computer_observe → 截圖 +（瀏覽器時）CDP 元素
|
||||
→（否則）只有 wmctrl 視窗級元素
|
||||
computer_act → element id 先換成中心點 XY → xdotool 點像素
|
||||
|
||||
關鍵缺口：
|
||||
|
||||
P0｜沒有 AT-SPI／無障礙樹
|
||||
Dockerfile 有 xdotool、wmctrl，沒有 at-spi2／python3-pyatspi 之類。
|
||||
原生「elements」在 window_list_command 只是 整窗標題，不是按鈕／輸入框。模型對 Thunar、對話框、面板仍要看圖或瞎點。
|
||||
|
||||
P0｜element 最終還是 XY
|
||||
apply_element_targets 填的是中心座標，執行仍走 ComputerAction::Pointer → xdotool。
|
||||
視窗一動、動畫、縮放就歪。agent-desktop／Cua 是 對 element 做 AX／AT-SPI action，不是點中心像素。
|
||||
|
||||
P1｜ComputerAction 契約只有像素／鍵
|
||||
action.rs 沒有 ActByRef { id }、SetValue、A11yPress 這類語意動作。擴 CDP／a11y 都會卡在契約層。
|
||||
|
||||
P1｜控的是容器桌面，不是你 Mac
|
||||
對「操控我電腦」若指本機 Finder／系統設定，現在這條 stack 接不到 Mac AX。那是另一條 provider（agent-desktop／Cua），不是再調 xdotool。
|
||||
|
||||
P2｜模型路由依賴 prompt
|
||||
tools 已寫「Chromium 用 browser」，但若模型仍狂 computer_observe＋座標，體感就跟我差一截。缺硬性策略（例如：有 CDP page 時禁止對 browser 窗做 computer_act click）。
|
||||
|
||||
跟「我為什麼滑」的對照
|
||||
路徑 我（Grok Bot） LazyBoy feat/test
|
||||
Shell／檔案 預設 有
|
||||
瀏覽器 DOM／element ref 有 CDP，方向對
|
||||
桌面原生 另有 a11y／委派；截圖是備援 截圖 + xdotool 仍是主幹
|
||||
本機 Mac 受限／核准 Shell 尚未做 Host provider
|
||||
所以：你換好幾套「控制」若不順，多半都還停在 視覺／座標層；你 repo 差的是把 AT-SPI（容器內） 接到 observe／act，並讓 browser／shell 硬優先。
|
||||
|
||||
建議下一步（只做這 3 件）
|
||||
容器加 AT-SPI：image 裝 at-spi；observe 回傳 button／textfield 樹（精簡 YAML／JSON + stable id）；act 對 id 做 DoAction／set text，失敗才 fallback XY。
|
||||
擴 ComputerAction／UiElement：kind: a11y|dom|window，ref 語意動作；computer_act 有 selector／a11y ref 時不要先轉座標。
|
||||
路由硬規則：Chromium 前台 → 強制 browser；能 shell 完成 → 禁止截圖迴圈；並加「連續同點無畫面變化 → 強制換策略」（你已有一點 stale-click 提示，可再硬一點）。
|
||||
本機 Mac 若也要滑：另開 HostComputerProvider（agent-desktop／Cua），別硬塞進現在的 X11 xdotool 路徑。
|
||||
|
||||
要的話我可以直接幫你開一張 P0 AT-SPI observe/act 的實作規格（改哪些檔、契約長怎樣），或直接在 feat/test 上開工。
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons";
|
||||
import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons";
|
||||
import UseAnimations from "react-useanimations";
|
||||
import loading from "react-useanimations/lib/loading";
|
||||
import loading2 from "react-useanimations/lib/loading2";
|
||||
|
|
@ -18,7 +18,7 @@ import searchToX from "react-useanimations/lib/searchToX";
|
|||
import { api, ApiError } from "./api";
|
||||
import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar";
|
||||
import { t, type MessageKey } from "./i18n";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types";
|
||||
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types";
|
||||
|
||||
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false};
|
||||
const SESSION_STORE="lazyboy.sessionByBot";
|
||||
|
|
@ -67,7 +67,7 @@ export function App(){
|
|||
const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null);
|
||||
const [roomToDelete,setRoomToDelete]=useState<Room|null>(null); const [mcpServers,setMcpServers]=useState<McpServer[]>([]);
|
||||
const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState<AccountDialog>(null);
|
||||
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false);
|
||||
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState<string|null>(null);
|
||||
const [workspaceName,setWorkspaceName]=useState(workspaceStart.name);
|
||||
const [looks,setLooks]=useState(readAvatarLooks);
|
||||
const sendingRef=useRef(false); const refreshSeqRef=useRef(0);
|
||||
|
|
@ -119,7 +119,7 @@ export function App(){
|
|||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
|
||||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]);
|
||||
useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]);
|
||||
useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]);
|
||||
useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]);
|
||||
useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]);
|
||||
useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]);
|
||||
useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]);
|
||||
|
|
@ -137,6 +137,9 @@ export function App(){
|
|||
async function saveSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}))}
|
||||
async function testSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(async()=>{await api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook})});await api(`/api/skills/${skill.id}/test`,{method:"POST",body:"{}"})})}
|
||||
async function discardSkill(skill:TaughtSkill){await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}))}
|
||||
async function updateSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}));setEditingSkillId(null)}
|
||||
async function deleteSkill(skill:TaughtSkill){if(!window.confirm(t("deleteSkillConfirm",{name:skill.name||skill.goal})))return;await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}));setEditingSkillId(null)}
|
||||
const editingSkill=editingSkillId?skills.find(skill=>skill.id===editingSkillId)||null:null;
|
||||
function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}執行「${skill.name}」`);document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus()}
|
||||
async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();if((!active&&!activeRoom)||!activeSessionId||!text)return;sendingRef.current=true;setDraft("");historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});sentHistoryRef.current.push(text);if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();await loadSessions()})}finally{sendingRef.current=false}}
|
||||
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
|
||||
|
|
@ -202,10 +205,10 @@ export function App(){
|
|||
|
||||
<main className="chat-panel">
|
||||
<header className="topbar"><button className="icon-button mobile-menu" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="#dfdfe2"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}<span className="message-body">{message.body}</span>{message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{workingMembers.map(member=><div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-label">{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}</span></div>)}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}<span className="message-body">{message.body}</span>{message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{workingMembers.map(member=><div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-label">{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}</span></div>)}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
|
||||
{otherSessionBusy&&<div className="queue-hint">{t("anotherConversationQueued")}</div>}
|
||||
<form className="composer" onSubmit={send}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled title={t("attachmentsUnavailable")}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button>{savedSkills.length>0&&<><hr/><small className="plus-menu-label">{t("taughtSkills")}</small>{savedSkills.map(skill=><button type="button" role="menuitem" key={skill.id} title={skill.playbook.whenToUse||skill.goal} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button>)}</>}</div>}</div><textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||!draft.trim()||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
<form className="composer" onSubmit={send}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled title={t("attachmentsUnavailable")}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button>{savedSkills.length>0&&<><hr/><small className="plus-menu-label">{t("taughtSkills")}</small>{savedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}</>}</div>}</div><textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||!draft.trim()||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
|
||||
</main>
|
||||
|
||||
{!rightCollapsed&&<div className="side-card-backdrop" onClick={()=>setRightCollapsed(true)}/>}
|
||||
|
|
@ -230,6 +233,7 @@ export function App(){
|
|||
|
||||
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.busyBotName?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" title={t("moreActions")}><Ellipsis/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen">{frame}</div>{error&&<div className="overlay-error">{error}</div>}</div>}
|
||||
{teachOpen&&active&&<TeachDialog bot={active} busy={busy} close={()=>setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>}
|
||||
{editingSkill&&<SkillEditDialog key={editingSkill.id} skill={editingSkill} busy={busy} close={()=>setEditingSkillId(null)} save={(name,playbook)=>void updateSkill(editingSkill,name,playbook)} test={(name,playbook)=>void testSkill(editingSkill,name,playbook)} remove={()=>void deleteSkill(editingSkill)}/>}
|
||||
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{document.querySelectorAll<HTMLIFrameElement>(".desktop-frame").forEach(frame=>frame.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin));setClipboardOpen(false)}}/>}
|
||||
|
||||
{createOpen&&<CreateDialog close={()=>setCreateOpen(false)} created={async bot=>{setCreateOpen(false);await loadBots();setActiveId(bot.id)}}/>}
|
||||
|
|
@ -452,7 +456,7 @@ function TeachDialog({bot,busy,close,start}:{bot:Bot;busy:boolean;close:()=>void
|
|||
</form></div>;
|
||||
}
|
||||
function stepText(step:unknown):string{if(typeof step==="string")return step;if(step&&typeof step==="object"&&"do" in step)return String((step as {do:unknown}).do||"");return ""}
|
||||
function SkillDraftCard({skill,busy,onSave,onTest,onDiscard}:{skill:TaughtSkill;busy:boolean;onSave:(name:string,playbook:Playbook)=>void;onTest:(name:string,playbook:Playbook)=>void;onDiscard:()=>void}){
|
||||
function SkillDraftCard({skill,busy,onSave,onTest,onDiscard,onEdit}:{skill:TaughtSkill;busy:boolean;onSave:(name:string,playbook:Playbook)=>void;onTest:(name:string,playbook:Playbook)=>void;onDiscard:()=>void;onEdit:()=>void}){
|
||||
const [name,setName]=useState(skill.name||skill.playbook.name||"");
|
||||
const [stepsText,setStepsText]=useState((skill.playbook.steps||[]).map(step=>stepText(step)).join("\n"));
|
||||
const [expanded,setExpanded]=useState(false);
|
||||
|
|
@ -467,9 +471,43 @@ function SkillDraftCard({skill,busy,onSave,onTest,onDiscard}:{skill:TaughtSkill;
|
|||
<label className="skill-steps">{t("skillSteps")}{expanded?<textarea rows={Math.min(12,Math.max(4,stepsText.split("\n").length+1))} value={stepsText} onChange={e=>setStepsText(e.target.value)}/>:<ol onClick={()=>setExpanded(true)}>{stepsText.split("\n").filter(Boolean).slice(0,6).map((line,index)=><li key={index}>{line}</li>)}{stepsText.split("\n").filter(Boolean).length>6&&<li className="more">{t("moreSteps",{count:stepsText.split("\n").filter(Boolean).length-6})}</li>}</ol>}{!expanded&&<button type="button" className="link" onClick={()=>setExpanded(true)}>{t("editSteps")}</button>}</label>
|
||||
{skill.playbook.howToCheck&&<p className="skill-check">{t("skillCheck")}：{skill.playbook.howToCheck}</p>}
|
||||
{skill.error&&<p className="skill-error">{t("skillDistillFailed")}</p>}
|
||||
<div className="skill-actions"><button type="button" className="outline danger-ghost" disabled={busy} onClick={onDiscard}>{t("discard")}</button><span className="grow"/><button type="button" className="outline" disabled={busy||!name.trim()} onClick={()=>onTest(name.trim(),build())}>{t("testRun")}</button><button type="button" className="primary" disabled={busy||!name.trim()} onClick={()=>onSave(name.trim(),build())}>{t("saveSkill")}</button></div>
|
||||
<div className="skill-actions"><button type="button" className="outline danger-ghost" disabled={busy} onClick={onDiscard}>{t("discard")}</button><button type="button" className="link" disabled={busy} onClick={onEdit}><Pencil/>{t("editSkillFull")}</button><span className="grow"/><button type="button" className="outline" disabled={busy||!name.trim()} onClick={()=>onTest(name.trim(),build())}>{t("testRun")}</button><button type="button" className="primary" disabled={busy||!name.trim()} onClick={()=>onSave(name.trim(),build())}>{t("saveSkill")}</button></div>
|
||||
</section>;
|
||||
}
|
||||
const lines=(text:string)=>text.split("\n").map(line=>line.replace(/^\s*(?:\d+[.、)]|[-•*])\s*/,"").trim()).filter(Boolean);
|
||||
function stepsToText(steps:(PlaybookStep|string)[]|undefined){return (steps||[]).map(step=>{if(typeof step==="string")return step;return step.expect?`${step.do} → ${step.expect}`:step.do}).join("\n")}
|
||||
function textToSteps(text:string,previous:(PlaybookStep|string)[]|undefined):PlaybookStep[]{const old=(previous||[]).filter((step):step is PlaybookStep=>typeof step==="object"&&step!==null);return lines(text).map(line=>{const [doPart,...rest]=line.split(/\s*(?:→|->|=>)\s*/);const expect=rest.join(" → ").trim();const match=old.find(step=>step.do===doPart.trim());return {do:doPart.trim(),expect:expect||(match&&!rest.length?match.expect:undefined)||"",note:match?.note||""}})}
|
||||
function inputsToText(inputs:PlaybookInput[]|undefined){return (inputs||[]).map(input=>`${input.name}${input.description?`：${input.description}`:""}${input.example?`｜例：${input.example}`:""}`).join("\n")}
|
||||
function textToInputs(text:string):PlaybookInput[]{return lines(text).map(line=>{const [head,...exampleParts]=line.split(/\s*[|｜]\s*/);const example=exampleParts.join("｜").replace(/^例[:：]\s*/,"").trim();const [name,...descParts]=head.split(/[:：]/);return {name:name.trim(),description:descParts.join("：").trim()||undefined,example:example||undefined}}).filter(input=>input.name)}
|
||||
function SkillEditDialog({skill,busy,close,save,test,remove}:{skill:TaughtSkill;busy:boolean;close:()=>void;save:(name:string,playbook:Playbook)=>void;test:(name:string,playbook:Playbook)=>void;remove:()=>void}){
|
||||
const p=skill.playbook;
|
||||
const [name,setName]=useState(skill.name||p.name||"");
|
||||
const [intent,setIntent]=useState(p.intent||"");
|
||||
const [whenToUse,setWhenToUse]=useState(p.whenToUse||"");
|
||||
const [inputs,setInputs]=useState(inputsToText(p.inputs));
|
||||
const [preconditions,setPreconditions]=useState((p.preconditions||[]).join("\n"));
|
||||
const [steps,setSteps]=useState(stepsToText(p.steps));
|
||||
const [howToCheck,setHowToCheck]=useState(p.howToCheck||"");
|
||||
const [whatToReturn,setWhatToReturn]=useState(p.whatToReturn||"");
|
||||
const [cautions,setCautions]=useState((p.cautions||[]).join("\n"));
|
||||
const build=():Playbook=>({...p,name:name.trim(),intent:intent.trim(),whenToUse:whenToUse.trim(),inputs:textToInputs(inputs),preconditions:lines(preconditions),steps:textToSteps(steps,p.steps),howToCheck:howToCheck.trim(),whatToReturn:whatToReturn.trim(),cautions:lines(cautions)});
|
||||
const valid=name.trim().length>0&&lines(steps).length>0;
|
||||
const rows=(text:string,min:number,max:number)=>Math.min(max,Math.max(min,text.split("\n").length+1));
|
||||
return <div className="modal-backdrop" onClick={close}><form className="dialog skill-edit" onClick={e=>e.stopPropagation()} onSubmit={e=>{e.preventDefault();if(valid)save(name.trim(),build())}}>
|
||||
<div className="dialog-title"><h2><Sparkle/>{t("editSkill")}</h2><button type="button" onClick={close}><X/></button></div>
|
||||
<p className="dialog-lead">{t("skillEditLead")}</p>
|
||||
<label>{t("skillName")}<input autoFocus value={name} maxLength={40} onChange={e=>setName(e.target.value)} placeholder={t("skillNamePlaceholder")}/></label>
|
||||
<label>{t("skillIntent")}<textarea rows={rows(intent,2,5)} value={intent} onChange={e=>setIntent(e.target.value)}/></label>
|
||||
<label>{t("skillWhenToUse")}<input value={whenToUse} onChange={e=>setWhenToUse(e.target.value)}/></label>
|
||||
<label>{t("skillInputsLabel")}<textarea rows={rows(inputs,2,5)} value={inputs} onChange={e=>setInputs(e.target.value)} placeholder={t("skillInputsPlaceholder")}/></label>
|
||||
<label>{t("skillPreconditions")}<textarea rows={rows(preconditions,2,5)} value={preconditions} onChange={e=>setPreconditions(e.target.value)}/></label>
|
||||
<label>{t("skillStepsLabel")}<textarea rows={rows(steps,5,14)} value={steps} onChange={e=>setSteps(e.target.value)} placeholder={t("skillStepsPlaceholder")}/></label>
|
||||
<label>{t("skillHowToCheck")}<textarea rows={rows(howToCheck,2,4)} value={howToCheck} onChange={e=>setHowToCheck(e.target.value)}/></label>
|
||||
<label>{t("skillWhatToReturn")}<input value={whatToReturn} onChange={e=>setWhatToReturn(e.target.value)}/></label>
|
||||
<label>{t("skillCautions")}<textarea rows={rows(cautions,2,5)} value={cautions} onChange={e=>setCautions(e.target.value)}/></label>
|
||||
<div className="dialog-actions"><button type="button" className="outline danger-ghost" disabled={busy} onClick={remove}>{t("deleteSkill")}</button><span className="grow"/><button type="button" className="outline" disabled={busy} onClick={close}>{t("cancel")}</button><button type="button" className="outline" disabled={busy||!valid} onClick={()=>test(name.trim(),build())}>{t("testRun")}</button><button className="primary" disabled={busy||!valid}>{t("saveSkill")}</button></div>
|
||||
</form></div>;
|
||||
}
|
||||
function ClipboardDialog({close,paste}:{close:()=>void;paste:(text:string)=>void}){const[text,setText]=useState("");return <div className="modal-backdrop"><div className="dialog compact"><div className="dialog-title"><h2>{t("pasteToRemoteComputer")}</h2><button onClick={close}><X/></button></div><p>{t("pasteRemoteHelp")}</p><textarea className="clipboard-text" autoFocus value={text} onChange={e=>setText(e.target.value)} placeholder={t("pasteTextPlaceholder")}/><div className="dialog-actions"><button className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={!text} onClick={()=>paste(text)}>{t("pasteIntoVnc")}</button></div></div></div>}
|
||||
function CreateDialog({close,created}:{close:()=>void;created:(bot:Bot)=>void}){const[name,setName]=useState("");const[mode,setMode]=useState<ComputerMode>("team");const[busy,setBusy]=useState(false);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();if(!name.trim())return;setBusy(true);try{created(await api<Bot>("/api/bots",{method:"POST",body:JSON.stringify({name:name.trim(),computerMode:mode})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addBot")}</h2><button type="button" onClick={close}><X/></button></div><label>{t("name")}<input autoFocus value={name} onChange={e=>setName(e.target.value)} placeholder={t("botNamePlaceholder")}/></label><div className="mode-grid"><button type="button" className={mode==="team"?"picked":""} onClick={()=>setMode("team")}><BotIcon/><strong>{t("sharedComputer")}</strong><small>{t("sharedComputerHint")}</small></button><button type="button" className={mode==="dedicated"?"picked":""} onClick={()=>setMode("dedicated")}><Computer/><strong>{t("privateComputer")}</strong><small>{t("privateComputerHint")}</small></button></div><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()}>{t("create")}</button></div></form></div>}
|
||||
function CreateGroupDialog({bots,close,created}:{bots:Bot[];close:()=>void;created:(room:Room)=>void}){const[name,setName]=useState("");const[selected,setSelected]=useState<string[]>([]);const[busy,setBusy]=useState(false);const visible=bots.filter(bot=>!bot.hidden);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();const groupName=name.trim();if(!groupName||selected.length<2)return;setBusy(true);try{created(await api<Room>("/api/rooms",{method:"POST",body:JSON.stringify({name:groupName,memberIds:selected})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addGroup")}</h2><button type="button" onClick={close}><X/></button></div><p className="dialog-lead">{t("groupDescription")}</p><label>{t("groupName")}<input autoFocus value={name} maxLength={30} onChange={e=>setName(e.target.value)} placeholder={t("groupNamePlaceholder")}/></label><fieldset className="group-picker"><legend>{t("chooseBots")}</legend>{visible.map(bot=><label key={bot.id}><input type="checkbox" checked={selected.includes(bot.id)} onChange={()=>setSelected(ids=>ids.includes(bot.id)?ids.filter(id=>id!==bot.id):[...ids,bot.id])}/><Avatar lookId={bot.id} name={bot.name} color={bot.avatarColor} shape={bot.avatarShape} online/><span>{bot.name}</span></label>)}</fieldset><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()||selected.length<2}>{busy?t("creating"):t("createGroup")}</button></div></form></div>}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { Animation } from "react-useanimations/utils";
|
|||
import activity from "react-useanimations/lib/activity";
|
||||
import archive from "react-useanimations/lib/archive";
|
||||
import star from "react-useanimations/lib/star";
|
||||
import edit from "react-useanimations/lib/edit";
|
||||
import airplay from "react-useanimations/lib/airplay";
|
||||
import arrowDown from "react-useanimations/lib/arrowDown";
|
||||
import arrowRightCircle from "react-useanimations/lib/arrowRightCircle";
|
||||
|
|
@ -50,6 +51,7 @@ export const Megaphone=animatedIcon(notification,16);
|
|||
export const Paperclip=animatedIcon(archive,16);
|
||||
export const Pin=animatedIcon(pocket,14);
|
||||
export const Sparkle=animatedIcon(star,16);
|
||||
export const Pencil=animatedIcon(edit,14);
|
||||
export const Plug=animatedIcon(toggle,16);
|
||||
export const Plus=animatedIcon(plusToX,17);
|
||||
export const RefreshCw=animatedIcon(skipBack,14);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ export const zhTW = {
|
|||
skillDraftTitle: "新技能草稿", skillDraftFrom: "從 {count} 個動作整理而來", skillName: "技能名稱", skillNamePlaceholder: "例如：完成 STAR 訓練課程",
|
||||
skillInputs: "可變輸入", skillSteps: "步驟（依意圖描述，可修改）", moreSteps: "…還有 {count} 步", editSteps: "編輯步驟", skillCheck: "怎麼確認完成",
|
||||
skillDistillFailed: "模型整理失敗，這是依動作直接列出的版本，建議修改後再儲存。", discard: "捨棄", testRun: "試跑", saveSkill: "儲存技能", messageTo: "傳訊息給 {name}", chooseConversationFirst: "先選擇對話", stopConversation: "停止對話",
|
||||
editSkill: "編輯技能", editSkillFull: "完整編輯", runSkillNamed: "執行「{name}」", skillEditLead: "寫給機器人看的說明，用意圖與畫面上的文字描述，不要寫座標。每行一項。",
|
||||
skillIntent: "意圖（這個技能要達成什麼）", skillWhenToUse: "使用時機", skillInputsLabel: "可變輸入（每行：名稱：說明｜例：範例值）", skillInputsPlaceholder: "courseName：課程名稱｜例：【TW】Workplace Bullying Prevention (2026)",
|
||||
skillPreconditions: "前提（開始前要成立的條件）", skillStepsLabel: "步驟（每行一步，可用 → 接上預期看到的畫面）", skillStepsPlaceholder: "點「Next」翻頁 → 頁碼加一；若 Next 鎖住就等它啟用", skillHowToCheck: "怎麼確認完成", skillWhatToReturn: "做完要回報什麼", skillCautions: "注意事項（每行一項）",
|
||||
deleteSkill: "刪除技能", deleteSkillConfirm: "確定刪除技能「{name}」？機器人之後就不會再認得它。", skillSaved: "已儲存",
|
||||
collapseSidebar: "收合側欄", botComputer: "{name} 的電腦", dedicatedScreen: "獨立螢幕", enlarge: "放大",
|
||||
userControlling: "你正在控制", aiReadOnly: "AI 操作中（唯讀）", readOnly: "唯讀", pasteClipboard: "貼上剪貼簿", copyDesktopClipboard: "複製桌面剪貼簿", moreActions: "更多操作",
|
||||
unpin: "取消釘選", pin: "釘選", markUnread: "標示為未讀", enterGroupName: "輸入分組名稱", changeGroup: "變更分組", createOrMoveGroup: "建立／移入分組", removeFromGroup: "移出分組", unhide: "取消隱藏", hide: "隱藏", delete: "刪除",
|
||||
|
|
|
|||
|
|
@ -305,3 +305,20 @@
|
|||
.skill-error{color:#fca5a5;font-size:12.5px}
|
||||
.skill-actions{display:flex;align-items:center;gap:8px}
|
||||
.skill-actions .grow{flex:1}
|
||||
.skill-actions .link{display:inline-flex;align-items:center;gap:5px}
|
||||
.skill-actions .link svg{width:13px;height:13px}
|
||||
/* Saved-skill rows in the composer menu: run on the left, pencil on the right */
|
||||
.plus-menu-skill{display:flex;align-items:center;gap:2px}
|
||||
.plus-menu-skill>button:first-child{flex:1;min-width:0}
|
||||
.plus-menu-skill .skill-edit{flex:0 0 30px;width:30px;height:30px;padding:0;justify-content:center;color:var(--muted)}
|
||||
.plus-menu-skill .skill-edit:hover{color:var(--ink)}
|
||||
.plus-menu-skill .skill-edit svg{width:14px;height:14px;flex-basis:14px}
|
||||
/* Full skill editor */
|
||||
.dialog.skill-edit{width:min(600px,calc(100vw - 28px));max-height:calc(100dvh - 28px);overflow:auto}
|
||||
.dialog.skill-edit h2{display:flex;align-items:center;gap:9px}
|
||||
.dialog.skill-edit h2 svg{width:17px;height:17px;color:var(--accent)}
|
||||
.dialog.skill-edit label{display:grid;gap:6px;color:var(--muted);font-size:12.5px}
|
||||
.dialog.skill-edit input,.dialog.skill-edit textarea{width:100%;border:1px solid var(--border);border-radius:10px;background:#0c0c0d;color:var(--ink);padding:9px 12px;outline:0;font:inherit;font-size:13.5px;line-height:1.5;resize:vertical}
|
||||
.dialog.skill-edit input:focus,.dialog.skill-edit textarea:focus{border-color:#4b4b50}
|
||||
.dialog.skill-edit .dialog-actions{align-items:center;gap:8px}
|
||||
.dialog.skill-edit .dialog-actions .grow{flex:1}
|
||||
|
|
|
|||
|
|
@ -947,7 +947,11 @@ pub async fn idle_loop(state: AppState) {
|
|||
for computer in rows {
|
||||
let active: Result<Option<(i64,)>, _> = sqlx::query_as(
|
||||
"SELECT 1 FROM runs WHERE status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||||
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1) LIMIT 1",
|
||||
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
|
||||
UNION ALL
|
||||
SELECT 1 FROM taught_skills WHERE status IN ('recording','drafting')
|
||||
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
|
||||
LIMIT 1",
|
||||
)
|
||||
.bind(&computer.id)
|
||||
.fetch_optional(state.pool())
|
||||
|
|
|
|||
|
|
@ -32,9 +32,17 @@ Prefer the fast path, in this order:
|
|||
When you use the browser tool:
|
||||
- snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate.
|
||||
- Yellow numbered marks on the screenshot match the element list. Click the number, not guessed pixels.
|
||||
- If the control is not in the element list, take a fresh snapshot or scroll; it may be off-screen or not rendered yet.
|
||||
- Elements tagged [below viewport ...] / [above viewport ...] are outside the visible area but still clickable by id; the click scrolls to them. Do not scroll manually just to reach them.
|
||||
- A control is disabled only when its entry says [disabled]. Never claim a button is disabled, counting down or loading unless the element list or the screenshot shows that.
|
||||
- Clicking a [disabled] control is fine: the click waits up to 45s for it to become enabled (stay timers, short videos) and then clicks. So when Next is disabled, just click Next. Only if the click reports it is still disabled, call wait with a longer time and click again.
|
||||
- Element ids are renumbered whenever the page changes; after navigation use the ids from the newest result, not older ones.
|
||||
- If the control is not in the element list at all, take a fresh snapshot; it may not be rendered yet.
|
||||
|
||||
When a click changes nothing: do not repeat it. A button is often disabled until a video, timer or page load finishes; call wait (up to 60s) and re-observe, then click by element id. Pages that need patience (training videos, quizzes, slow forms) are normal: keep working through them step by step and report progress in one short sentence when done.
|
||||
When a click changes nothing: do not repeat it blindly. Take a fresh snapshot, read the [disabled] tags and the page text, then act. Pages that need patience (training videos, quizzes, slow forms) are normal: keep working through them step by step and report progress in one short sentence when done.
|
||||
|
||||
Waiting is a tool call, never a reply. Ending your turn with \"waiting for X\" stops the whole run; nobody resumes it. If something must finish first, call wait (or click, which waits) and continue.
|
||||
|
||||
Multi-step tasks and taught skills: you are done only when the playbook's check passes (for example the course shows completed, the form shows a confirmation). Do not stop with a status sentence in the middle; keep calling tools until the check passes or you are truly blocked, then say exactly why. Never repeat an earlier reply word for word; describe the current screen.
|
||||
|
||||
Call request_takeover only for passwords, 2FA, CAPTCHA, payment, or a decision only the human can make. Never call it just because a click missed. When you do call it, say exactly what the human must do.
|
||||
|
||||
|
|
@ -490,13 +498,29 @@ async fn execute_run(
|
|||
} else {
|
||||
vec![UserContent::text(prompt)]
|
||||
};
|
||||
let mut skill_check: Option<String> = None;
|
||||
if !resume_after_takeover {
|
||||
// The user named a taught skill: hand the model the full playbook up
|
||||
// front so it does not have to guess or call use_skill first.
|
||||
if let Some(skill) = crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await {
|
||||
first.push(UserContent::text(crate::skills::format_playbook_for_run(&skill)));
|
||||
skill_check = Some(crate::skills::skill_check_hint(&skill));
|
||||
}
|
||||
}
|
||||
let mut earlier_replies: Vec<String> = assistant_texts(&history);
|
||||
if skill_check.is_some() {
|
||||
// A skill run is self-contained. Old chat turns about the same site
|
||||
// ("Next is still counting down" x6) otherwise anchor the model into
|
||||
// repeating its past conclusions instead of reading the screen.
|
||||
history.clear();
|
||||
first.push(UserContent::text(
|
||||
"Earlier chat history is intentionally omitted for this skill run. Work only from the playbook above and the current screen.",
|
||||
));
|
||||
}
|
||||
// Taught skills run long (a 24-page course is 24 clicks); plain chats stay
|
||||
// bounded tighter so a confused model cannot burn budget for as long.
|
||||
let max_turns = if skill_check.is_some() { 80 } else { 40 };
|
||||
let mut nudges: u8 = 0;
|
||||
let mut screenshots: u32 = 0;
|
||||
let mut screenshot_bytes: u64 = 0;
|
||||
if resume_after_takeover && ctx.gui_block.is_none() {
|
||||
|
|
@ -574,7 +598,7 @@ async fn execute_run(
|
|||
preamble.push_str(&index);
|
||||
}
|
||||
|
||||
for _ in 0..40 {
|
||||
for _ in 0..max_turns {
|
||||
turns += 1;
|
||||
if let Some(halt) = renew_or_halt(state, run_id, lease_owner).await? {
|
||||
return finish_halt(
|
||||
|
|
@ -633,7 +657,45 @@ async fn execute_run(
|
|||
"model turn"
|
||||
);
|
||||
if calls.is_empty() {
|
||||
break;
|
||||
// A model that quits a playbook early, or parrots an earlier reply
|
||||
// instead of describing the current screen, gets pushed back to
|
||||
// the tools a couple of times before we accept the text.
|
||||
let parroted = earlier_replies
|
||||
.iter()
|
||||
.any(|earlier| earlier == final_text.trim());
|
||||
let nudge = if parroted {
|
||||
Some(
|
||||
"Your reply repeats an earlier message word for word, so it cannot describe the current screen. Below is what the screen shows RIGHT NOW. Act on it with a tool call. Waiting is done by calling wait or by clicking the control (the click waits for it to enable), never by replying. Reply in text only once the task is finished or you are truly blocked (say why).".to_string(),
|
||||
)
|
||||
} else {
|
||||
skill_check.as_ref().map(|check| {
|
||||
format!(
|
||||
"The run is not finished; your text reply ended nothing but your own turn. Check: {check}\nBelow is the current screen. If the next control is [disabled], click it anyway — the click waits up to 45s for it to enable — or call wait. Ids marked [below viewport] scroll automatically. Only reply in text when the check passes or you are truly blocked, and then say exactly what blocks you."
|
||||
)
|
||||
})
|
||||
};
|
||||
match nudge {
|
||||
Some(text) if nudges < 6 && turns + 2 < max_turns => {
|
||||
nudges += 1;
|
||||
tracing::info!(run_id, turn = turns, parroted, "nudging model back to tools");
|
||||
earlier_replies.push(final_text.trim().to_string());
|
||||
final_text.clear();
|
||||
let mut content = vec![UserContent::text(text)];
|
||||
if ctx.gui_block.is_none() {
|
||||
set_run_step(state, run_id, "computer_observe: 重新確認畫面").await;
|
||||
let outcome = dispatch(&ctx, "computer_observe", &json!({})).await;
|
||||
content.push(UserContent::text(outcome.text));
|
||||
if let Some(image) = outcome.image {
|
||||
screenshot_bytes += image.len() as u64;
|
||||
screenshots += 1;
|
||||
content.extend(screenshot_parts(image));
|
||||
}
|
||||
}
|
||||
pending = Message::User { content };
|
||||
continue;
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
final_text.clear();
|
||||
let mut results = Vec::new();
|
||||
|
|
@ -677,12 +739,12 @@ async fn execute_run(
|
|||
.await;
|
||||
}
|
||||
outcome = tokio::time::timeout(
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(150),
|
||||
dispatch(&ctx, &name, &call.function.arguments),
|
||||
) => match outcome {
|
||||
Ok(outcome) => outcome,
|
||||
Err(_) => crate::tools::ToolOutcome {
|
||||
text: format!("工具 {name} 執行逾時（90 秒），請稍後重試。"),
|
||||
text: format!("工具 {name} 執行逾時（150 秒），請稍後重試。"),
|
||||
image: None,
|
||||
pause: false,
|
||||
},
|
||||
|
|
@ -927,6 +989,27 @@ fn has_screenshot(message: &Message) -> bool {
|
|||
/// if it carries a fresh frame, otherwise the most recent one already in
|
||||
/// history. Without the fallback a "(screen unchanged)" turn would leave the
|
||||
/// model with no picture of the desktop at all.
|
||||
/// Plain-text bodies of the assistant turns in a history, used to catch a model
|
||||
/// that answers by repeating an earlier reply instead of reading the screen.
|
||||
fn assistant_texts(history: &[Message]) -> Vec<String> {
|
||||
history
|
||||
.iter()
|
||||
.filter_map(|message| match message {
|
||||
Message::Assistant { content, .. } => Some(
|
||||
content
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
AssistantContent::Text(text) => Some(text.text.trim().to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<String>(),
|
||||
),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|text| !text.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn drop_history_screenshots(history: &mut [Message], pending: &Message) {
|
||||
let keep = if has_screenshot(pending) {
|
||||
None
|
||||
|
|
|
|||
|
|
@ -1368,10 +1368,28 @@ pub fn format_playbook_for_run(skill: &SavedSkill) -> String {
|
|||
if !cautions.is_empty() {
|
||||
out.push_str(&format!("\nCautions: {}", cautions.join("; ")));
|
||||
}
|
||||
out.push_str("\nExecution rules: observe or snapshot before each step; find controls by their label/text/role on the current page and click by element id; if a step's expectation is not met, wait and re-observe once, then try the equivalent control; never replay coordinates from memory; if login, 2FA, CAPTCHA or payment appears, request_takeover and say what the human must do. When finished, report in one or two sentences what was done and what you saw.");
|
||||
out.push_str("\nExecution rules: observe or snapshot before each step; find controls by their label/text/role on the current page and click by element id (controls tagged [below viewport] are clickable too); if a step's expectation is not met, wait and re-observe once, then try the equivalent control; never replay coordinates from memory; if login, 2FA, CAPTCHA or payment appears, request_takeover and say what the human must do. Keep going until \"How to check\" is satisfied — a course with 24 pages means 24 Next clicks. When finished, report in one or two sentences what was done and what you saw.");
|
||||
out
|
||||
}
|
||||
|
||||
/// The completion criterion the run loop uses to push a model that stops
|
||||
/// early back to work.
|
||||
pub(crate) fn skill_check_hint(skill: &SavedSkill) -> String {
|
||||
skill
|
||||
.playbook
|
||||
.get("howToCheck")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|check| !check.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| {
|
||||
format!(
|
||||
"the goal of「{}」is visibly achieved on the current screen",
|
||||
skill.name
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|||
},
|
||||
ToolDefinition {
|
||||
name: "browser".into(),
|
||||
description: "Control Chromium through the page DOM. Prefer this over computer_act for anything in the browser. snapshot returns numbered elements and visible text (no screenshot). click/type/navigate by element id or CSS selector. The human still sees the live window.".into(),
|
||||
description: "Control Chromium through the page DOM. Prefer this over computer_act for anything in the browser. snapshot returns numbered elements and visible text (no screenshot). click/type/navigate by element id or CSS selector. click scrolls off-screen elements into view and, if the control is [disabled], waits up to 45s (waitMs to change) for it to enable before clicking. Ids are renumbered after every page change. The human still sees the live window.".into(),
|
||||
parameters: json!({
|
||||
"type":"object",
|
||||
"properties":{
|
||||
|
|
@ -132,7 +132,8 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|||
"text":{"type":"string"},
|
||||
"key":{"type":"string"},
|
||||
"url":{"type":"string"},
|
||||
"ms":{"type":"number"}
|
||||
"ms":{"type":"number"},
|
||||
"waitMs":{"type":"number","description":"click only: how long to wait for a disabled control to enable (default 45000, max 120000)"}
|
||||
},
|
||||
"required":["action"]
|
||||
}),
|
||||
|
|
@ -453,6 +454,17 @@ async fn cdp_snapshot(ctx: &ToolCtx, ensure: bool) -> Option<CdpPage> {
|
|||
|
||||
async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
|
||||
let display = ctx.context.display.as_deref().unwrap_or(":1");
|
||||
// Clicks may sit through a page's stay timer (CLICK_WAIT_MS in cdp.py).
|
||||
let timeout_ms = if request.get("action").and_then(Value::as_str) == Some("click") {
|
||||
let wait = request
|
||||
.get("waitMs")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(45_000)
|
||||
.min(120_000);
|
||||
wait + 25_000
|
||||
} else {
|
||||
20_000
|
||||
};
|
||||
let argv = cdp_command_on(display, ctx.context.profile_path.as_deref(), &request);
|
||||
match ctx
|
||||
.sandbox
|
||||
|
|
@ -461,7 +473,7 @@ async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
|
|||
CommandRequest {
|
||||
argv,
|
||||
cwd: None,
|
||||
timeout_ms: Some(20_000),
|
||||
timeout_ms: Some(timeout_ms),
|
||||
},
|
||||
&ctx.context,
|
||||
)
|
||||
|
|
@ -495,7 +507,7 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
"action": action,
|
||||
"ensure": true,
|
||||
});
|
||||
for key in ["url", "text", "key", "ms", "selector"] {
|
||||
for key in ["url", "text", "key", "ms", "selector", "waitMs"] {
|
||||
if let Some(value) = args.get(key) {
|
||||
request[key] = value.clone();
|
||||
}
|
||||
|
|
@ -537,13 +549,27 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
));
|
||||
}
|
||||
let page = cdp_call(ctx, request).await;
|
||||
if !page.ok {
|
||||
return text_outcome(page.error.unwrap_or_else(|| "browser failed".into()));
|
||||
}
|
||||
if !page.elements.is_empty() {
|
||||
*ctx.elements.lock().unwrap() = page.elements.clone();
|
||||
}
|
||||
let text = browser_result_text(action, &page);
|
||||
if !page.ok {
|
||||
let mut text = page.error.unwrap_or_else(|| "browser failed".into());
|
||||
if !page.elements.is_empty() {
|
||||
text.push_str(&format!(
|
||||
"\nPage: {} {}\nClickable page elements: {}",
|
||||
page.title,
|
||||
page.url,
|
||||
format_ui_elements(&page.elements)
|
||||
));
|
||||
}
|
||||
return text_outcome(text);
|
||||
}
|
||||
let mut text = browser_result_text(action, &page);
|
||||
if let Some(seconds) = page.waited_seconds {
|
||||
text.push_str(&format!(
|
||||
" (the control was disabled; waited {seconds:.0}s for it to enable before clicking)"
|
||||
));
|
||||
}
|
||||
if action == "snapshot" {
|
||||
return ToolOutcome {
|
||||
text,
|
||||
|
|
|
|||
|
|
@ -101,6 +101,12 @@ impl UiElement {
|
|||
self.y.saturating_add(self.h / 2),
|
||||
)
|
||||
}
|
||||
|
||||
/// Page controls outside the viewport are reported with zero size: they
|
||||
/// can be clicked through the DOM but have no pixel position.
|
||||
pub fn is_offscreen(&self) -> bool {
|
||||
self.w == 0 && self.h == 0 && self.selector.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ pub enum ActionError {
|
|||
BadCoordinate(&'static str),
|
||||
#[error("computer action element {0} is not on the current screen")]
|
||||
UnknownElement(u32),
|
||||
#[error(
|
||||
"element {0} is outside the browser viewport, so it has no screen position. Use browser {{\"action\":\"click\",\"element\":{0}}} (it scrolls into view) or scroll the page first."
|
||||
)]
|
||||
OffscreenElement(u32),
|
||||
}
|
||||
|
||||
/// Models write element ids as `3`, `3.0`, `"3"` or `"[3]"`; accept them all
|
||||
|
|
@ -54,6 +58,9 @@ pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Resul
|
|||
let Some(element) = elements.iter().find(|element| u64::from(element.id) == id) else {
|
||||
return Err(ActionError::UnknownElement(id as u32));
|
||||
};
|
||||
if element.is_offscreen() {
|
||||
return Err(ActionError::OffscreenElement(id as u32));
|
||||
}
|
||||
let (x, y) = element.center();
|
||||
action.insert("x".into(), json!(x));
|
||||
action.insert("y".into(), json!(y));
|
||||
|
|
@ -369,4 +376,20 @@ mod tests {
|
|||
ActionError::UnknownElement(9)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offscreen_page_elements_cannot_be_pixel_clicked() {
|
||||
let below_fold = UiElement {
|
||||
id: 3,
|
||||
title: "Next [below viewport]".into(),
|
||||
selector: Some("[data-lazyboy=\"3\"]".into()),
|
||||
kind: Some("dom".into()),
|
||||
..UiElement::default()
|
||||
};
|
||||
let mut actions = json!([{"kind": "click", "element": 3}]);
|
||||
assert_eq!(
|
||||
apply_element_targets(&mut actions, &[below_fold]).unwrap_err(),
|
||||
ActionError::OffscreenElement(3)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -172,34 +172,61 @@ SNAP_JS = r"""
|
|||
const sx0 = (window.screenX || 0) + Math.floor(chromeW / 2);
|
||||
const sy0 = (window.screenY || 0) + chromeH;
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
let n = 1;
|
||||
const inView = [];
|
||||
const offView = [];
|
||||
for (const el of document.querySelectorAll(sels)) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width < 4 || r.height < 4) continue;
|
||||
if (r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth) continue;
|
||||
const st = getComputedStyle(el);
|
||||
if (st.visibility === "hidden" || st.display === "none" || Number(st.opacity) === 0) continue;
|
||||
const text = (el.innerText || el.value || el.getAttribute("aria-label") || el.getAttribute("placeholder") || el.getAttribute("name") || el.tagName)
|
||||
const type = (el.getAttribute("type") || "").toLowerCase();
|
||||
let text;
|
||||
if (el.tagName === "INPUT" && (type === "radio" || type === "checkbox")) {
|
||||
// Quiz answers: the value is usually "on"; the label next to it is what
|
||||
// the model must read to pick the right option.
|
||||
const owner = (el.labels && el.labels[0]) || el.closest("label") || el.parentElement;
|
||||
const label = (owner && owner.innerText || el.getAttribute("aria-label") || el.value || "").replace(/\s+/g, " ").trim().slice(0, 70);
|
||||
text = type + " " + label + (el.checked ? " (checked)" : "");
|
||||
} else {
|
||||
text = (el.innerText || el.value || el.getAttribute("aria-label") || el.getAttribute("placeholder") || el.getAttribute("name") || el.tagName)
|
||||
.replace(/\s+/g, " ").trim().slice(0, 80);
|
||||
if (!text) continue;
|
||||
}
|
||||
if (!text.trim()) continue;
|
||||
const key = [el.tagName, text, Math.round(r.x), Math.round(r.y)].join("|");
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
el.setAttribute("data-lazyboy", String(n));
|
||||
if (el.disabled || el.getAttribute("aria-disabled") === "true") text += " [disabled]";
|
||||
const visible = !(r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth);
|
||||
if (visible) {
|
||||
inView.push({el, text, x: Math.max(0, Math.round(sx0 + r.x)), y: Math.max(0, Math.round(sy0 + r.y)), w: Math.round(r.width), h: Math.round(r.height)});
|
||||
} else {
|
||||
// Controls outside the viewport are still clickable by id: the click
|
||||
// handler scrolls them into view. Zero size tells the desktop side not
|
||||
// to paint or pixel-click them.
|
||||
const where = r.top > innerHeight ? "below" : r.bottom < 0 ? "above" : "beside";
|
||||
// Buttons and form controls (Next, Submit, radios) matter more than the
|
||||
// hundredth body link, so they win the limited off-screen slots.
|
||||
const link = el.tagName === "A" || el.getAttribute("role") === "link";
|
||||
offView.push({el, text: text + " [" + where + " viewport]", x: 0, y: 0, w: 0, h: 0, rank: (link ? 1 : 0), dist: Math.abs(r.top > innerHeight ? r.top - innerHeight : r.bottom)});
|
||||
}
|
||||
}
|
||||
offView.sort((a, b) => a.rank - b.rank || a.dist - b.dist);
|
||||
const out = [];
|
||||
let n = 1;
|
||||
for (const item of inView.slice(0, 50).concat(offView.slice(0, 20))) {
|
||||
item.el.setAttribute("data-lazyboy", String(n));
|
||||
out.push({
|
||||
id: n,
|
||||
title: text,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
title: item.text,
|
||||
tag: item.el.tagName.toLowerCase(),
|
||||
selector: '[data-lazyboy="' + n + '"]',
|
||||
kind: "dom",
|
||||
x: Math.max(0, Math.round(sx0 + r.x)),
|
||||
y: Math.max(0, Math.round(sy0 + r.y)),
|
||||
w: Math.round(r.width),
|
||||
h: Math.round(r.height)
|
||||
x: item.x,
|
||||
y: item.y,
|
||||
w: item.w,
|
||||
h: item.h
|
||||
});
|
||||
n += 1;
|
||||
if (out.length >= 50) break;
|
||||
}
|
||||
const body = (document.body && document.body.innerText || "").replace(/\s+/g, " ").trim().slice(0, 3000);
|
||||
return {url: location.href, title: document.title || "", text: body, elements: out};
|
||||
|
|
@ -222,6 +249,39 @@ CLICK_JS = r"""
|
|||
}
|
||||
"""
|
||||
|
||||
STATE_JS = r"""
|
||||
(sel) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (!el) return {found: false};
|
||||
const disabled = !!el.disabled || el.getAttribute("aria-disabled") === "true";
|
||||
// Training sites explain the lock next to the button ("Please watch the
|
||||
// video", a countdown); surface that text so the model can decide how
|
||||
// long to wait.
|
||||
let hint = "";
|
||||
const near = el.parentElement && el.parentElement.parentElement;
|
||||
if (disabled && near) hint = (near.innerText || "").replace(/\s+/g, " ").trim().slice(0, 120);
|
||||
return {found: true, disabled, hint};
|
||||
}
|
||||
"""
|
||||
|
||||
# Pages often lock Next for a few seconds (stay timers) or until a video ends.
|
||||
# A human just waits and clicks; do the same instead of making the model plan
|
||||
# a wait/observe/click loop it tends to abandon.
|
||||
CLICK_WAIT_MS = 45000
|
||||
|
||||
def wait_until_enabled(ws, sel, wait_ms=None):
|
||||
budget = CLICK_WAIT_MS if wait_ms is None else max(0, min(int(wait_ms), 120000))
|
||||
started = time.time()
|
||||
while True:
|
||||
state = evaluate(ws, STATE_JS, sel) or {}
|
||||
if not state.get("found"):
|
||||
return None
|
||||
if not state.get("disabled"):
|
||||
return time.time() - started
|
||||
if (time.time() - started) * 1000 >= budget:
|
||||
return None
|
||||
time.sleep(0.5)
|
||||
|
||||
def evaluate(ws, expression, args=None):
|
||||
params = {"expression": expression, "returnByValue": True, "awaitPromise": True}
|
||||
if args is not None:
|
||||
|
|
@ -554,12 +614,27 @@ def main():
|
|||
sel = req.get("selector") or ""
|
||||
if not sel:
|
||||
fail("selector required")
|
||||
waited = wait_until_enabled(ws, sel, req.get("waitMs"))
|
||||
if waited is None:
|
||||
state = evaluate(ws, STATE_JS, sel) or {}
|
||||
if not state.get("found"):
|
||||
# Ids are renumbered whenever the page changes; hand back
|
||||
# the fresh numbering so the model does not have to ask.
|
||||
body = snapshot(ws)
|
||||
body.update({"ok": False, "error": "element gone: the page changed and ids were renumbered. Use the fresh element list in this result."})
|
||||
print(json.dumps(body))
|
||||
return
|
||||
fail("control %s is still disabled after waiting %ss (page says: %s). Use wait for longer if a video or timer must finish, then click again."
|
||||
% (sel, int(req.get("waitMs") or CLICK_WAIT_MS) // 1000, state.get("hint") or "nothing"))
|
||||
val = evaluate(ws, CLICK_JS, sel) or {}
|
||||
if not val.get("ok"):
|
||||
fail(val.get("error") or "click failed")
|
||||
pointer(display, val.get("x") or 0, val.get("y") or 0)
|
||||
wait_for_visual_update(ws)
|
||||
print(json.dumps({"ok": True, "action": "click", "selector": sel, "restarted": restarted}))
|
||||
out = {"ok": True, "action": "click", "selector": sel, "restarted": restarted}
|
||||
if waited >= 1.0:
|
||||
out["waitedSeconds"] = round(waited, 1)
|
||||
print(json.dumps(out))
|
||||
return
|
||||
if action == "type":
|
||||
sel = req.get("selector") or ""
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ pub fn cdp_record_stop_command(skill_id: &str) -> Vec<String> {
|
|||
vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)]
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct CdpPage {
|
||||
pub ok: bool,
|
||||
pub error: Option<String>,
|
||||
|
|
@ -88,25 +88,26 @@ pub struct CdpPage {
|
|||
pub title: String,
|
||||
pub text: String,
|
||||
pub restarted: bool,
|
||||
/// Seconds the click waited for a disabled control to become enabled.
|
||||
pub waited_seconds: Option<f64>,
|
||||
pub elements: Vec<UiElement>,
|
||||
}
|
||||
|
||||
pub fn parse_cdp_page(raw: &str) -> CdpPage {
|
||||
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
|
||||
if value.get("ok").and_then(Value::as_bool) != Some(true) {
|
||||
return CdpPage {
|
||||
ok: false,
|
||||
error: value
|
||||
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
|
||||
CdpPage {
|
||||
ok,
|
||||
error: if ok {
|
||||
None
|
||||
} else {
|
||||
value
|
||||
.get("error")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
.or_else(|| Some("cdp unavailable".into())),
|
||||
..CdpPage::default()
|
||||
};
|
||||
}
|
||||
CdpPage {
|
||||
ok: true,
|
||||
error: None,
|
||||
.or_else(|| Some("cdp unavailable".into()))
|
||||
},
|
||||
waited_seconds: value.get("waitedSeconds").and_then(Value::as_f64),
|
||||
url: value
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
|
|
|
|||
|
|
@ -56,7 +56,11 @@ pub fn overlay_elements(bytes: &[u8], elements: &[UiElement]) -> Vec<u8> {
|
|||
if width == 0 || height == 0 {
|
||||
return bytes.to_vec();
|
||||
}
|
||||
for element in elements.iter().take(50) {
|
||||
for element in elements
|
||||
.iter()
|
||||
.filter(|element| !element.is_offscreen())
|
||||
.take(50)
|
||||
{
|
||||
paint_element(&mut img, element, width, height);
|
||||
}
|
||||
encode_jpeg(&img).unwrap_or_else(|| bytes.to_vec())
|
||||
|
|
|
|||
Loading…
Reference in New Issue