diff --git a/a.txt b/a.txt new file mode 100644 index 0000000..aaa90fc --- /dev/null +++ b/a.txt @@ -0,0 +1,51 @@ +我這邊體感滑,是因為多數工作走 Shell/API/瀏覽器 DOM,很少靠截圖猜 XY。你 repo 裡 瀏覽器這條其實已經對了;卡的是 桌面原生 App。 + +你已經有的(別拆) +能力 在哪 +Docker 真實 Linux 桌面 + Screen/lease/takeover supervisor / sandbox / control +shell 工具 crates/api/src/tools.rs +browser(CDP DOM snapshot/click by id/selector) control/cdp.rs + cdp.py;工具描述已寫「網頁優先用 browser」 +編號元素、apply_element_targets contracts/action.rs、control/actions.rs +畫面沒變就不重傳圖(frame signature) control/observe.rs +原生動作執行:xdotool / wmctrl control/x11.rs、image/computer/Dockerfile +這已經比純「截圖 computer-use」產品完整一截。 + +現在實際主路徑(為什麼不順) +computer_observe → 截圖 +(瀏覽器時)CDP 元素 + →(否則)只有 wmctrl 視窗級元素 +computer_act → element id 先換成中心點 XY → xdotool 點像素 + +關鍵缺口: + +P0|沒有 AT-SPI/無障礙樹 +Dockerfile 有 xdotool、wmctrl,沒有 at-spi2/python3-pyatspi 之類。 +原生「elements」在 window_list_command 只是 整窗標題,不是按鈕/輸入框。模型對 Thunar、對話框、面板仍要看圖或瞎點。 + +P0|element 最終還是 XY +apply_element_targets 填的是中心座標,執行仍走 ComputerAction::Pointer → xdotool。 +視窗一動、動畫、縮放就歪。agent-desktop/Cua 是 對 element 做 AX/AT-SPI action,不是點中心像素。 + +P1|ComputerAction 契約只有像素/鍵 +action.rs 沒有 ActByRef { id }、SetValue、A11yPress 這類語意動作。擴 CDP/a11y 都會卡在契約層。 + +P1|控的是容器桌面,不是你 Mac +對「操控我電腦」若指本機 Finder/系統設定,現在這條 stack 接不到 Mac AX。那是另一條 provider(agent-desktop/Cua),不是再調 xdotool。 + +P2|模型路由依賴 prompt +tools 已寫「Chromium 用 browser」,但若模型仍狂 computer_observe+座標,體感就跟我差一截。缺硬性策略(例如:有 CDP page 時禁止對 browser 窗做 computer_act click)。 + +跟「我為什麼滑」的對照 +路徑 我(Grok Bot) LazyBoy feat/test +Shell/檔案 預設 有 +瀏覽器 DOM/element ref 有 CDP,方向對 +桌面原生 另有 a11y/委派;截圖是備援 截圖 + xdotool 仍是主幹 +本機 Mac 受限/核准 Shell 尚未做 Host provider +所以:你換好幾套「控制」若不順,多半都還停在 視覺/座標層;你 repo 差的是把 AT-SPI(容器內) 接到 observe/act,並讓 browser/shell 硬優先。 + +建議下一步(只做這 3 件) +容器加 AT-SPI:image 裝 at-spi;observe 回傳 button/textfield 樹(精簡 YAML/JSON + stable id);act 對 id 做 DoAction/set text,失敗才 fallback XY。 +擴 ComputerAction/UiElement:kind: a11y|dom|window,ref 語意動作;computer_act 有 selector/a11y ref 時不要先轉座標。 +路由硬規則:Chromium 前台 → 強制 browser;能 shell 完成 → 禁止截圖迴圈;並加「連續同點無畫面變化 → 強制換策略」(你已有一點 stale-click 提示,可再硬一點)。 +本機 Mac 若也要滑:另開 HostComputerProvider(agent-desktop/Cua),別硬塞進現在的 X11 xdotool 路徑。 + +要的話我可以直接幫你開一張 P0 AT-SPI observe/act 的實作規格(改哪些檔、契約長怎樣),或直接在 feat/test 上開工。 \ No newline at end of file diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 455e3d9..204e6ba 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,5 +1,5 @@ import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons"; +import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons"; import UseAnimations from "react-useanimations"; import loading from "react-useanimations/lib/loading"; import loading2 from "react-useanimations/lib/loading2"; @@ -18,7 +18,7 @@ import searchToX from "react-useanimations/lib/searchToX"; import { api, ApiError } from "./api"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { t, type MessageKey } from "./i18n"; -import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types"; +import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types"; const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; const SESSION_STORE="lazyboy.sessionByBot"; @@ -67,7 +67,7 @@ export function App(){ const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null); const [roomToDelete,setRoomToDelete]=useState(null); const [mcpServers,setMcpServers]=useState([]); const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState(null); - const [skills,setSkills]=useState([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false); + const [skills,setSkills]=useState([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState(null); const [workspaceName,setWorkspaceName]=useState(workspaceStart.name); const [looks,setLooks]=useState(readAvatarLooks); const sendingRef=useRef(false); const refreshSeqRef=useRef(0); @@ -119,7 +119,7 @@ export function App(){ useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); - useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); + useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]); useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]); useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]); @@ -137,6 +137,9 @@ export function App(){ async function saveSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}))} async function testSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(async()=>{await api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook})});await api(`/api/skills/${skill.id}/test`,{method:"POST",body:"{}"})})} async function discardSkill(skill:TaughtSkill){await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}))} + async function updateSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}));setEditingSkillId(null)} + async function deleteSkill(skill:TaughtSkill){if(!window.confirm(t("deleteSkillConfirm",{name:skill.name||skill.goal})))return;await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}));setEditingSkillId(null)} + const editingSkill=editingSkillId?skills.find(skill=>skill.id===editingSkillId)||null:null; function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}執行「${skill.name}」`);document.querySelector(".composer textarea")?.focus()} async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();if((!active&&!activeRoom)||!activeSessionId||!text)return;sendingRef.current=true;setDraft("");historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});sentHistoryRef.current.push(text);if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();await loadSessions()})}finally{sendingRef.current=false}} function composerKeyDown(event:ReactKeyboardEvent){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current
{activeRoom?<>member.id)}/>{activeRoom.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:active?<>{active.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:<>{t("chooseBot")}{topTools}}
-
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return
{spoken&&}{spoken&&{speakerName}}{message.body}{message.body.trim()&&}
})}{workingMembers.map(member=>
{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}
)}{pausedForUser&&paneBot&&
{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}}{computer.controlHolder==="user"?:}
}{teaching&&active&&
{t("teachingLive",{goal:teaching.goal})}{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}}
}{drafting&&
{t("distilling",{goal:drafting.goal})}
}{skillDraft&&active&&void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)}/>} +
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return
{spoken&&}{spoken&&{speakerName}}{message.body}{message.body.trim()&&}
})}{workingMembers.map(member=>
{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}
)}{pausedForUser&&paneBot&&
{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}}{computer.controlHolder==="user"?:}
}{teaching&&active&&
{t("teachingLive",{goal:teaching.goal})}{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}}
}{drafting&&
{t("distilling",{goal:drafting.goal})}
}{skillDraft&&active&&void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)}/>} {error&&
{error}
} {otherSessionBusy&&
{t("anotherConversationQueued")}
} -
event.stopPropagation()}>{plusOpen&&
{savedSkills.length>0&&<>
{t("taughtSkills")}{savedSkills.map(skill=>)}}
}