diff --git a/Makefile b/Makefile index 70482e1..5be5aa8 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ $(if $(filter 1,$(PUSH)),$(eval MULTI_FLAGS := --push)) cua-smoke prepare-screen-network \ build build-api build-supervisor build-controld \ fmt fmt-check clippy lint audit test clean \ - web \ + web web-dev web-watch \ dev dev-supervisor dev-api help: ## Show this help @@ -59,10 +59,12 @@ help: ## Show this help @echo " make test cargo test --workspace (DB tests need: make postgres)" @echo " make test-agent-computer Screenshot / locator / native-file contract tests" @echo " make bench-agent-computer Disposable shared/private native benchmark" - @echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)" + @echo " make web Production build of the frontend (needs node/npm)" + @echo " make web-dev Vite HMR at :5173 — edit CSS/React without rebuilding API" + @echo " make web-watch Rebuild dist on save; pair with docker-compose.web.yml on :3101" @echo " make clean cargo clean" @echo "" - @echo " After 'make up': open http://127.0.0.1:3101, register an account, and add a model API key in 設定 → 模型." + @echo " After 'make up': open http://127.0.0.1:3101. To iterate on the UI: make web-dev → http://127.0.0.1:5173" # --- Environment ----------------------------------------------------------- @@ -81,6 +83,7 @@ up: env ## Build every image and start the whole stack in Docker $(COMPOSE) up -d --build @echo "" @echo "stack launched. open http://127.0.0.1:3101 and register your account." + @echo "frontend iteration: make web-dev (http://127.0.0.1:5173, no image rebuild)" $(COMPOSE) ps # Compose owns `lazyboy_screen` (internal, labeled). A leftover from host-dev or @@ -265,9 +268,18 @@ test-agent-computer: ## Agent-computer contract tests (screenshot, locators, nat bench-agent-computer: ## Disposable shared/private native benchmark (no model or viewer claim) python3 scripts/bench-agent-computer.py -web: ## Build the frontend (needs node/npm) +web: ## Production build of the frontend (needs node/npm) cd $(WEB_DIR) && npm install && npm run build +web-dev: ## Vite HMR at http://127.0.0.1:5173; proxies /api and /view to :3101 + @echo "open http://127.0.0.1:5173 (API must already be on :3101; saving a file hot-reloads)" + cd $(WEB_DIR) && npm install && npm run dev + +web-watch: ## Rebuild apps/web/dist on save (for a Docker API that mounts dist) + @echo "writing $(WEB_DIR)/dist on each save. mount it with:" + @echo " docker compose -f docker-compose.yml -f docker-compose.web.yml up -d api" + cd $(WEB_DIR) && npm install && npm run watch + clean: ## Remove cargo build artifacts cargo clean diff --git a/README.md b/README.md index fa4e646..5880eeb 100644 --- a/README.md +++ b/README.md @@ -97,15 +97,13 @@ make dev-supervisor # terminal 1 make dev-api # terminal 2 ``` -Frontend hot reload in another terminal: +Iterate on the UI without rebuilding the API image: ```bash -cd apps/web -npm install -npm run dev +make web-dev ``` -Open [http://127.0.0.1:5173](http://127.0.0.1:5173). Tests and the tree layout are in the [development guide](./docs/development.md). +Open [http://127.0.0.1:5173](http://127.0.0.1:5173) (`/api` is proxied to `:3101`). Tests and the tree layout are in the [development guide](./docs/development.md). ## Docs diff --git a/README.zh-TW.md b/README.zh-TW.md index bdb0782..1f5b713 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -97,15 +97,13 @@ make dev-supervisor # 終端 1 make dev-api # 終端 2 ``` -前端熱更新使用另一個終端： +改前端請用熱更新，不必重建 API 映像： ```bash -cd apps/web -npm install -npm run dev +make web-dev ``` -開啟 [http://127.0.0.1:5173](http://127.0.0.1:5173)。測試指令與專案目錄說明請見 [開發指南](./docs/development.md)。 +開啟 [http://127.0.0.1:5173](http://127.0.0.1:5173)（`/api` 會轉到已在跑的 `:3101`）。測試指令與專案目錄說明請見 [開發指南](./docs/development.md)。 ## 文件 diff --git a/apps/web/package.json b/apps/web/package.json index 50adb75..902b445 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,7 +4,8 @@ "version": "0.1.0-alpha", "type": "module", "scripts": { - "dev": "vite --host 0.0.0.0", + "dev": "vite --host 0.0.0.0 --port 5173 --strictPort", + "watch": "vite build --watch", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit --pretty false" }, diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index d562d33..9967083 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,6 +1,6 @@ import { FormEvent, Fragment, KeyboardEvent as ReactKeyboardEvent, MouseEvent as ReactMouseEvent, TouchEvent as ReactTouchEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { createPortal } from "react-dom"; -import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Download, Ellipsis, Info, LogOut, MeetingLayout, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Reply, Settings, Smartphone, Sparkle, Square, Upload, Users, X } from "./animated-icons"; +import { BotIcon, Brain, ChevronDown, ChevronsLeft, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Download, Ellipsis, Info, LogOut, MeetingLayout, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Reply, Settings, Smartphone, Sparkle, Square, Upload, Users, X } from "./animated-icons"; import UseAnimations from "./use-animations"; import loading from "react-useanimations/lib/loading"; import arrowUp from "react-useanimations/lib/arrowUp"; @@ -14,16 +14,18 @@ import trash2 from "react-useanimations/lib/trash2"; import visibility from "react-useanimations/lib/visibility"; import visibility2 from "react-useanimations/lib/visibility2"; import searchToX from "react-useanimations/lib/searchToX"; +import { TaskStatus, type TaskSnapshot } from "./task-status"; import { api, ApiError } from "./api"; import { createCoalescer, subscribeToSession } from "./live"; -import { applyReplyEvent, type ReplyDrafts } from "./reply-stream"; +import { applyReplyEvent, replyStillStreaming, shownReplyText, visibleReplyDrafts, type ReplyDrafts } from "./reply-stream"; import { acceptMention, ensureMention, mentionChoices, mentionToken, replySnippet } from "./mentions"; import { clockTime, dayLabel, sameDay } from "./chat-time"; import { HANDOFF_MS, VEIL_FADE_MS, handoffRemaining, keepScreenUrl, nextVeil, viewOnlyFor, viewerPath, type Veil } from "./handoff"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n"; -import type { AvatarShape, Bot, ComputerMode, ComputerStatus, ComputerToolBinding, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, MemoryStatus, Message, MessageFile, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings } from "./types"; +import type { AvatarShape, Bot, BotPresence, ComputerMode, ComputerStatus, ComputerToolBinding, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, MemoryStatus, Message, MessageFile, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings } from "./types"; import { ChatMarkdown, MentionText, copyText } from "./markdown"; +import { ComputerFilesList, FilePreviewDialog, type PreviewTarget } from "./file-preview"; import { RunProbe, errorActions, errorTitle } from "./run-monitor"; import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, scheduleWhen, type CronPreset, type ScheduleItem } from "./schedule"; import { CallOverlay, PhoneIcon } from "./call"; @@ -49,12 +51,12 @@ const FALLBACK_POLL_MS=2000; // A heartbeat buys a 15 minute control lease and keeps the 10 minute idle // cutoff from parking the computer, so a minute between beats is a wide margin. const HEARTBEAT_MS=60000; -type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts"; +type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts"|"results"; type AccountDialog="phone"|"settings"|"model"|"voice"|"about"|"help"|"feedback"|null; function readSessionStore():Record{try{const raw=localStorage.getItem(SESSION_STORE);return raw?JSON.parse(raw) as Record:{}}catch{return {}}} function writeSessionStore(botId:string,sessionId:string){const store=readSessionStore();store[botId]=sessionId;localStorage.setItem(SESSION_STORE,JSON.stringify(store))} function isPhoneLayout(){return typeof window!=="undefined"&&window.matchMedia("(max-width:700px)").matches} -function readPaneStore():{collapsed:boolean;part:RightPart;meeting:boolean}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer",meeting:false};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string;meeting?:boolean};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"||value.part==="accounts"?value.part:"computer",meeting:Boolean(value.meeting)&&!isPhoneLayout()}}catch{return{collapsed:false,part:"computer",meeting:false}}} +function readPaneStore():{collapsed:boolean;part:RightPart;meeting:boolean;navCollapsed:boolean}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:true,part:"computer",meeting:false,navCollapsed:false};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string;meeting?:boolean;navCollapsed?:boolean};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"||value.part==="accounts"||value.part==="results"?value.part:"computer",meeting:Boolean(value.meeting)&&!isPhoneLayout(),navCollapsed:Boolean(value.navCollapsed)&&!isPhoneLayout()}}catch{return{collapsed:true,part:"computer",meeting:false,navCollapsed:false}}} type WorkspacePrefs={name:string;showHidden:boolean}; function isDefaultWorkspaceName(name?:string|null){return !name||name==="Local workspace"||name==="本機工作區"} function readWorkspace():WorkspacePrefs{try{const raw=localStorage.getItem(WORKSPACE_STORE);if(!raw)return{name:t("localWorkspace"),showHidden:false};const value=JSON.parse(raw) as {name?:string;showHidden?:boolean};const name=value.name?.trim();return{name:isDefaultWorkspaceName(name)?t("localWorkspace"):name||t("localWorkspace"),showHidden:Boolean(value.showHidden)}}catch{return{name:t("localWorkspace"),showHidden:false}}} @@ -124,14 +126,43 @@ function attachAllowed(file:File){const mime=(file.type||"").toLowerCase();if(mi function readAsBase64(file:File){return new Promise((resolve,reject)=>{const reader=new FileReader();reader.onload=()=>{const value=String(reader.result||"");const comma=value.indexOf(",");resolve(comma>=0?value.slice(comma+1):value)};reader.onerror=()=>reject(reader.error||new Error("read failed"));reader.readAsDataURL(file)})} function formatBytes(size:number){if(size<1024)return `${size} B`;if(size<1024*1024)return `${Math.round(size/102.4)/10} KB`;return `${Math.round(size/104857.6)/10} MB`} function fileExt(name:string){const dot=name.lastIndexOf(".");const ext=dot>=0?name.slice(dot+1).replace(/[^a-z0-9]/gi,""):"";return (ext||"FILE").slice(0,4).toUpperCase()} -function messageFiles(blocks:unknown):MessageFile[]{if(!Array.isArray(blocks))return [];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;name?:string;mimeType?:string;size?:number};if(value.kind!=="file"&&value.kind!=="image")return [];return [{kind:value.kind,name:value.name||"file",mimeType:value.mimeType,size:value.size}]})} -type MessageChip={kind:string;site?:string;why?:string;name?:string;human?:string;cron?:string;reason?:string;turns?:number;limit?:number;code?:string;retryable?:boolean;runId?:string;turn?:number;step?:string|null}; +function messageFiles(blocks:unknown):MessageFile[]{if(!Array.isArray(blocks))return [];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;name?:string;mimeType?:string;size?:number;path?:string;botId?:string;verified?:boolean};if(value.kind!=="file"&&value.kind!=="image")return [];return [{botId:value.botId,verified:value.verified,kind:value.kind,name:value.name||"file",mimeType:value.mimeType,size:value.size,path:typeof value.path==="string"&&value.path?value.path:undefined}]})} +type MessageChip={kind:string;intervention?:string;site?:string;why?:string;name?:string;human?:string;cron?:string;reason?:string;turns?:number;limit?:number;code?:string;retryable?:boolean;runId?:string;turn?:number;step?:string|null}; function chipBlocks(blocks:unknown){if(!Array.isArray(blocks))return [] as MessageChip[];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as MessageChip;if(value.kind==="login"||value.kind==="schedule"||value.kind==="scheduleRun"||value.kind==="resume"||value.kind==="error")return [value];return []})} function replyQuote(blocks:unknown){if(!Array.isArray(blocks))return null;for(const block of blocks){if(!block||typeof block!=="object")continue;const value=block as {kind?:string;name?:string;body?:string};if(value.kind==="reply")return {name:value.name||"",body:value.body||""}}return null} -function resumeTitle(reason?:string){return reason==="budget_exhausted"?t("resumeBudget"):reason==="loop_detected"?t("resumeLoop"):t("resumeMidTask")} +function resumeTitle(reason?:string){return reason==="budget_exhausted"?t("resumeBudget"):reason==="loop_detected"?t("resumeLoop"):reason==="needs_plan"?t("resumePlan"):t("resumeMidTask")} +/** + * Presence for every bot in the sidebar. + * + * The API is the only side that can see the worker lease, so `bot.presence` + * decides between idle, waiting and stalled. Local signals are folded on top + * for bots we are actively watching: the bot list is polled and can be a beat + * behind, so it may still claim a finished run is working, while a streaming + * draft or an SSE run id proves work is in flight right now. + */ +function botPresenceMap(bots:Bot[],workingMembers:RoomMember[],busyMembers:RoomMember[],computer:ComputerStatus,replyDrafts:ReplyDrafts,liveRuns:Record){ + const liveWorking=new Set(); + for(const member of workingMembers) liveWorking.add(member.id); + for(const member of busyMembers) liveWorking.add(member.id); + if(computer.busyRunId&&computer.botId) liveWorking.add(computer.botId); + for(const draft of Object.values(replyDrafts)) if(draft.botId&&!draft.messageId) liveWorking.add(draft.botId); + for(const botId of Object.values(liveRuns)) if(botId) liveWorking.add(botId); + const watched=new Set(liveWorking); + if(computer.botId) watched.add(computer.botId); + const map:Record={}; + for(const bot of bots){ + const reported=bot.presence||(bot.working?"working":"idle"); + const stale=reported==="working"&&watched.has(bot.id)&&!liveWorking.has(bot.id); + const state=stale?"idle":reported; + if(state!=="idle") map[bot.id]=state; + } + // `working` outranks every other state, so live evidence simply wins. + for(const id of liveWorking) if(!bots.some(bot=>bot.id===id&&bot.presence)) if(map[id]!=="stalled"&&map[id]!=="waiting") map[id]="working"; + return map; +} function isAutoAttachCaption(body:string,files:MessageFile[]){const text=body.trim();if(!files.length)return false;if(!text)return true;return files.some(file=>text===file.name||text===`附件 ${file.name}`||text===`Attached ${file.name}`||text===t("attachedFile",{name:file.name}))} function isAutoScheduleCaption(body:string,chips:{kind?:string}[]){if(!chips.some(chip=>chip.kind==="schedule"||chip.kind==="scheduleRun"))return false;const text=body.trim();return /^\[排程(試跑)?\]/.test(text)||/^\[Schedule( test)?\]/i.test(text)} -function FileCard({file,preview,onRemove}:{file:{name:string;size?:number};preview?:string|null;onRemove?:()=>void}){const ext=fileExt(file.name);return
{preview?:}
{file.name}{typeof file.size==="number"?formatBytes(file.size):ext}
{onRemove&&}
} +function FileCard({file,preview,onRemove,onOpen}:{file:{name:string;size?:number};preview?:string|null;onRemove?:()=>void;onOpen?:()=>void}){const ext=fileExt(file.name);const body=<>{preview?:}
{file.name}{typeof file.size==="number"?formatBytes(file.size):ext}
;return
{onOpen?:body}{onRemove&&}
} function clientNonce(){ const webCrypto=globalThis.crypto; if(webCrypto&&typeof webCrypto.randomUUID==="function")return webCrypto.randomUUID(); @@ -145,17 +176,20 @@ function clientNonce(){ } export function App(){ + const [tasks,setTasks]=useState([]); const locale=useLocale(); const [bots,setBots]=useState([]); const [rooms,setRooms]=useState([]); const [activeId,setActiveId]=useState(null); const [activeRoomId,setActiveRoomId]=useState(null); const [busyMembers,setBusyMembers]=useState([]); const [sessions,setSessions]=useState([]); const [activeSessionId,setActiveSessionId]=useState(null); const [messages,setMessages]=useState([]); const [computer,setComputer]=useState(blankComputer); const [screenUrl,setScreenUrl]=useState(null); const [draft,setDraft]=useState(""); const [query,setQuery]=useState(""); - const [createOpen,setCreateOpen]=useState(false); const [createMenuOpen,setCreateMenuOpen]=useState(false); const [groupOpen,setGroupOpen]=useState(false); const [deleteOpen,setDeleteOpen]=useState(false); const [computerOpen,setComputerOpen]=useState(false); + const [createOpen,setCreateOpen]=useState(false); const [createMenuOpen,setCreateMenuOpen]=useState(false); const [groupOpen,setGroupOpen]=useState(false); const [deleteOpen,setDeleteOpen]=useState(false); const [computerOpen,setComputerOpen]=useState(false); const [previewFile,setPreviewFile]=useState(null); const paneStart=readPaneStore(); const [rightCollapsed,setRightCollapsed]=useState(paneStart.collapsed||isPhoneLayout()); const [rightPart,setRightPart]=useState(paneStart.part); const [meetingMode,setMeetingMode]=useState(paneStart.meeting); + const [leftCollapsed,setLeftCollapsed]=useState(paneStart.navCollapsed); const [sessionMenuOpen,setSessionMenuOpen]=useState(false); const [clearOpen,setClearOpen]=useState(false); const [sessionToDelete,setSessionToDelete]=useState(null); const [remembered,setRemembered]=useState>({}); const [resumedChips,setResumedChips]=useState>({}); const [retriedRuns,setRetriedRuns]=useState>({}); const [mobileNav,setMobileNav]=useState(false); const [error,setError]=useState(null); const [busy,setBusy]=useState(false); + const [liveStep,setLiveStep]=useState<{botId:string;step:string}|null>(null); useEffect(()=>{ if(!mobileNav)return; const dismiss=(event:KeyboardEvent)=>{if(event.key==="Escape")setMobileNav(false)}; @@ -185,7 +219,8 @@ export function App(){ const currentSessionRef=useRef(activeSessionId);currentSessionRef.current=activeSessionId; const [sendingMessage,setSendingMessage]=useState(false); const [replyDrafts,setReplyDrafts]=useState({}); - const sendingRef=useRef(false); const refreshSeqRef=useRef(0); const importRef=useRef(null); const attachRef=useRef(null); + const [liveRuns,setLiveRuns]=useState>({}); + const sendingRef=useRef(false); const refreshSeqRef=useRef(0); const computerSeqRef=useRef(0); const endedRunsRef=useRef(new Set()); const importRef=useRef(null); const attachRef=useRef(null); const desktopFrameRef=useRef(null); const holderRef=useRef(computer.controlHolder); const paneBotRef=useRef(null); const skipHandoffRef=useRef(true); const [desktopReady,setDesktopReady]=useState(false); const [handingOff,setHandingOff]=useState(false); @@ -199,6 +234,9 @@ export function App(){ const sentHistoryRef=useRef([]); const historyIndexRef=useRef(null); const historyDraftRef=useRef(""); const roomsRef=useRef(rooms); roomsRef.current=rooms; + const botsRef=useRef(bots); + botsRef.current=bots; + const openBotRef=useRef<(bot:Bot)=>void>(()=>{}); const filtered=useMemo(()=>bots.filter(b=>(showHidden||!b.hidden)&&b.name.toLowerCase().includes(query.toLowerCase())),[bots,query,showHidden]); const filteredRooms=useMemo(()=>{const q=query.trim().toLowerCase();return rooms.filter(room=>!q||room.name.toLowerCase().includes(q)||room.members.some(member=>member.name.toLowerCase().includes(q)))},[rooms,query]); const sections=useMemo(()=>{const map=new Map();for(const bot of filtered){const key=bot.pinned?t("pinned"):bot.groupName||t("agentGroup");map.set(key,[...(map.get(key)||[]),bot])}return [...map.entries()]},[filtered]); @@ -214,7 +252,10 @@ export function App(){ const currentPaneRef=useRef(paneBotId);currentPaneRef.current=paneBotId; const pasteQueueRef=useRef>(Promise.resolve()); const [clipboardStatus,setClipboardStatus]=useState(""); - const workingMembers=activeRoom?busyMembers:active&&activeSessionId&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[]; + const liveWorkerId=liveStep?.botId||Object.values(replyDrafts).find(draft=>!draft.messageId)?.botId; + const roomFallback=activeRoom&&liveWorkerId?activeRoom.members.filter(member=>member.id===liveWorkerId):[]; + const workingMembers=activeRoom?(busyMembers.length?busyMembers:roomFallback):active&&activeSessionId&&(computer.busySessionId===activeSessionId||replyStillStreaming(replyDrafts))?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[]; + const presenceById=useMemo(()=>botPresenceMap(bots,workingMembers,busyMembers,computer,replyDrafts,liveRuns),[bots,workingMembers,busyMembers,computer,replyDrafts,liveRuns]); const lastMessageId=messages[messages.length-1]?.id||""; // The bot is parked in waiting_takeover: nothing moves (including queued // messages) until the human releases the screen, so say so loudly. @@ -258,34 +299,40 @@ export function App(){ const sessionStoreKey=activeRoomId?`room:${activeRoomId}`:activeId; const sessionsPath=activeRoomId?`/api/rooms/${activeRoomId}/sessions`:activeId?`/api/bots/${activeId}/sessions`:null; const loadSessions=useCallback(async()=>{if(!sessionsPath){setSessions([]);setActiveSessionId(null);return}const next=await api(sessionsPath);setSessions(next);setActiveSessionId(id=>{if(id&&next.some(session=>session.id===id))return id;const stored=sessionStoreKey?readSessionStore()[sessionStoreKey]:undefined;if(stored&&next.some(session=>session.id===stored))return stored;return next[0]?.id||null})},[sessionsPath,sessionStoreKey]); - const refresh=useCallback(async()=>{if(!activeSessionId)return;const refreshSeq=++refreshSeqRef.current;let nextBusy:RoomMember[]=[]; - // Transcript delivery is independent of desktop health and screen discovery. - // Resolve errors as values until joined below, so an early rejection cannot - // become unhandled while the computer request is still in flight. - const messagesJob=api(`/api/sessions/${activeSessionId}/messages`).then(next=>{if(refreshSeq===refreshSeqRef.current)setMessages(next);return null},error=>error); + useEffect(()=>{setTasks([])},[activeSessionId]); + const refreshTranscript=useCallback(async()=>{ + if(!activeSessionId)return; + const seq=++refreshSeqRef.current; + const [next,taskList]=await Promise.all([api(`/api/sessions/${activeSessionId}/messages`),api(`/api/sessions/${activeSessionId}/task`).catch(()=>null)]); + if(seq===refreshSeqRef.current){setMessages(next);if(taskList)setTasks(taskList.map(task=>({...task,receivedAt:Date.now()})))} + },[activeSessionId]); + const refreshComputer=useCallback(async()=>{ + if(!activeSessionId)return; + const seq=++computerSeqRef.current; + let nextBusy:RoomMember[]=[]; let computerBot=activeId; if(activeRoomId){ - try{const roomStatus=await api<{busy:RoomMember[]}>(`/api/rooms/${activeRoomId}/status`);if(refreshSeq!==refreshSeqRef.current)return;nextBusy=roomStatus.busy;computerBot=roomStatus.busy[0]?.id||roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null}catch{if(refreshSeq!==refreshSeqRef.current)return;computerBot=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null} + try{const roomStatus=await api<{busy:RoomMember[]}>(`/api/rooms/${activeRoomId}/status`);if(seq!==computerSeqRef.current)return;nextBusy=roomStatus.busy;computerBot=roomStatus.busy[0]?.id||roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null}catch{if(seq!==computerSeqRef.current)return;computerBot=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null} } - if(!computerBot){const messageError=await messagesJob;if(messageError)throw messageError;if(refreshSeq===refreshSeqRef.current){setBusyMembers(nextBusy);setComputer(blankComputer);setSkills([]);setScreenUrl(null)}return} - const skillsJob=activeRoomId?Promise.resolve([] as TaughtSkill[]):api(`/api/bots/${computerBot}/skills`).catch(()=>[] as TaughtSkill[]); - const status=await api(`/api/computer/${computerBot}/status`);if(refreshSeq!==refreshSeqRef.current)return; - const [messageError,screen,nextSkills]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null}),skillsJob]); - if(refreshSeq!==refreshSeqRef.current)return; - if(messageError)throw messageError; - setBusyMembers(nextBusy);setComputer(status);setSkills(nextSkills); + if(!computerBot){if(seq===computerSeqRef.current){setBusyMembers(nextBusy);setComputer(blankComputer);setScreenUrl(null)}return} + const status=await api(`/api/computer/${computerBot}/status`);if(seq!==computerSeqRef.current)return; + const screen=status.state==="running"?await api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):{url:null}; + if(seq!==computerSeqRef.current)return; + setBusyMembers(nextBusy);setComputer(status); // A missing url is not a dead desktop. While it boots or wakes the mounted // frame keeps its VNC session instead of blacking out and reconnecting. setScreenUrl(current=>status.botId===computerBot?keepScreenUrl(current,screen.url,status.state):null); - // Agreement with the server is what ends a handoff. The beat has a floor so - // the swap is still visible on a LAN, and the ceiling timer still covers a - // reply that never lands. if(expectedHolderRef.current&&status.controlHolder===expectedHolderRef.current){ expectedHolderRef.current=null; const wait=handoffRemaining(handoffAtRef.current,Date.now()); if(wait>0)window.setTimeout(()=>setHandingOff(false),wait);else setHandingOff(false); } },[activeId,activeRoomId,activeSessionId]); + const refresh=useCallback(async()=>{ + const skillsJob=!activeRoomId&&activeId?api(`/api/bots/${activeId}/skills`).catch(()=>[] as TaughtSkill[]):Promise.resolve([] as TaughtSkill[]); + const [skills]=await Promise.all([skillsJob,refreshTranscript(),refreshComputer()]); + setSkills(skills); + },[activeId,activeRoomId,refreshTranscript,refreshComputer]); useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(localizeError(e.message))})},[loadBots]); useEffect(()=>{if(!voiceSettings?.enabled)setCallOpen(false)},[voiceSettings?.enabled]); useEffect(()=>{api("/api/voice/settings").then(setVoiceSettings).catch(()=>setVoiceSettings(null))},[]); @@ -295,25 +342,43 @@ export function App(){ useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current="";setReplyTarget(null);setMessageMenu(null);setPendingFiles(current=>{current.forEach(file=>file.preview&&URL.revokeObjectURL(file.preview));return []})},[activeSessionId]); useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]); useEffect(()=>{const panel=messageEndRef.current?.parentElement;if(panel&&panel.scrollHeight-panel.scrollTop-panel.clientHeight<180)panel.scrollTop=panel.scrollHeight},[replyDrafts]); - useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]); + useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]); + useEffect(()=>{if(computer.busySessionId===activeSessionId||replyStillStreaming(replyDrafts))return;setLiveStep(null)},[computer.busySessionId,activeSessionId,replyDrafts]); // Live transcript: the event stream says when to look, so a reply appears as // soon as it is written. Polling stays as a backstop that cannot starve the // database, and a hidden tab stops re-reading until it comes back to the // foreground - which is itself a "what happened" moment worth refreshing on. useEffect(()=>{ - if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return} + if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;computerSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return} setReplyDrafts({}); + setLiveStep(null); + setLiveRuns({}); + endedRunsRef.current=new Set(); setScreenUrl(null); refresh().catch(e=>setError(localizeError(e.message))); - const settle=createCoalescer(()=>{if(!document.hidden)refresh().catch(()=>{})},EVENT_SETTLE_MS); + const settle=createCoalescer(()=>{if(!document.hidden)Promise.all([refreshTranscript(),refreshComputer()]).catch(()=>{})},EVENT_SETTLE_MS); let live=false,opened=false,timer=0; - const tick=()=>{if(!document.hidden)refresh().catch(()=>{});timer=window.setTimeout(tick,live?LIVE_POLL_MS:FALLBACK_POLL_MS)}; + const tick=()=>{if(!document.hidden)(live?refreshComputer:refresh)().catch(()=>{});timer=window.setTimeout(tick,live?LIVE_POLL_MS:FALLBACK_POLL_MS)}; timer=window.setTimeout(tick,FALLBACK_POLL_MS); const heartbeat=window.setInterval(()=>{const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},HEARTBEAT_MS); - const feed=subscribeToSession(activeSessionId,event=>{setReplyDrafts(current=>applyReplyEvent(current,event));if(!event.kind.startsWith("reply."))settle.kick()},{onStatus:connected=>{live=connected;if(connected&&opened)refresh().catch(()=>{});opened=true}}); + const feed=subscribeToSession(activeSessionId,event=>{ + setReplyDrafts(current=>applyReplyEvent(current,event)); + const runId=typeof event.payload.runId==="string"?event.payload.runId:""; + if(event.kind==="reply.started"&&runId)endedRunsRef.current.delete(runId); + if(event.kind==="run.started"&&runId){ + const botId=typeof event.payload.botId==="string"?event.payload.botId:""; + if(botId)setLiveRuns(current=>({...current,[runId]:botId})); + } + if(event.kind==="tool.started"&&typeof event.payload.step==="string"&&typeof event.payload.botId==="string"&&(!runId||!endedRunsRef.current.has(runId)))setLiveStep({botId:event.payload.botId,step:event.payload.step}); + if(["run.completed","run.failed","run.paused","run.cancelled"].includes(event.kind)){ + if(runId){endedRunsRef.current.add(runId);setLiveRuns(current=>{const next={...current};delete next[runId];return next})} + setLiveStep(null); + } + if(event.kind!=="tool.started"&&!event.kind.startsWith("reply."))settle.kick(); + },{onStatus:connected=>{live=connected;if(connected&&opened)refresh().catch(()=>{});opened=true}}); const resume=()=>{if(!document.hidden)refresh().catch(()=>{})}; document.addEventListener("visibilitychange",resume); - return()=>{window.clearTimeout(timer);window.clearInterval(heartbeat);document.removeEventListener("visibilitychange",resume);feed.close();settle.cancel();refreshSeqRef.current+=1}; + return()=>{window.clearTimeout(timer);window.clearInterval(heartbeat);document.removeEventListener("visibilitychange",resume);feed.close();settle.cancel();refreshSeqRef.current+=1;computerSeqRef.current+=1}; },[activeId,activeRoomId,activeSessionId,refresh]); useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&desktopInteractive)void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&desktopInteractive)setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]); @@ -331,19 +396,31 @@ export function App(){ useEffect(()=>{setWorkspaceName(name=>isDefaultWorkspaceName(name)?t("localWorkspace"):name)},[locale]); useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]); useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]); - useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart,meeting:meetingMode}))},[rightCollapsed,rightPart,meetingMode]); + useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart,meeting:meetingMode,navCollapsed:leftCollapsed}))},[rightCollapsed,rightPart,meetingMode,leftCollapsed]); useEffect(()=>{const mq=window.matchMedia("(max-width:700px)");const stop=()=>{if(!mq.matches)return;setMeetingMode(false);setRightCollapsed(true)};stop();mq.addEventListener("change",stop);return()=>mq.removeEventListener("change",stop)},[]); useEffect(()=>{if(!paneBotId){setSchedules([]);return}api(`/api/bots/${paneBotId}/schedules`).then(setSchedules).catch(()=>setSchedules([]))},[paneBotId]); - useEffect(()=>{if(!computer.takeoverRequested)return;setRightPart("computer");if(isPhoneLayout())setComputerOpen(true);else setRightCollapsed(false)},[computer.takeoverRequested]); + const beforeTakeover=useRef<{part:RightPart;collapsed:boolean;open:boolean}|null>(null); + useEffect(()=>{if(!computer.takeoverRequested)return;if(!beforeTakeover.current)beforeTakeover.current={part:rightPart,collapsed:rightCollapsed,open:computerOpen};setRightPart("computer");if(isPhoneLayout())setComputerOpen(true);else setRightCollapsed(false)},[computer.takeoverRequested]); function openPane(part:RightPart){ setMobileNav(false); - if(part==="computer"&&isPhoneLayout()){setRightPart("computer");setRightCollapsed(true);setComputerOpen(true);return} - if(meetingMode&&part==="computer"){setRightPart("computer");setRightCollapsed(true);return} - setRightPart(part);setRightCollapsed(false); + setMeetingMode(false); + if(part==="computer"&&isPhoneLayout()){setRightPart("computer");setRightCollapsed(true);setComputerOpen(!computerOpen);return} + setComputerOpen(false); + setRightPart(part); + setRightCollapsed(!meetingMode&&!rightCollapsed&&rightPart===part); } - function toggleMeeting(){if(isPhoneLayout())return;setMeetingMode(on=>{if(!on){setComputerOpen(false);setRightPart("computer");setRightCollapsed(false)}return !on})} + function toggleMeeting(){ + if(isPhoneLayout())return; + setMobileNav(false); + setComputerOpen(false); + setRightPart("computer"); + setRightCollapsed(false); + setMeetingMode(!meetingMode); + } + function toggleNav(){if(isPhoneLayout())return;setCreateMenuOpen(false);setMobileNav(false);setLeftCollapsed(on=>!on)} async function openLoginScreen(){ const id=paneBot?.id||active?.id;if(!id)return; + if(!beforeTakeover.current)beforeTakeover.current={part:rightPart,collapsed:rightCollapsed,open:computerOpen}; setRightPart("computer");setRightCollapsed(false);setComputerOpen(true); await action(async()=>{try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */}}); // Opening the shared screen never pauses a running task. @@ -363,6 +440,8 @@ export function App(){ } const mentionAgents=useMemo(()=>activeRoom?activeRoom.members.map(member=>({id:member.id,name:member.name})):[],[activeRoom]); + const onMention=useCallback((id:string)=>{const bot=botsRef.current.find(item=>item.id===id);if(!bot)return;setMessageMenu(null);openBotRef.current(bot)},[]); + const onOpenFile=useCallback((path:string)=>{const botId=currentPaneRef.current;if(!botId)return;setPreviewFile({botId,path,name:path.split("/").filter(Boolean).pop()||path})},[]); const sessionBusy=workingMembers.length>0; const otherSessionBusy=Boolean(computer.busyBotName&&!sessionBusy); const chatName=activeRoom?.name||active?.name||""; @@ -466,7 +545,7 @@ export function App(){ } function copyChatMessage(message:Message){ const selected=window.getSelection()?.toString().trim(); - const text=selected||message.body.trim()||messageFiles(message.blocks).map(file=>file.name).join("\n"); + const text=selected||shownReplyText(message,replyDrafts).trim()||messageFiles(message.blocks).map(file=>file.name).join("\n"); if(text)void copyText(text); } function beginReply(message:Message){ @@ -483,7 +562,7 @@ export function App(){ // A run that stopped to ask keeps its harness state, so the answer is just a // normal message: the backend folds it into the paused run instead of // starting a new task. - async function continueRun(messageId:string){if(!activeSessionId||sendingRef.current||busy)return;sendingRef.current=true;const text=t("resumeSent");try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});setResumedChips(current=>({...current,[messageId]:true}));await loadSessions()})}finally{sendingRef.current=false}} + async function continueRun(messageId:string,reason?:string){if(!activeSessionId||sendingRef.current||busy)return;sendingRef.current=true;const text=reason==="needs_plan"?t("resumePlanSent"):t("resumeSent");try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});setResumedChips(current=>({...current,[messageId]:true}));await loadSessions()})}finally{sendingRef.current=false}} async function retryRun(runId?:string){if(!runId||busy)return;await action(()=>api(`/api/runs/${runId}/retry`,{method:"POST",body:"{}"}));setRetriedRuns(current=>({...current,[runId]:true}))} async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})} async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}} @@ -497,28 +576,48 @@ export function App(){ const botId=message.speakerBotId||active?.id; if(botId)await saveMessageMemory(message,botId,activeSessionId); } - function pasteText(text:string){queueDesktopInput({kind:"clipboard",text})} - function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string}){ + function fallbackHostClipboard(text:string){ + desktopFrameRef.current?.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin); + } + function pasteText(text:string){queueDesktopInput({kind:"clipboard",text},text)} + function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string},fallback?:string){ const botId=paneBotId; - if(!botId||!desktopInteractive)return; + if(!botId||!desktopInteractive){ + if(fallback)fallbackHostClipboard(fallback); + return; + } pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{ if(currentPaneRef.current!==botId)return; await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify(input)}); if(input.kind==="clipboard"&¤tPaneRef.current===botId)setClipboardStatus(t("clipboardPasted")); - }).catch(()=>setClipboardStatus(t("clipboardPasteFailed"))); + }).catch(()=>{ + setClipboardStatus(t("clipboardPasteFailed")); + if(fallback)fallbackHostClipboard(fallback); + }); } async function copySelection(){ const botId=paneBotId;if(!botId)return; - try{const result=await api<{text:string}>(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"copy"})}); + try{const result=await api<{text:string|null}>(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"copy"})}); if(currentPaneRef.current!==botId)return; - setDesktopClipboard(result.text);await navigator.clipboard.writeText(result.text);setClipboardStatus(t("clipboardCopiedSelection")); + const text=typeof result.text==="string"?result.text:""; + if(!text){setClipboardStatus(t("clipboardCopyFailed"));return} + setDesktopClipboard(text); + try{await navigator.clipboard.writeText(text);setClipboardStatus(t("clipboardCopiedSelection"))} + catch{setClipboardStatus(t("clipboardSyncBlocked"))} }catch{setClipboardStatus(t("clipboardCopyFailed"));} } async function pasteClipboard(){try{pasteText(await navigator.clipboard.readText())}catch{setClipboardOpen(true)}} - async function copyClipboard(){try{await navigator.clipboard.writeText(desktopClipboard)}catch{setError(t("clipboardWriteBlocked"))}} + async function copyClipboard(){ + if(desktopClipboard){ + try{await navigator.clipboard.writeText(desktopClipboard);setClipboardStatus(t("clipboardSynced"))} + catch{setError(t("clipboardWriteBlocked"))} + return; + } + await copySelection(); + } async function inbox(bot:Bot,actionName:string,groupName?:string|null){await api(`/api/bots/${bot.id}/inbox`,{method:"POST",body:JSON.stringify({action:actionName,groupName})});await loadBots()} function openBot(bot:Bot){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)} - function openMention(id:string){const bot=bots.find(item=>item.id===id);if(!bot)return;setMessageMenu(null);openBot(bot)} + openBotRef.current=openBot; function openRoom(room:Room){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(rightPart==="settings")setRightPart("computer");if(room.id===activeRoomId){if(!activeSessionId){const stored=readSessionStore()[`room:${room.id}`];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveId(null);setBusyMembers([]);setActiveRoomId(room.id)} async function deleteRoom(room:Room){setRoomToDelete(null);await action(async()=>{await api(`/api/rooms/${room.id}`,{method:"DELETE"});if(activeRoomId===room.id){setActiveRoomId(null);setActiveSessionId(null);setMessages([]);setBusyMembers([])}await loadBots()})} function openAccount(dialog:AccountDialog){setAccountOpen(false);setAccountDialog(dialog)} @@ -563,7 +662,7 @@ export function App(){ setComputer(current=>({...current,controlHolder:holder,takeoverRequested:holder==="user"?false:current.takeoverRequested})); if(!computer.sharedInput)setHandingOff(true); pushViewOnly(viewOnlyFor(holder,computer.sharedInput)); - try{await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`,{method:"POST",body:"{}"})} + try{await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`,{method:"POST",body:"{}"});if(holder==="none"&&beforeTakeover.current){const previous=beforeTakeover.current;beforeTakeover.current=null;setRightPart(previous.part);setRightCollapsed(previous.collapsed);setComputerOpen(previous.open)}} catch(error){setError(localizeError(error instanceof Error?error.message:t("operationFailed")))} finally{ // Read the truth back once: a refused takeover has to give the mouse up @@ -588,31 +687,28 @@ export function App(){ // that was dropped. Status blips never get to strobe it. useEffect(()=>{setVeil(current=>nextVeil(overlayLabel,current))},[overlayLabel]); useEffect(()=>{if(!veil.leaving)return;const timer=setTimeout(()=>setVeil({label:null,leaving:false}),VEIL_FADE_MS);return()=>clearTimeout(timer)},[veil.leaving]); - const frame=screenUrl?