diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index d82e399..455e3d9 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,5 +1,5 @@ import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Square, Users, X } from "./animated-icons"; +import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Users, X } from "./animated-icons"; import UseAnimations from "react-useanimations"; import loading from "react-useanimations/lib/loading"; import loading2 from "react-useanimations/lib/loading2"; @@ -18,9 +18,9 @@ import searchToX from "react-useanimations/lib/searchToX"; import { api, ApiError } from "./api"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { t, type MessageKey } from "./i18n"; -import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Room, RoomMember, Session, WorkspaceSettings } from "./types"; +import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, ModelProviderId, Playbook, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types"; -const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,display:null,profileMode:"per-bot",screenAvailable:false}; +const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; const SESSION_STORE="lazyboy.sessionByBot"; const PANE_STORE="lazyboy.rightPane"; const WORKSPACE_STORE="lazyboy.workspace"; @@ -67,6 +67,7 @@ export function App(){ const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null); const [roomToDelete,setRoomToDelete]=useState(null); const [mcpServers,setMcpServers]=useState([]); const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState(null); + const [skills,setSkills]=useState([]); const [plusOpen,setPlusOpen]=useState(false); const [teachOpen,setTeachOpen]=useState(false); const [workspaceName,setWorkspaceName]=useState(workspaceStart.name); const [looks,setLooks]=useState(readAvatarLooks); const sendingRef=useRef(false); const refreshSeqRef=useRef(0); @@ -81,6 +82,15 @@ export function App(){ const paneBot=bots.find(bot=>bot.id===paneBotId)||active; const workingMembers=activeRoom?busyMembers:active&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[]; const lastMessageId=messages[messages.length-1]?.id||""; + // The bot is parked in waiting_takeover: nothing moves (including queued + // messages) until the human releases the screen, so say so loudly. + const pausedForUser=computer.takeoverRequested&&workingMembers.length===0&&(!computer.waitingSessionId||computer.waitingSessionId===activeSessionId||Boolean(activeRoom)); + // Teaching by demonstration: one recording at a time per bot, then a draft + // playbook the human names and saves before it becomes a real skill. + const teaching=skills.find(skill=>skill.status==="recording")||null; + const drafting=skills.find(skill=>skill.status==="drafting")||null; + const skillDraft=teaching||drafting?null:skills.find(skill=>skill.status==="draft")||null; + const savedSkills=skills.filter(skill=>skill.status==="saved"); const loadMcp=useCallback(async()=>{setMcpServers(await api("/api/mcp-servers").catch(()=>[] as McpServer[]))},[]); const loadBots=useCallback(async()=>{const [next,nextRooms]=await Promise.all([api("/api/bots"),api("/api/rooms").catch(()=>[] as Room[])]);setBots(next);setRooms(nextRooms);setActiveRoomId(id=>id&&nextRooms.some(room=>room.id===id)?id:null);setActiveId(id=>id&&next.some(b=>b.id===id)?id:null);await loadMcp()},[loadMcp]); @@ -93,22 +103,23 @@ export function App(){ try{const roomStatus=await api<{busy:RoomMember[]}>(`/api/rooms/${activeRoomId}/status`);if(refreshSeq!==refreshSeqRef.current)return;nextBusy=roomStatus.busy;computerBot=roomStatus.busy[0]?.id||roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null}catch{if(refreshSeq!==refreshSeqRef.current)return;computerBot=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||null} } const messagesJob=api(`/api/sessions/${activeSessionId}/messages`); - if(!computerBot){const nextMessages=await messagesJob;if(refreshSeq===refreshSeqRef.current){setBusyMembers(nextBusy);setMessages(nextMessages);setComputer(blankComputer);setScreenUrl(null)}return} + if(!computerBot){const nextMessages=await messagesJob;if(refreshSeq===refreshSeqRef.current){setBusyMembers(nextBusy);setMessages(nextMessages);setComputer(blankComputer);setSkills([]);setScreenUrl(null)}return} + const skillsJob=activeRoomId?Promise.resolve([] as TaughtSkill[]):api(`/api/bots/${computerBot}/skills`).catch(()=>[] as TaughtSkill[]); const status=await api(`/api/computer/${computerBot}/status`);if(refreshSeq!==refreshSeqRef.current)return; - const [nextMessages,screen]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null})]); + const [nextMessages,screen,nextSkills]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null}),skillsJob]); if(refreshSeq!==refreshSeqRef.current)return; - setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setScreenUrl(status.botId===computerBot?screen.url:null) + setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null) },[activeId,activeRoomId,activeSessionId]); useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e.message)})},[loadBots]); useEffect(()=>{if(activeId||activeRoomId||bots.length===0)return;setActiveId(bots[0].id)},[bots,activeId,activeRoomId]); useEffect(()=>{setMessages([]);loadSessions().catch(e=>setError(e.message))},[loadSessions]); useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current=""},[activeSessionId]); - useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length]); + useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]); useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(e.message));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]); useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); - useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); - useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); + useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); + useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]); useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]); useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]); @@ -120,6 +131,13 @@ export function App(){ async function action(work:()=>Promise){setBusy(true);setError(null);try{await work();await refresh()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}} async function stopChat(){if(activeSessionId)await api(`/api/sessions/${activeSessionId}/stop`,{method:"POST",body:"{}"});else if(active)await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});} + async function startTeaching(goal:string){if(!active)return;setTeachOpen(false);await action(async()=>{await api(`/api/bots/${active.id}/skills/start`,{method:"POST",body:JSON.stringify({goal})});setComputerOpen(true)})} + async function stopTeaching(){if(!active)return;await action(()=>api(`/api/bots/${active.id}/skills/stop`,{method:"POST",body:"{}"}));setComputerOpen(false)} + async function cancelTeaching(){if(!active)return;await action(()=>api(`/api/bots/${active.id}/skills/cancel`,{method:"POST",body:"{}"}));setComputerOpen(false)} + async function saveSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}))} + async function testSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(async()=>{await api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook})});await api(`/api/skills/${skill.id}/test`,{method:"POST",body:"{}"})})} + async function discardSkill(skill:TaughtSkill){await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}))} + function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}執行「${skill.name}」`);document.querySelector(".composer textarea")?.focus()} async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();if((!active&&!activeRoom)||!activeSessionId||!text)return;sendingRef.current=true;setDraft("");historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});sentHistoryRef.current.push(text);if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();await loadSessions()})}finally{sendingRef.current=false}} function composerKeyDown(event:ReactKeyboardEvent){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current
{activeRoom?<>member.id)}/>{activeRoom.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:active?<>{active.name}{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/>{topTools}:<>{t("chooseBot")}{topTools}}
-
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return
{spoken&&}{spoken&&{speakerName}}{message.body}{message.body.trim()&&}
})}{workingMembers.map(member=>
{t("working",{name:member.name})}
)} +
{(activeRoom||active)&&messages.length===0?
{activeRoom?:}

{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}

{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}

:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;return
{spoken&&}{spoken&&{speakerName}}{message.body}{message.body.trim()&&}
})}{workingMembers.map(member=>
{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}
)}{pausedForUser&&paneBot&&
{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}}{computer.controlHolder==="user"?:}
}{teaching&&active&&
{t("teachingLive",{goal:teaching.goal})}{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}}
}{drafting&&
{t("distilling",{goal:drafting.goal})}
}{skillDraft&&active&&void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)}/>} {error&&
{error}
} {otherSessionBusy&&
{t("anotherConversationQueued")}
} -