119 lines
11 KiB
JavaScript
119 lines
11 KiB
JavaScript
/** Run against a disposable API with SANDBOX_PROVIDER=fake and a fresh DB.
|
|
* LAZYBOY_TEST_API=http://127.0.0.1:3118 node --test tests/task-experience.test.mjs
|
|
* The mock provider is reachable from the API at host.docker.internal:3119.
|
|
* Never point this test at a workspace containing real user data.
|
|
*/
|
|
import {test} from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import http from 'node:http';
|
|
import {randomUUID} from 'node:crypto';
|
|
const base=process.env.LAZYBOY_TEST_API;
|
|
test('one assignment: verified completion, recovery, and same-run takeover resume',{skip:!base,timeout:60000},async()=>{
|
|
assert.equal(new URL(base).port,'3118','use the isolated test API');
|
|
let cookie='';const calls={};const histories={};let handoffs=0;let finalRecall=0;let releasePriority;const priorityGate=new Promise(resolve=>{releasePriority=resolve});
|
|
const report=(state,summary,remaining=[],extra={})=>({state,summary,completed:['Compared the two supplied source excerpts'],remaining,verification:'Checked both supplied excerpts and the requested comparison',artifacts:[],...extra});
|
|
const fixture=http.createServer(async(req,res)=>{
|
|
let body='';for await(const chunk of req)body+=chunk;
|
|
const input=JSON.parse(body||'{}');const serialized=JSON.stringify(input.messages||[]);
|
|
const scenario=['UX_RESEARCH','UX_RECOVERY','UX_TAKEOVER','UX_CONTEXT','UX_PRIORITY'].find(tag=>serialized.includes(tag));
|
|
if(serialized.includes('previous_handoff')){
|
|
handoffs++;
|
|
const summary=JSON.stringify({objectives:'Newest task first',constraints:'Never publish; palette violet',decisions:'Older unfinished work is parked',completed:'Prior local comparisons delivered',remaining:'Handle newest assignment only',references:'Read message 1 for exact wording'});
|
|
res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id:randomUUID(),object:'chat.completion',created:1,model:input.model,choices:[{index:0,message:{role:'assistant',content:handoffs===1?'invalid handoff':summary},finish_reason:'stop'}],usage:{prompt_tokens:10,completion_tokens:10,total_tokens:20}}));return;
|
|
}
|
|
if(!scenario){res.writeHead(400);res.end('unknown fixture');return;}
|
|
const index=calls[scenario]||0;calls[scenario]=index+1;(histories[scenario]??=[]).push(serialized);
|
|
let name,args,text='Finished.';
|
|
if(scenario==='UX_RESEARCH'){
|
|
if(index===0)text='I have only planned it.\n[GOAL_COMPLETE]';
|
|
if(index===1){name='report_task';args=report('progress','I checked the first excerpt.',['Compare the second excerpt']);}
|
|
if(index===2){name='report_task';args=report('complete','A and B both support delegation; B also requires verification.');}
|
|
}else if(scenario==='UX_RECOVERY'){
|
|
if(index===0){name='read_file';args={path:'missing-source.txt'};}
|
|
if(index===1){name='report_task';args=report('recovering','The file is missing. I will use the source excerpts you supplied.',['Compare the excerpts'],{attempts:['read_file returned not found; switching to supplied source text']});}
|
|
if(index===2){name='report_task';args=report('complete','Completed the comparison using the supplied sources.');}
|
|
}else if(scenario==='UX_CONTEXT'){
|
|
if(serialized.includes('UX_CONTEXT_FINAL')&&finalRecall++===0){name='read_conversation';args={query:'UX_CONTEXT_SEED',limit:1};}
|
|
else{name='report_task';args=report('complete','Completed the latest comparison; original constraint retained.');}
|
|
}else if(scenario==='UX_PRIORITY'){
|
|
if(index===0)await priorityGate;
|
|
name='report_task';args=report('complete',index===0?'Old comparison done.':'Newest comparison delivered; older work parked.');
|
|
}else{
|
|
if(index===0){name='request_takeover' ;args={intervention:'verification',site:'Local test fixture',reason:'Complete the human verification step.',why:'I will finish the original comparison.'};}
|
|
if(index===1){name='report_task';args=report('complete','Resumed and completed the original comparison.');}
|
|
}
|
|
const id=randomUUID();const choice=name?{role:'assistant',tool_calls:[{id,type:'function',function:{name,arguments:JSON.stringify(args)}}]}:{role:'assistant',content:text};
|
|
if(input.stream){
|
|
res.writeHead(200,{'content-type':'text/event-stream'});
|
|
const chunk=delta=>({id,object:'chat.completion.chunk',created:1,model:input.model,choices:[{index:0,delta,finish_reason:null}]});
|
|
res.write(`data: ${JSON.stringify(chunk(name?{role:'assistant',tool_calls:[{index:0,...choice.tool_calls[0]}]}:{role:'assistant',content:text}))}\n\n`);
|
|
res.write(`data: ${JSON.stringify({id,object:'chat.completion.chunk',created:1,model:input.model,choices:[{index:0,delta:{},finish_reason:name?'tool_calls':'stop'}],usage:{prompt_tokens:10,completion_tokens:10,total_tokens:20}})}\n\ndata: [DONE]\n\n`);res.end();
|
|
}else{res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify({id,object:'chat.completion',created:1,model:input.model,choices:[{index:0,message:choice,finish_reason:name?'tool_calls':'stop'}],usage:{prompt_tokens:10,completion_tokens:10,total_tokens:20}}));}
|
|
});
|
|
await new Promise(resolve=>fixture.listen(3119,'0.0.0.0',resolve));
|
|
async function api(path,body,method=body?'POST':'GET'){
|
|
const r=await fetch(base+path,{method,headers:{'content-type':'application/json',cookie},body:body?JSON.stringify(body):undefined});
|
|
if(r.headers.get('set-cookie'))cookie=r.headers.get('set-cookie').split(';')[0];
|
|
const raw=await r.text();assert.ok(r.ok,`${path}: ${r.status} ${raw}`);return raw?JSON.parse(raw):null;
|
|
}
|
|
async function until(fn,label){for(let i=0;i<150;i++){const found=await fn();if(found)return found;await new Promise(r=>setTimeout(r,150));}throw Error(`Timed out: ${label}`);}
|
|
try{
|
|
await api('/api/auth/register',{username:'ux'+randomUUID().replaceAll('-','').slice(0,12),password:'FixtureOnly123!'});
|
|
await api('/api/workspace/settings',{provider:'openai-compatible',modelId:'fixture-vision',baseUrl:'http://host.docker.internal:3119/v1',apiKey:'fixture-only'},'PATCH');
|
|
const bot=await api('/api/bots',{name:'UX fixture',computerMode:'team',memoryEnabled:false});
|
|
for(const scenario of ['UX_RESEARCH','UX_RECOVERY','UX_TAKEOVER']){
|
|
const session=await api(`/api/bots/${bot.id}/sessions`,{title:scenario});
|
|
await api(`/api/sessions/${session.id}/messages`,{text:`${scenario}: Compare these sources and deliver a concise report. Source A: assign the outcome once. Source B: assign the outcome once and verify the result. Use supplied excerpts if a file is missing.`,clientNonce:randomUUID()});
|
|
let run;
|
|
if(scenario==='UX_TAKEOVER'){
|
|
run=await until(async()=>{const [r]=await api(`/api/sessions/${session.id}/task`);return r?.status==='waiting_takeover'?r:null;},'takeover');
|
|
const messages=await api(`/api/sessions/${session.id}/messages`);
|
|
assert.equal(messages.flatMap(m=>m.blocks||[]).find(b=>b.kind==='login')?.intervention,'verification');
|
|
await api(`/api/computer/${bot.id}/release`,{});
|
|
}
|
|
const completed=await until(async()=>{const [r]=await api(`/api/sessions/${session.id}/task`);if(r?.status==='failed')throw Error(`run failed: ${scenario}`);return r?.status==='completed'?r:null;},scenario);
|
|
assert.equal(completed.report.state,'complete');assert.deepEqual(completed.report.remaining,[]);
|
|
if(run)assert.equal(completed.runId,run.runId,'human releases control; original run resumes');
|
|
const messages=await api(`/api/sessions/${session.id}/messages`);
|
|
assert.equal(messages.filter(m=>m.role==='user').length,1,'no repeated instruction');
|
|
assert.match(messages.at(-1).body,/comparison|delegation/);
|
|
if(scenario==='UX_RESEARCH')assert.equal(calls[scenario],3,'reject bare completion, then deliver without an extra model call');
|
|
if(scenario==='UX_RECOVERY')assert.equal(messages.filter(m=>m.blocks?.some(b=>b.kind==='progress')).length,1);
|
|
if(scenario==='UX_TAKEOVER')assert.match(histories[scenario][1],/CURRENT screen|current screen/);
|
|
console.log(`${scenario}: complete, one user assignment, ${run?'one verification step':'zero intervention'}`);
|
|
}
|
|
const contextSession=await api(`/api/bots/${bot.id}/sessions`,{title:'UX_CONTEXT'});
|
|
for(let i=0;i<18;i++){
|
|
await api(`/api/sessions/${contextSession.id}/messages`,{text:i===0?'UX_CONTEXT_SEED: Compare A and B locally. Constraint: Never publish; palette violet.':`UX_CONTEXT: Complete local comparison number ${i}, supplied A=one B=two.`,clientNonce:randomUUID()});
|
|
await until(async()=>{const [r]=await api(`/api/sessions/${contextSession.id}/task`);return r?.status==='completed';},'long conversation');
|
|
}
|
|
await until(async()=>{const sessions=await api(`/api/bots/${bot.id}/sessions`);return sessions.find(s=>s.id===contextSession.id)?.historySummarySeq>0;},'background handoff persisted');
|
|
assert.ok(handoffs>=2,'failed handoff is retried automatically on a later turn');
|
|
await api(`/api/sessions/${contextSession.id}/messages`,{text:'UX_CONTEXT_FINAL: Retrieve the exact first constraint and deliver the newest comparison.',clientNonce:randomUUID()});
|
|
await until(async()=>{const [r]=await api(`/api/sessions/${contextSession.id}/task`);return r?.status==='completed';},'archive recall');
|
|
const transcript=await api(`/api/sessions/${contextSession.id}/messages`);
|
|
assert.equal(transcript.filter(m=>m.role==='user').length,19,'original messages stay in the same session');
|
|
assert.match(transcript[0].body,/UX_CONTEXT_SEED/);
|
|
assert.match(histories.UX_CONTEXT.at(-1),/Never publish; palette violet/);
|
|
assert.match(histories.UX_CONTEXT.at(-1),/UX_CONTEXT_SEED/,'assistant retrieves original text outside recent window');
|
|
const sessionList=await api(`/api/bots/${bot.id}/sessions`);
|
|
const compacted=sessionList.find(s=>s.id===contextSession.id);
|
|
assert.ok(compacted.historySummarySeq>0);
|
|
console.log('UX_CONTEXT: automatic handoff, original transcript retained, exact old message recalled');
|
|
|
|
const prioritySession=await api(`/api/bots/${bot.id}/sessions`,{title:'UX_PRIORITY'});
|
|
const original=await api(`/api/sessions/${prioritySession.id}/messages`,{text:'UX_PRIORITY: Complete old comparison A.',clientNonce:randomUUID()});
|
|
await until(async()=>calls.UX_PRIORITY>0,'old model request');
|
|
const newest=await api(`/api/sessions/${prioritySession.id}/messages`,{text:'UX_PRIORITY_NEW: Prioritize new comparison B; park A.',clientNonce:randomUUID()});
|
|
releasePriority();
|
|
assert.equal(newest.runId,original.runId,'newest instruction steers without requiring session changes');
|
|
await until(async()=>{const [r]=await api(`/api/sessions/${prioritySession.id}/task`);return r?.status==='completed';},'new task priority');
|
|
const priorityMessages=await api(`/api/sessions/${prioritySession.id}/messages`);
|
|
assert.match(priorityMessages.at(-1).body,/Newest comparison/);
|
|
assert.match(histories.UX_PRIORITY.at(-1),/newest user message has highest task priority/i);
|
|
assert.match(histories.UX_PRIORITY.at(-1),/UX_PRIORITY_NEW/);
|
|
assert.match(histories.UX_PRIORITY.at(-1),/Not executed: a newer user instruction arrived/,'old planned tool never executes after a new assignment arrives');
|
|
console.log('UX_PRIORITY: newest instruction wins before old completion can end the run');
|
|
}finally{fixture.closeAllConnections();await new Promise(resolve=>fixture.close(resolve));}
|
|
});
|