336 lines
18 KiB
Python
336 lines
18 KiB
Python
|
"""Durable GUI admission and real cross-process lock tests; not yet GUI integration."""
|
||
|
|
import importlib.util
|
||
|
|
import hashlib
|
||
|
|
import hmac
|
||
|
|
import json
|
||
|
|
from contextlib import contextmanager
|
||
|
|
import multiprocessing
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
import tempfile
|
||
|
|
import sqlite3
|
||
|
|
import subprocess
|
||
|
|
import time
|
||
|
|
import unittest
|
||
|
|
from unittest.mock import patch
|
||
|
|
|
||
|
|
SOURCE = Path(__file__).resolve().parents[1] / 'crates/supervisor/src/runner_gui_fences.py'
|
||
|
|
spec = importlib.util.spec_from_file_location('gui_fences', SOURCE)
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
Store = module.GuiFenceStore
|
||
|
|
Error = module.GuiFenceError
|
||
|
|
|
||
|
|
|
||
|
|
def hold_admission(root, claim, entered, release):
|
||
|
|
with Store(root).admit(claim) as admission:
|
||
|
|
entered.set()
|
||
|
|
if not release.wait(10):
|
||
|
|
raise RuntimeError('fixture release timeout')
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
|
||
|
|
|
||
|
|
@unittest.skipUnless(os.geteuid() == 0, 'protected Runner state requires root fixture')
|
||
|
|
class GuiFences(unittest.TestCase):
|
||
|
|
def setUp(self):
|
||
|
|
self.directory = tempfile.TemporaryDirectory(prefix='lazyboy-gui-fences-')
|
||
|
|
self.addCleanup(self.directory.cleanup)
|
||
|
|
self.root = Path(self.directory.name)
|
||
|
|
self.store = Store(self.root, lock_timeout=.15)
|
||
|
|
self.claim = dict(provider='provider', generation=1, slot=0, screen='screen-a', bot='bot-a', epoch=1, lease='run-a:1', profile='/home/lazyboy/.browser-profiles/bots/bot-a')
|
||
|
|
self.expires = int(time.time() * 1000) + 120000
|
||
|
|
|
||
|
|
def ready(self):
|
||
|
|
self.store.publish_generation('provider', 1)
|
||
|
|
self.store.publish_screen(self.claim, self.expires)
|
||
|
|
|
||
|
|
def test_admission_never_creates_or_advances_authority(self):
|
||
|
|
with self.assertRaisesRegex(Error, 'STALE_GUI_GENERATION'):
|
||
|
|
with self.store.admit(self.claim):
|
||
|
|
self.fail('unknown generation admitted')
|
||
|
|
self.store.publish_generation('provider', 1)
|
||
|
|
with self.assertRaisesRegex(Error, 'STALE_GUI_EPOCH'):
|
||
|
|
with self.store.admit(self.claim):
|
||
|
|
self.fail('unknown screen admitted')
|
||
|
|
self.store.publish_screen(self.claim, self.expires)
|
||
|
|
for field, value in [('provider', 'other'), ('generation', 2), ('slot', 1),
|
||
|
|
('screen', 'other'), ('bot', 'other'), ('epoch', 2), ('lease', None), ('profile', '/home/lazyboy/.browser-profiles/other')]:
|
||
|
|
with self.assertRaises(Error):
|
||
|
|
with self.store.admit({**self.claim, field: value}):
|
||
|
|
self.fail(field)
|
||
|
|
with self.store.admit(self.claim) as expiry:
|
||
|
|
self.assertEqual(expiry.expires_at_ms, self.expires)
|
||
|
|
expiry.confirm_quiescent()
|
||
|
|
|
||
|
|
def test_durable_high_watermarks_and_owner_change(self):
|
||
|
|
self.ready()
|
||
|
|
newer = {**self.claim, 'epoch': 2, 'screen': 'screen-b', 'bot': 'bot-b', 'lease': 'run-b:2'}
|
||
|
|
self.store.publish_screen(newer, self.expires)
|
||
|
|
reopened = Store(self.root)
|
||
|
|
with self.assertRaisesRegex(Error, 'STALE_GUI_EPOCH'):
|
||
|
|
reopened.publish_screen(self.claim, self.expires)
|
||
|
|
with self.assertRaisesRegex(Error, 'CONFLICTING_GUI_EPOCH'):
|
||
|
|
reopened.publish_screen({**newer, 'bot': 'attacker'}, self.expires)
|
||
|
|
with self.assertRaises(Error):
|
||
|
|
with reopened.admit(self.claim):
|
||
|
|
self.fail('old owner admitted')
|
||
|
|
reopened.publish_generation('replacement', 2)
|
||
|
|
for provider, generation in [('provider', 1), ('different', 2)]:
|
||
|
|
with self.assertRaisesRegex(Error, 'STALE_GUI_GENERATION'):
|
||
|
|
reopened.publish_generation(provider, generation)
|
||
|
|
with self.assertRaises(Error):
|
||
|
|
with reopened.admit(newer):
|
||
|
|
self.fail('old generation admitted')
|
||
|
|
current = {**newer, 'provider': 'replacement', 'generation': 2, 'epoch': 0}
|
||
|
|
reopened.publish_screen(current, self.expires)
|
||
|
|
reopened.publish_generation('replacement', 2)
|
||
|
|
with Store(self.root).admit(current) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
|
||
|
|
def test_pause_resume_expiry_and_replay(self):
|
||
|
|
self.ready()
|
||
|
|
paused = {**self.claim, 'epoch': 2}
|
||
|
|
self.store.publish_screen(paused, self.expires, paused=True)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_SCOPE_PAUSED'):
|
||
|
|
with self.store.admit(paused):
|
||
|
|
self.fail('paused admission')
|
||
|
|
with self.assertRaisesRegex(Error, 'CONFLICTING_GUI_EPOCH'):
|
||
|
|
self.store.publish_screen(paused, self.expires, paused=False)
|
||
|
|
resumed = {**paused, 'epoch': 3}
|
||
|
|
self.store.publish_screen(resumed, self.expires)
|
||
|
|
self.store.publish_screen(resumed, self.expires - 1000)
|
||
|
|
with self.store.admit(resumed) as expiry:
|
||
|
|
self.assertEqual(expiry.expires_at_ms, self.expires)
|
||
|
|
expiry.confirm_quiescent()
|
||
|
|
with patch.object(module.time, 'time', return_value=self.expires / 1000):
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_LEASE_EXPIRED'):
|
||
|
|
with self.store.admit(resumed):
|
||
|
|
self.fail('expired admission')
|
||
|
|
|
||
|
|
def test_real_process_barrier_drains_only_its_scope_and_generation(self):
|
||
|
|
self.ready()
|
||
|
|
peer = {**self.claim, 'slot': 1, 'screen': 'peer', 'bot': 'peer', 'profile': '/home/lazyboy/.browser-profiles/bots/peer'}
|
||
|
|
self.store.publish_screen(peer, self.expires)
|
||
|
|
context = multiprocessing.get_context('spawn')
|
||
|
|
entered, release = context.Event(), context.Event()
|
||
|
|
child = context.Process(target=hold_admission, args=(str(self.root), self.claim, entered, release))
|
||
|
|
child.start()
|
||
|
|
try:
|
||
|
|
self.assertTrue(entered.wait(5))
|
||
|
|
with self.store.admit(peer) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
self.store.publish_screen({**peer, 'epoch': 2}, self.expires)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_FENCE_BUSY'):
|
||
|
|
self.store.publish_screen({**self.claim, 'epoch': 2}, self.expires, paused=True)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_FENCE_BUSY'):
|
||
|
|
self.store.publish_generation('replacement', 2)
|
||
|
|
finally:
|
||
|
|
release.set()
|
||
|
|
child.join(5)
|
||
|
|
if child.is_alive():
|
||
|
|
child.kill()
|
||
|
|
child.join()
|
||
|
|
self.assertEqual(child.exitcode, 0)
|
||
|
|
self.store.publish_screen({**self.claim, 'epoch': 2}, self.expires, paused=True)
|
||
|
|
self.store.publish_generation('replacement', 2)
|
||
|
|
|
||
|
|
def test_crashed_executor_does_not_turn_lock_release_into_quiescence(self):
|
||
|
|
self.ready()
|
||
|
|
context = multiprocessing.get_context('spawn')
|
||
|
|
entered, release = context.Event(), context.Event()
|
||
|
|
child = context.Process(target=hold_admission, args=(str(self.root), self.claim, entered, release))
|
||
|
|
child.start()
|
||
|
|
try:
|
||
|
|
self.assertTrue(entered.wait(5))
|
||
|
|
finally:
|
||
|
|
child.kill()
|
||
|
|
child.join(5)
|
||
|
|
reopened = Store(self.root, lock_timeout=.15)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
reopened.publish_screen({**self.claim, 'epoch': 2}, self.expires, paused=True)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
reopened.publish_generation('replacement', 2)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
with reopened.admit(self.claim):
|
||
|
|
self.fail('crash was silently recovered')
|
||
|
|
peer = {**self.claim, 'slot': 1, 'screen': 'peer', 'bot': 'peer', 'profile': '/home/lazyboy/.browser-profiles/bots/peer'}
|
||
|
|
reopened.publish_screen(peer, self.expires)
|
||
|
|
with reopened.admit(peer) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
|
||
|
|
def test_exception_preserves_unknown_effect(self):
|
||
|
|
self.ready()
|
||
|
|
with self.assertRaisesRegex(RuntimeError, 'lost response'):
|
||
|
|
with self.store.admit(self.claim):
|
||
|
|
raise RuntimeError('lost response')
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
self.store.publish_screen({**self.claim, 'epoch': 2}, self.expires)
|
||
|
|
|
||
|
|
def test_storage_failures_never_publish_a_false_barrier(self):
|
||
|
|
self.ready()
|
||
|
|
with sqlite3.connect(self.root / 'gui-fences.sqlite') as db:
|
||
|
|
db.execute("CREATE TRIGGER fail_generation BEFORE DELETE ON screens BEGIN SELECT RAISE(ABORT,'fixture'); END")
|
||
|
|
with self.assertRaises(sqlite3.DatabaseError):
|
||
|
|
self.store.publish_generation('replacement', 2)
|
||
|
|
with Store(self.root).admit(self.claim) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
with sqlite3.connect(self.root / 'gui-fences.sqlite') as db:
|
||
|
|
db.execute('DROP TRIGGER fail_generation')
|
||
|
|
db.execute("CREATE TRIGGER fail_intent BEFORE INSERT ON active BEGIN SELECT RAISE(ABORT,'fixture'); END")
|
||
|
|
entered = False
|
||
|
|
with self.assertRaises(sqlite3.DatabaseError):
|
||
|
|
with self.store.admit(self.claim):
|
||
|
|
entered = True
|
||
|
|
self.assertFalse(entered)
|
||
|
|
with sqlite3.connect(self.root / 'gui-fences.sqlite') as db:
|
||
|
|
db.execute('DROP TRIGGER fail_intent')
|
||
|
|
db.execute("CREATE TRIGGER fail_completion BEFORE DELETE ON active BEGIN SELECT RAISE(ABORT,'fixture'); END")
|
||
|
|
with self.assertRaises(sqlite3.DatabaseError):
|
||
|
|
with self.store.admit(self.claim) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
self.store.publish_generation('provider', 1)
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
self.store.publish_generation('replacement', 2)
|
||
|
|
|
||
|
|
def test_lease_expiring_while_waiting_cannot_publish(self):
|
||
|
|
self.ready()
|
||
|
|
original = self.store._lock
|
||
|
|
with patch.object(module.time, 'time', return_value=1000) as clock:
|
||
|
|
@contextmanager
|
||
|
|
def delayed(name, exclusive):
|
||
|
|
with original(name, exclusive):
|
||
|
|
clock.return_value = 1002
|
||
|
|
yield
|
||
|
|
with patch.object(self.store, '_lock', delayed):
|
||
|
|
with self.assertRaisesRegex(Error, 'INVALID_GUI_LEASE'):
|
||
|
|
self.store.publish_screen({**self.claim, 'epoch': 2}, 1001000)
|
||
|
|
with self.store.admit(self.claim) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
|
||
|
|
def test_normal_return_without_quiescence_is_not_completion(self):
|
||
|
|
self.ready()
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
with self.store.admit(self.claim):
|
||
|
|
pass
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_EFFECT_UNRESOLVED'):
|
||
|
|
self.store.publish_screen({**self.claim, 'epoch': 2}, self.expires)
|
||
|
|
|
||
|
|
def test_task_uid_cannot_open_or_publish_root_state(self):
|
||
|
|
program = SOURCE.read_text() + '\nGuiFenceStore(' + repr(str(self.root)) + ')\n'
|
||
|
|
result = subprocess.run(['python3', '-I', '-c', program],
|
||
|
|
user=1000, group=1000, extra_groups=[], cwd='/tmp',
|
||
|
|
env={'PATH': '/usr/bin:/bin', 'HOME': '/tmp', 'LANG': 'C.UTF-8'},
|
||
|
|
capture_output=True, text=True, timeout=5)
|
||
|
|
self.assertNotEqual(result.returncode, 0)
|
||
|
|
self.assertIn('GUI_FENCE_ROOT_REQUIRED', result.stderr)
|
||
|
|
database = self.root / 'gui-fences.sqlite'
|
||
|
|
database.chmod(0o640)
|
||
|
|
with self.assertRaisesRegex(Error, 'UNPROTECTED_GUI_FENCE_FILE'):
|
||
|
|
Store(self.root)
|
||
|
|
|
||
|
|
def test_profile_binding_and_shared_profile_exclusion(self):
|
||
|
|
self.ready()
|
||
|
|
peer = {**self.claim, 'slot': 1, 'screen': 'peer', 'bot': 'peer'}
|
||
|
|
self.store.publish_screen(peer, self.expires)
|
||
|
|
independent = {**peer, 'slot': 2, 'profile': '/home/lazyboy/.browser-profiles/independent'}
|
||
|
|
self.store.publish_screen(independent, self.expires)
|
||
|
|
with self.store.admit(self.claim) as admission:
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_PROFILE_BUSY'):
|
||
|
|
with self.store.admit(peer):
|
||
|
|
self.fail('shared profile admitted twice')
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_PROFILE_BUSY'):
|
||
|
|
self.store.publish_screen({**peer, 'epoch': 2}, self.expires)
|
||
|
|
with self.store.admit(independent) as other:
|
||
|
|
other.confirm_quiescent()
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
with self.assertRaisesRegex(Error, 'CONFLICTING_GUI_EPOCH'):
|
||
|
|
self.store.publish_screen({**self.claim, 'profile': independent['profile']}, self.expires)
|
||
|
|
for profile in ['/tmp/foreign', '/home/lazyboy/.browser-profiles/../foreign',
|
||
|
|
'/home/lazyboy/.browser-profiles//a', '/home/lazyboy/.browser-profiles/a/', None]:
|
||
|
|
with self.assertRaisesRegex(Error, 'INVALID_GUI_PROFILE'):
|
||
|
|
self.store.publish_screen({**self.claim, 'profile': profile}, self.expires)
|
||
|
|
|
||
|
|
def test_signed_publication_binds_audience_home_identity_and_time(self):
|
||
|
|
key = '11' * 32
|
||
|
|
now = int(time.time() * 1000)
|
||
|
|
payload = dict(version=1, audience='lazyboy-gui-publication-v1', home='home',
|
||
|
|
issued_at_ms=now, expires_at_ms=now+60000, claim=self.claim,
|
||
|
|
lease_expires_at_ms=self.expires, paused=False)
|
||
|
|
identity = dict(computer_id='provider', bot_id='bot-a', generation=1)
|
||
|
|
def sign(value):
|
||
|
|
raw = json.dumps(value) if isinstance(value, dict) else value
|
||
|
|
return dict(payload=raw, signature=hmac.new(bytes.fromhex(key), raw.encode(), hashlib.sha256).hexdigest())
|
||
|
|
verified = module.verify_gui_publication(sign(payload), identity, 'home', key)
|
||
|
|
receipt = module.apply_gui_publication(self.store, verified)
|
||
|
|
self.assertEqual(receipt['claim'], self.claim)
|
||
|
|
self.assertNotIn('signature', receipt)
|
||
|
|
peer = {**self.claim, 'slot': 1, 'screen': 'peer', 'bot': 'peer',
|
||
|
|
'profile': '/home/lazyboy/.browser-profiles/peer'}
|
||
|
|
peer_proof = module.verify_gui_publication(sign({**payload, 'claim': peer}),
|
||
|
|
{**identity, 'bot_id': 'peer'}, 'home', key)
|
||
|
|
with self.store.admit(self.claim) as admission:
|
||
|
|
self.assertTrue(module.apply_gui_publication(self.store, peer_proof)['published'])
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
for modified in [dict(audience='other'), dict(home='other'), dict(version=True),
|
||
|
|
dict(expires_at_ms=now-1), dict(issued_at_ms=now+10000),
|
||
|
|
dict(claim={**self.claim, 'bot':'foreign'}), dict(extra=1)]:
|
||
|
|
with self.assertRaisesRegex(Error, '^INVALID_GUI_PUBLICATION$'):
|
||
|
|
module.verify_gui_publication(sign({**payload, **modified}), identity, 'home', key)
|
||
|
|
for envelope in [dict(payload=json.dumps(payload),signature='00'*32),
|
||
|
|
sign(json.dumps(payload).replace('"version": 1', '"version": 1, "version": 1'))]:
|
||
|
|
with self.assertRaisesRegex(Error, '^INVALID_GUI_PUBLICATION$'):
|
||
|
|
module.verify_gui_publication(envelope, identity, 'home', key)
|
||
|
|
with patch.object(module.time, 'time', return_value=(now+60000)/1000):
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_PUBLICATION_EXPIRED'):
|
||
|
|
module.apply_gui_publication(self.store, verified)
|
||
|
|
with self.store.admit(self.claim) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
|
||
|
|
def test_legacy_state_without_profile_binding_fails_closed(self):
|
||
|
|
self.ready()
|
||
|
|
with sqlite3.connect(self.root / 'gui-fences.sqlite') as db:
|
||
|
|
db.execute('DROP TABLE screens')
|
||
|
|
db.execute('CREATE TABLE screens(slot INTEGER PRIMARY KEY,screen TEXT,bot TEXT,epoch INTEGER,lease TEXT,expires INTEGER,paused INTEGER)')
|
||
|
|
db.execute('INSERT INTO screens VALUES (0,?,?,?,?,?,0)',
|
||
|
|
(self.claim['screen'],self.claim['bot'],self.claim['epoch'],self.claim['lease'],self.expires))
|
||
|
|
reopened = Store(self.root)
|
||
|
|
with self.assertRaisesRegex(Error, 'STALE_GUI_EPOCH'):
|
||
|
|
with reopened.admit(self.claim):
|
||
|
|
self.fail('missing legacy profile accepted')
|
||
|
|
with self.assertRaisesRegex(Error, 'CONFLICTING_GUI_EPOCH'):
|
||
|
|
reopened.publish_screen(self.claim,self.expires)
|
||
|
|
upgraded = {**self.claim,'epoch':2}
|
||
|
|
reopened.publish_screen(upgraded,self.expires)
|
||
|
|
with reopened.admit(upgraded) as admission:
|
||
|
|
admission.confirm_quiescent()
|
||
|
|
with sqlite3.connect(self.root / 'gui-fences.sqlite') as db:
|
||
|
|
db.execute('DROP TABLE active')
|
||
|
|
db.execute('CREATE TABLE active(slot INTEGER PRIMARY KEY,request TEXT)')
|
||
|
|
db.execute("INSERT INTO active VALUES (0,'legacy-unknown')")
|
||
|
|
reopened = Store(self.root)
|
||
|
|
peer = {**upgraded,'slot':1,'screen':'peer','bot':'peer','profile':'/home/lazyboy/.browser-profiles/peer'}
|
||
|
|
with self.assertRaisesRegex(Error, 'GUI_PROFILE_BUSY'):
|
||
|
|
reopened.publish_screen(peer,self.expires)
|
||
|
|
|
||
|
|
def test_invalid_claims_and_unprotected_state_are_rejected(self):
|
||
|
|
for change in [dict(slot=True), dict(slot=8), dict(epoch=-1), dict(epoch=True),
|
||
|
|
dict(generation=0), dict(bot='bad/bot'), dict(lease=''), dict(extra=1)]:
|
||
|
|
with self.assertRaises(Error):
|
||
|
|
self.store.publish_screen({**self.claim, **change}, self.expires)
|
||
|
|
self.store.publish_generation('provider', 1)
|
||
|
|
for expiry in [True, 0, int(time.time() * 1000) + 600000]:
|
||
|
|
with self.assertRaisesRegex(Error, 'INVALID_GUI_LEASE'):
|
||
|
|
self.store.publish_screen(self.claim, expiry)
|
||
|
|
self.root.chmod(0o777)
|
||
|
|
with self.assertRaisesRegex(Error, 'UNPROTECTED_GUI_FENCE_DIRECTORY'):
|
||
|
|
Store(self.root)
|
||
|
|
self.root.chmod(0o700)
|
||
|
|
(self.root / 'gui-slot-0.lock').symlink_to('/dev/null')
|
||
|
|
with self.assertRaises(OSError):
|
||
|
|
self.store.publish_screen(self.claim, self.expires)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
unittest.main()
|