diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bc9f7..43a92f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,10 @@ jobs: run: | python scripts/acceptance/installed-native/test_run.py python -m unittest discover -s scripts/acceptance/installed-native -p test_cell_lifetime.py + - name: Test the condition-history runner without Docker or Cargo + run: python -B -m unittest discover -s scripts/acceptance/condition-history -p 'test_*.py' + - name: Test the one-shot Colima window with fake native controls + run: python -B -m unittest discover -s scripts/acceptance/colima-window -p 'test_*.py' - name: Check committed Cargo locks for known advisories run: python scripts/ci/check-dependency-advisories.py diff --git a/scripts/acceptance/colima-window/README.md b/scripts/acceptance/colima-window/README.md new file mode 100644 index 0000000..2d65a48 --- /dev/null +++ b/scripts/acceptance/colima-window/README.md @@ -0,0 +1,121 @@ +# One-shot default-Colima handoff proposal + +This source accompanies #210. It is not admitted for execution. It preserves the +separately reviewed task controller at `df617fb681958801182e67ca63e886041bec6de0` +and its 13-member manifest +`9ce029dcfb64220ff233e9f42605400ea4a73aceb598272a4034b3e8a65d6e38`. +The outer source is a separate review subject. It does not change the Linux +process epochs, guest/network controls, fixed task store or acceptance rules. + +The selected existing Intel macOS/VZ host cannot currently meet the task's +10-GiB available-memory gate while its default Colima profile runs. General owns +the coordinated idle handoff and final execution decision. Security reviews the +source and exact private invocation. No service stop, launchd bootstrap, VM, +disk allocation, pull or consumer build follows from these source tests. + +Run the controls from the repository root: + +```sh +python3 -B -m unittest discover -s scripts/acceptance/colima-window -p 'test_*.py' +``` + +The tests use synthetic native responses and disposable ordinary files. They +exercise readiness, original-state drift, owner death/PID reuse, partial stop, +restoration faults, survivors and exclusive independent auditing. They neither +contact Docker nor load/unload a service or create a VM. + +## Contingent invocation and custody + +A private proposal supplies the exact source digest, host boot/UID, original +LaunchAgent bytes/effective properties and arguments, seven launcher hashes (including `/bin/ps`), +original profile/configuration hashes, original disk device/inode identities, +local Docker endpoint and idle engine ID/version/architecture/CPU/image/volume +inventory. The supported original service invocation is its existing foreground +`start -f --cpu 4 --memory 6 --disk 60 --vm-type vz --mount-type virtiofs`. +Its existing `KeepAlive={SuccessfulExit:false}` policy is preserved. Files are +never edited, disks never removed, and images/volumes never pruned. Disk contents +can change during an ordinary boot; disk identity, configurations and the engine +inventory must remain unchanged. Usable Linux memory can vary across boots and +must stay within the captured 6-GiB allocation rather than match one page count. + +The proposal leaves the owner epoch null. The foreground native `owner` +invocation fills its own kernel PID/start-microsecond/boot epoch, validates the +proposal, and installs a one-shot user LaunchAgent. That controller has launchd +as its parent, `KeepAlive=false`, `LaunchOnlyOnce=true`, no inherited credentials, +and no output files beyond bounded exclusive JSON receipts. Its source, frozen +SDK source, intent and guest profile are captured arguments. It does not reopen +mutable source or watch-input files. The owner releases `go.json` only after the +actual launchd PID and kernel epoch match the controller's ready receipt. + +Both initial controller bootstrap and later original-service restoration use a +held, unlinked plist descriptor. Read-only native `plutil -lint /dev/fd/N` works +on the selected host. **Actual launchd descriptor consumption is still unproven.** +The initial bootstrap must establish the independent controller before any +original-service stop. Failure has no pathname fallback and authorizes no stop. +The controller rechecks the bound tools, original service/configuration, idle +engine, VM epochs and its own ready receipt immediately before bootout/stop. + +Let `S` be the final admitted outer Unix epoch. No deadline is extended: + +| Stage | Absolute bound | +| --- | --- | +| Owner/watch startup | `S` through `S+30` | +| Original service stop | before `S+120` | +| Frozen task start | `S+120`, at most five seconds late | +| Frozen task active / cleanup | `S+1860` / `S+1920` | +| Original restoration | before `S+2220` | +| Controller removal / final audit | before `S+2280` | + +The frozen task still owns its 300-second setup, 4-vCPU/6-GiB guest, fixed +32-GiB host store, 20-GiB guest store, fresh capacity refusal and unchanged inner +deadlines. This outer controller starts the frozen preparation as its own child, +then consumes its exact ready authority before issuing the create/start request. +No extra VM or dependency operation is added. + +If the foreground owner is hard-stopped, the independent controller retains +custody. Before VM launch it may terminate only its own child and use the +complete frozen cleanup authority. After launch it lets the frozen child reach +its unchanged cleanup cutoff. Restoration requires the child, task VM, image, +mount and instance to be absent; malformed or missing authority cannot substitute +for absence. Any uncertain task cleanup leaves the original service stopped +and calls the existing host `dm` only to report that real fault. Restoration +failure is terminal and also reports the fault. There is no retry or provisioning +fallback. Host reboot or killing the independent controller is outside this +same-boot guarantee and requires operator recovery; no unconditional boot-time +restoration service is installed. + +Only exit 113 with empty stdout and the selected native two-line stderr schema +(`Bad request.` followed by the exact queried label and GUI UID) means a service +is absent. Any mixed, additional or mismatched diagnostic refuses restoration +and auditing. Effective-service arguments are read through the separately +hash-bound and reverified `ps` tool. + +Native failures retain the failed stage, tool/operation, exit status or OS errno, +and each captured stdout/stderr byte count and SHA-256. Excerpts contain only a +fixed reviewed list of diagnostic phrases; arbitrary output, arguments, +environment values and credentials are never copied into receipts or terminals. +Each command still captures at most 2 MiB total. The final result retains the +first eight and latest eight faults plus their total count, including any +failed restoration attempt. Owner bootstrap and controller-removal failures +also retain bounded evidence. These private receipts are recovery evidence, +not additional operating authority. + +A normal restoration bootstraps the captured original plist, starts that same +service and independently reads back its effective arguments/properties, all +original file/disk bindings, idle engine fingerprint and inventories. A receipt +is written before the controller removes its own launchd registration. A +separate `audit` invocation must observe the controller process/job and task +absent and independently repeat the original-engine/file checks. Summary and +audit files are exclusive, and a self-removal failure marker refuses acceptance. +Operational window success never substitutes for the frozen SDK acceptance +summary and independent host/guest audit. + +Private invocation is captured from Git, without executing checkout files: +select the reviewed outer commit's `window.py` bytes, the exact frozen commit's +`guest-host.py` bytes and `guest-profile.json`, verify their reviewed hashes, +then exec the pinned native Python with `-I -S -c OUTER_SOURCE owner --record +PRIVATE_PROPOSAL --record-sha256 PROPOSAL_SHA --sdk-code FROZEN_SOURCE --profile +PRIVATE_PROFILE`. Keep the owner foreground TTY open. General receives the exact +argument vector and private contingent record separately. Final host boot, +launcher/cache/configuration/engine identities, owner epoch, capacity and `S` +remain unbound until a coordinated admission. A stale record refuses execution. diff --git a/scripts/acceptance/colima-window/test_window.py b/scripts/acceptance/colima-window/test_window.py new file mode 100644 index 0000000..0036764 --- /dev/null +++ b/scripts/acceptance/colima-window/test_window.py @@ -0,0 +1,691 @@ +"""Inert lifecycle faults. Native service, Docker and VM calls are all fake.""" +import copy +import importlib.util +import json +import os +import pathlib +import plistlib +import shutil +import sys +import tempfile +import time +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('colima_window_controls', HERE / 'window.py') +window = importlib.util.module_from_spec(spec) +spec.loader.exec_module(window) +CODE = (HERE.parent / 'condition-history/guest-host.py').read_text() +SOURCE = window.sha((HERE / 'window.py').read_bytes()) +TOKEN = '0123456789abcdef' +BOOT = '11111111-1111-1111-1111-111111111111' + + +class Child: + pid = 4321 + alive = True + + def poll(self):return None if self.alive else 0 + def terminate(self):self.alive = False + def wait(self, timeout):self.alive = False + + +class WindowControls(unittest.TestCase): + def setUp(self): + self.temp = pathlib.Path(tempfile.mkdtemp(prefix='sw-', dir='/tmp')).resolve() + self.addCleanup(shutil.rmtree, self.temp) + self.root = self.temp / ('sdk-rust-window-' + TOKEN) + self.root.mkdir(mode=0o700) + self.sdk = window.load_sdk(CODE) + self.owner_pid = 3456 + def epoch(pid): + return dict(pid=pid, pgid=pid, uid=os.getuid(), start_seconds=1000, + start_microseconds=123456, boot_id=BOOT) + self.epoch = epoch + self.processes = {pid:epoch(pid) for pid in (self.owner_pid, os.getpid(), 5678)} + self.sdk.process_identity = lambda pid:copy.deepcopy(self.processes.get(pid)) + self.sdk.boot_identity = lambda:BOOT + self.loaded, self.healthy, self.controller = True, True, True + self.calls, self.bootstraps, self.notices = [], [], [] + self.native_fault, self.bad_properties, self.bad_arguments = None, False, False + self.original_files = {} + tools = {} + for name in ('python', 'launchctl', 'plutil', 'colima', 'docker', 'ps', 'dm'): + path = self.temp / name + path.write_bytes(('fake executable ' + name).encode()) + tools[name] = self.bind(path) + # Native absolute launchctl/plutil paths have synthetic, checked bytes. + for name, path in [('launchctl', '/bin/launchctl'), ('plutil', '/usr/bin/plutil'), ('ps', '/bin/ps')]: + tools[name]['path'] = path + self.original_files[path] = ('fake executable ' + name).encode() + verify = window.verify_file + def fake_system_file(sdk, binding): + if binding['path'] in self.original_files: + raw = self.original_files[binding['path']] + window.require(set(binding) == {'path', 'bytes', 'sha256'} and len(raw) == binding['bytes'] + and window.sha(raw) == binding['sha256'], 'native tool changed') + else:verify(sdk, binding) + self.addCleanup(mock.patch.stopall) + mock.patch.object(window, 'verify_file', side_effect=fake_system_file).start() + home = self.temp / 'profile' + configs = [] + for suffix in ('default/colima.yaml', '_lima/colima/lima.yaml'): + path = home / suffix;path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'unchanged original configuration\n');configs.append(self.bind(path)) + disk = home / 'default/diffdisk';disk.write_bytes(b'original disk') + info = disk.stat() + plist = self.temp / 'original.plist' + self.original_plist = plistlib.dumps({'Label':'org.example.colima', 'RunAtLoad':True, + 'KeepAlive':{'SuccessfulExit':False}, 'ProgramArguments':[tools['colima']['path'], 'start', '-f', '--cpu', '4', '--memory', '6', '--disk', '60', '--vm-type', 'vz', '--mount-type', 'virtiofs'], + 'EnvironmentVariables':{'COLIMA_HOME':str(home), 'PATH':'/original/bin:/usr/bin:/bin'}}) + plist.write_bytes(self.original_plist) + started = int(time.time()) + task = {key:'unused' for key in self.sdk.INTENT_KEYS} + task.update(schema='durable-workflow.condition-host-setup/v1', token=TOKEN, + runner_commit=window.SDK_R, manifest_sha256=window.SDK_M, guardian_sha256=window.SDK_CODE_SHA, + started_unix=started + 120, active_unix=started + 1860, cleanup_unix=started + 1920, + parent=str(self.temp / ('sr-' + TOKEN)), host_boot_id=BOOT) + self.engine = {'info':{'ID':'original-engine', 'ServerVersion':'29.5.2', 'Architecture':'x86_64', + 'OSType':'linux', 'NCPU':4, 'DockerRootDir':'/var/lib/docker'}, + 'memory_ceiling_bytes':6 * 1024**3, 'images':['sha256:public'], 'volumes':['original-volume']} + self.record = dict(schema='durable-workflow.colima-window/v1', token=TOKEN, source_sha256=SOURCE, + host_boot_id=BOOT, uid=os.getuid(), scope=str(self.root), owner_process=epoch(self.owner_pid), + started_unix=started, sdk_intent=task, tools=tools, profile_home=str(home), + service=dict(label='org.example.colima', plist=self.bind(plist), + arguments=plistlib.loads(self.original_plist)['ProgramArguments'], keep_alive={'SuccessfulExit':False}, + properties='runatload | inferred program', vm_processes=[epoch(5678)]), + config_files=configs, disks=[dict(path=str(disk), device=info.st_dev, inode=info.st_ino)], + docker_host='unix://' + str(home / 'default/docker.sock'), engine=copy.deepcopy(self.engine)) + self.intent_sha = window.sha(window.canonical(self.record)) + self.w = window.Window(self.sdk, self.record, self.intent_sha, CODE) + self.w.call = self.call + self.put('go.json', {'intent_sha256':self.intent_sha, 'action':'one-window'}) + + def bind(self, path): + raw = path.read_bytes() + return dict(path=str(path), bytes=len(raw), sha256=window.sha(raw)) + + def put(self, name, value): + (self.root / name).write_bytes(window.canonical(value)) + + def ready(self): + self.put('window-ready.json', dict(intent_sha256=self.intent_sha, + source_sha256=SOURCE, process=self.epoch(os.getpid()))) + + def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): + window.require(cutoff > time.monotonic(), 'fake fixed cutoff expired') + self.calls.append(argv) + name = pathlib.Path(argv[0]).name + if name == self.native_fault:raise SystemExit('injected native fault') + args = argv[1:] + if name == 'launchctl': + if args[0] == 'print': + controller = args[1].endswith(window.job_label(self.record)) + if not (self.controller if controller else self.loaded): + label=args[1].split('/')[-1] + return 113, b'', ('Bad request.\nCould not find service \"'+label+'\" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + pid = os.getpid() if controller else 6789 + program = self.record['tools']['python' if controller else 'colima']['path'] + properties = 'wrong' if self.bad_properties else self.record['service']['properties'] + # Nested fields and source text must never become root authority. + return 0, (f'job = {{\n\tprogram = {program}\n\tpid = {pid}\n' + f'\tproperties = {properties}\n\targuments = {{\n\t\tpid = 9999\n' + '\t\tprogram = attacker\n pid = 8888\n\t}\n}\n').encode(), b'' + if args[0] == 'bootout':self.loaded = False + elif args[0] == 'bootstrap': + descriptor = int(args[-1].split('/')[-1]);self.assertIn(descriptor, pass_fds) + os.lseek(descriptor, 0, os.SEEK_SET) + self.bootstraps.append(os.read(descriptor, 512 * 1024)) + self.loaded = True + elif args[0] == 'kickstart':self.healthy = True + elif args[0] == 'remove':self.controller = False + else:self.failTest('unexpected service mutation ' + str(args)) + elif name == 'colima': + self.assertEqual(args, ['--profile', 'default', 'stop']);self.assertTrue(stop) + self.healthy = False;self.processes.pop(5678, None) + elif name == 'ps': + values = ['different'] if self.bad_arguments else self.record['service']['arguments'] + return 0, (' '.join(values)).encode(), b'' + elif name == 'docker': + self.assertEqual(args[:2], ['--host', self.record['docker_host']]) + window.require(self.healthy, 'fake original engine unavailable') + operation = args[2:] + if operation[0] == 'info':return 0, window.canonical(dict(self.engine['info'], MemTotal=6 * 1024**3)), b'' + if operation[0] == 'ps':return 0, b'', b'' + if operation[0] == 'image':return 0, '\n'.join(self.engine['images']).encode(), b'' + if operation[0] == 'volume':return 0, '\n'.join(self.engine['volumes']).encode(), b'' + self.failTest('unexpected Docker operation') + elif name == 'dm':self.notices.append(args[0]) + elif name != 'plutil':self.failTest('unexpected native command ' + name) + return 0, b'', b'' + + def run_fake_task(self, *, survive=False, fail=False): + def start(profile): + self.child = Child();self.w.child = self.child + self.processes.pop(self.owner_pid, None) # hard owner stop after handoff + def monitor(): + if not survive:self.child.alive = False + if fail:raise SystemExit('injected task failure') + self.w.start_task = start;self.w.monitor_task = monitor + self.w.run(SOURCE, {}) + return json.loads((self.root / 'window-result.json').read_bytes()) + + def test_frozen_source_is_hash_bound(self): + self.assertEqual(window.sha(CODE.encode()), window.SDK_CODE_SHA) + with self.assertRaises(SystemExit):window.load_sdk(CODE + '\n') + + def test_complete_private_record_validates(self): + window.validate(self.sdk, self.record, SOURCE) + self.assertEqual(self.w.engine(), self.record['engine']) + + def test_closed_source_epoch_cutoffs_and_scope(self): + mutations = [lambda r:r.update(extra=True), lambda r:r.update(source_sha256='f'*64), + lambda r:r.update(host_boot_id='old-boot'), lambda r:r['owner_process'].update(start_microseconds=1_000_000), + lambda r:r['sdk_intent'].update(runner_commit='f'*40), + lambda r:r['sdk_intent'].update(cleanup_unix=r['sdk_intent']['cleanup_unix']+1), + lambda r:r['tools']['launchctl'].update(path='/tmp/launchctl'), + lambda r:r['service'].update(keep_alive=True), + lambda r:r.update(docker_host='tcp://127.0.0.1:2375'), + lambda r:r.update(scope=str(self.temp)), lambda r:r.update(config_files=[])] + for mutate in mutations: + with self.subTest(mutate=mutate): + candidate = copy.deepcopy(self.record);mutate(candidate) + with self.assertRaises(SystemExit):window.validate(self.sdk, candidate, SOURCE) + + def test_original_tool_config_plist_and_disk_drift_refuse_before_stop(self): + paths = [self.record['tools']['docker']['path'], self.record['config_files'][0]['path'], + self.record['service']['plist']['path'], self.record['disks'][0]['path']] + for name in paths: + with self.subTest(path=name): + path = pathlib.Path(name);raw = path.read_bytes() + if name == self.record['disks'][0]['path']: + moved = path.with_suffix('.held');path.rename(moved);path.write_bytes(raw) + else:path.write_bytes(raw + b'changed') + self.ready() + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + path.unlink() + if name == self.record['disks'][0]['path']:moved.rename(path) + else:path.write_bytes(raw) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_missing_or_tampered_outside_ready_refuses_stop(self): + with self.assertRaises(FileNotFoundError):self.w.stop_default() + self.ready();value = json.loads((self.root/'window-ready.json').read_bytes()) + for key, replacement in [('source_sha256','f'*64), ('intent_sha256','f'*64), ('process',self.epoch(9999))]: + with self.subTest(key=key): + self.put('window-ready.json', dict(value, **{key:replacement})) + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_kernel_owner_loss_and_pid_reuse_refuse_stop(self): + self.ready() + for process in (None, dict(self.epoch(self.owner_pid), start_microseconds=123457)): + self.processes[self.owner_pid] = process + self.assertFalse(self.w.owner_alive()) + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_effective_service_arguments_properties_and_engine_drift_refuse(self): + self.ready() + for field in ('bad_arguments','bad_properties'): + setattr(self, field, True) + with self.assertRaises(SystemExit):self.w.stop_default() + setattr(self, field, False) + self.engine['images'].append('foreign-image') + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_top_level_job_schema_ignores_nested_source_fields(self): + self.assertEqual(self.w.job('org.example.colima')['pid'], 6789) + self.assertEqual(self.w.service_matches()['program'], self.record['tools']['colima']['path']) + + def test_original_vm_epoch_and_fixed_stop_cutoff_refuse(self): + self.ready();self.processes[5678]['start_microseconds'] += 1 + with self.assertRaises(SystemExit):self.w.stop_default() + self.processes[5678] = self.epoch(5678);self.w.stop_cutoff = time.monotonic()-1 + with self.assertRaises(SystemExit):self.w.stop_default() + self.assertFalse(self.w.stop_attempted) + + def test_hard_owner_stop_leaves_independent_restore_and_controller_cleanup(self): + with mock.patch.object(window.subprocess, 'Popen', side_effect=AssertionError('no native experiment')): + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'pending_independent_audit') + self.assertTrue(result['task_absent']);self.assertTrue(self.loaded and self.healthy) + self.assertFalse(self.controller) + self.assertEqual(self.bootstraps, [self.original_plist]) + self.assertFalse(self.notices) + + def test_partial_stop_fault_restores_original_service(self): + self.native_fault = 'colima' + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'fail') + self.assertTrue(result['original_engine_unchanged']);self.assertTrue(self.loaded) + self.assertEqual(self.bootstraps, [self.original_plist]);self.assertFalse(self.controller) + + def test_preflight_failure_still_removes_controller(self): + pathlib.Path(self.record['config_files'][0]['path']).write_bytes(b'drift') + result = self.run_fake_task() + self.assertEqual(result['outcome'], 'fail');self.assertFalse(self.controller) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_surviving_child_blocks_restore_and_notifies_operator(self): + result = self.run_fake_task(survive=True) + self.assertEqual(result['outcome'], 'fail');self.assertFalse(self.loaded) + self.assertTrue(result['operator_notified']);self.assertTrue(self.notices[0].startswith('sdk-rust:')) + self.assertFalse(self.bootstraps);self.assertFalse(self.controller) + + def test_task_failure_with_proven_absence_restores_without_claiming_pass(self): + result = self.run_fake_task(fail=True) + self.assertEqual(result['outcome'], 'fail');self.assertTrue(result['task_absent']) + self.assertTrue(self.loaded and self.healthy);self.assertFalse(self.notices) + + def test_restore_consumes_held_plist_not_mutable_path(self): + self.loaded = False + raw = self.original_plist + def mutate_during_lint(argv, cutoff, **kwargs): + if pathlib.Path(argv[0]).name == 'plutil': + pathlib.Path(self.record['service']['plist']['path']).write_bytes(b'edited then restored') + return self.call(argv, cutoff, **kwargs) + self.w.call = mutate_during_lint + self.w.bootstrap_fd(raw) + self.assertEqual(self.bootstraps, [raw]) + + def test_restoration_inventory_mismatch_is_terminal(self): + self.ready();self.w.stop_default() + self.engine['volumes'].append('foreign') + self.w.restore_cutoff = time.monotonic() + 0.02 + with mock.patch.object(window.time, 'sleep'), self.assertRaises(SystemExit):self.w.restore(self.original_plist) + + def test_uncertain_service_query_cannot_authorize_bootstrap(self): + self.w.call = lambda *args, **kwargs:(5,b'',b'permission denied') + with self.assertRaises(SystemExit):self.w.restore(self.original_plist) + self.assertFalse(self.bootstraps) + + def test_task_image_instance_and_recorded_process_survivors_refuse(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], TOKEN) + scoped['parent'].mkdir(mode=0o700) + for path in (scoped['image'], scoped['instance']): + path.parent.mkdir(parents=True, exist_ok=True);path.write_bytes(b'survivor') + self.assertFalse(self.w.task_absent());path.unlink() + for name,key in [('vm-process.json',None),('host-ready.json','process'),('vm-started.json','host_process')]: + process = self.epoch(5678) + path = scoped['parent']/name;path.write_bytes(window.canonical({key:process} if key else process)) + self.assertFalse(self.w.task_absent());path.unlink() + (scoped['parent']/'host-ready.json').write_bytes(b'not JSON') + with self.assertRaises(ValueError):self.w.task_absent() + + def test_prelaunch_owner_loss_aborts_owned_child_then_frozen_cleanup(self): + self.w.child = Child();self.w.task_record = {'frozen':'authority'} + self.processes.pop(self.owner_pid) + with mock.patch.object(self.sdk, 'cleanup') as cleanup: + self.w.monitor_task() + cleanup.assert_called_once_with(self.w.task_record) + self.assertIsNotNone(self.w.child.poll());self.assertFalse(self.w.launched) + + def test_postlaunch_owner_loss_preserves_absolute_child_cleanup(self): + self.w.child = Child();self.w.task_record = {};self.w.launched = True + self.processes.pop(self.owner_pid) + def finish(_):self.w.child.alive = False + with mock.patch.object(self.sdk, 'cleanup') as cleanup, mock.patch.object(window.time,'sleep',side_effect=finish): + self.w.monitor_task() + cleanup.assert_not_called();self.assertTrue(self.w.task_absent()) + + def test_live_owner_launches_exact_frozen_request_once(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'],TOKEN);scoped['parent'].mkdir() + self.w.child = Child();self.w.task_record = {};self.w.task_record_sha = 'd'*64 + def finish(_):self.w.child.alive = False + with mock.patch.object(window.time,'sleep',side_effect=finish):self.w.monitor_task() + self.assertEqual(json.loads((scoped['parent']/'launch.json').read_bytes()), + dict(record_sha256='d'*64,manifest_sha256=window.SDK_M,action='create-and-start')) + + def test_foreign_child_readiness_refuses_before_authority(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'],TOKEN);scoped['parent'].mkdir() + self.w.child = Child() + (scoped['parent']/'host-ready.json').write_bytes(window.canonical(dict(process=self.epoch(9999), + source_sha256=window.SDK_CODE_SHA, record_sha256='d'*64))) + with mock.patch.object(self.sdk,'authority') as authority, self.assertRaises(SystemExit):self.w.capture_ready() + authority.assert_not_called() + + def test_result_overwrite_refuses_and_still_removes_controller(self): + self.put('window-result.json', {'forged':'pass'}) + with self.assertRaises(FileExistsError):self.run_fake_task() + self.assertEqual(json.loads((self.root/'window-result.json').read_bytes()),{'forged':'pass'}) + self.assertFalse(self.controller) + + def completed(self): + self.run_fake_task();self.processes.pop(os.getpid()) + + def test_independent_final_audit_passes_and_is_exclusive(self): + self.completed();window.audit(self.w,SOURCE) + self.assertEqual(json.loads((self.root/'window-audit.json').read_bytes())['outcome'],'pass') + with self.assertRaises(FileExistsError):window.audit(self.w,SOURCE) + + def test_final_audit_refuses_controller_job_task_and_engine_survivors(self): + self.completed() + for fault in ('process','job','task','engine','marker','digest'): + with self.subTest(fault=fault): + if fault=='process':self.processes[os.getpid()] = self.epoch(os.getpid()) + elif fault=='job':self.controller = True + elif fault=='task':self.w.child.alive = True + elif fault=='engine':self.engine['images'].append('drift') + elif fault=='marker':self.put('controller-cleanup-failure.json',{'outcome':'fail'}) + else: + saved = (self.root/'window-result.json').read_bytes() + value=json.loads(saved);value['intent_sha256']='foreign';self.put('window-result.json',value) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + self.assertFalse((self.root/'window-audit.json').exists()) + self.processes.pop(os.getpid(),None);self.controller=False;self.w.child.alive=False + self.engine['images'][:]=['sha256:public'] + (self.root/'controller-cleanup-failure.json').unlink(missing_ok=True) + if fault=='digest':(self.root/'window-result.json').write_bytes(saved) + + def test_controller_remove_failure_records_fault_and_prevents_acceptance(self): + call=self.call + def broken(argv, cutoff, **kwargs): + if 'remove' in argv:raise SystemExit('self removal failed') + return call(argv,cutoff,**kwargs) + self.w.call=broken + with self.assertRaises(SystemExit):self.run_fake_task() + self.assertTrue((self.root/'controller-cleanup-failure.json').exists()) + self.assertTrue(self.notices) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + + def test_owner_binds_one_shot_launchd_job_and_releases_only_exact_ready(self): + proposal=copy.deepcopy(self.record);proposal['owner_process']=None + proposal['tools']['python']['path']=str(pathlib.Path(sys.executable).resolve()) + (self.root/'go.json').unlink() + captured=[] + def bootstrap(raw): + job=plistlib.loads(raw);captured.append(job) + argv=job['ProgramArguments'] + record=json.loads(argv[argv.index('--intent-json')+1]) + digest=argv[argv.index('--record-sha256')+1] + self.put('window-ready.json',dict(intent_sha256=digest,source_sha256=SOURCE,process=self.epoch(4321))) + self.assertEqual(record['owner_process'],self.epoch(os.getpid())) + self.assertFalse((self.root/'go.json').exists()) + self.processes[4321]=self.epoch(4321) + self.w.bootstrap_fd=bootstrap + self.w.job=mock.Mock(return_value={'pid':4321}) + args=type('Args',(),{'sdk_code':CODE,'profile':str(self.root/'profile')})() + profile=(HERE.parent/'condition-history/guest-profile.json').read_bytes() + code=(HERE/'window.py').read_text() + with mock.patch.object(window.os,'isatty',return_value=True), mock.patch.object(window.sys,'platform','darwin'), \ + mock.patch.object(window,'validate'), mock.patch.object(window,'Window',return_value=self.w), \ + mock.patch.object(window.selectors,'DefaultSelector',side_effect=SystemExit('test after go')), \ + mock.patch('builtins.print'), self.assertRaisesRegex(SystemExit,'test after go'): + window.owner(self.sdk,proposal,'a'*64,args,code,profile) + job=captured[0] + self.assertFalse(job['KeepAlive']);self.assertTrue(job['RunAtLoad'] and job['LaunchOnlyOnce']) + self.assertFalse(job['AbandonProcessGroup']) + self.assertEqual(job['StandardOutPath'],'/dev/null');self.assertEqual(job['StandardErrorPath'],'/dev/null') + self.assertEqual(job['ProgramArguments'][1:4],['-I','-S','-c']) + self.assertIn('--intent-json',job['ProgramArguments']);self.assertIn('--profile-json',job['ProgramArguments']) + self.assertTrue((self.root/'go.json').exists()) + self.assertEqual(json.loads((self.root/'docker-config/config.json').read_bytes()),{'auths':{}}) + + def test_late_audit_does_not_extend_absolute_cutoffs(self): + later = self.record['sdk_intent']['cleanup_unix'] + 500 + with mock.patch.object(window.time, 'time', return_value=later): + subject = window.Window(self.sdk, self.record, self.intent_sha, CODE) + self.assertLessEqual(subject.cleanup_cutoff, time.monotonic()) + self.assertLessEqual(subject.restore_cutoff, time.monotonic()) + + def test_watch_validation_failure_removes_its_own_registered_job(self): + args = ['watch', '--record', str(self.root/'unused'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'unused-profile'), + '--intent-json', window.canonical(self.record).decode(), + '--profile-json', (HERE.parent/'condition-history/guest-profile.json').read_text()] + self.w.job = mock.Mock(return_value={'pid':os.getpid()}) + with mock.patch.object(window.sys, 'orig_argv', ['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys, 'platform', 'darwin'), mock.patch.object(window.os, 'getppid', return_value=1), \ + mock.patch.object(window, 'load_sdk', return_value=self.sdk), \ + mock.patch.object(window, 'Window', return_value=self.w), \ + mock.patch.object(window, 'validate', side_effect=SystemExit('original drift')), self.assertRaises(SystemExit): + window.main(args) + self.assertFalse(self.controller) + self.assertFalse(any('bootout' in call for call in self.calls)) + + def test_watch_refuses_agent_parent_before_native_service_actions(self): + args = ['watch', '--record', str(self.root/'unused'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'unused-profile'), + '--intent-json', window.canonical(self.record).decode(), + '--profile-json', (HERE.parent/'condition-history/guest-profile.json').read_text()] + with mock.patch.object(window.sys, 'orig_argv', ['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys, 'platform', 'darwin'), mock.patch.object(window.os,'getppid',return_value=9999), \ + mock.patch.object(window, 'load_sdk',return_value=self.sdk), \ + mock.patch.object(window, 'Window',return_value=self.w), self.assertRaises(SystemExit):window.main(args) + self.assertFalse(self.calls) + + def test_captured_watch_does_not_reopen_mutable_record_or_profile(self): + profile = (HERE.parent/'condition-history/guest-profile.json').read_text() + args = ['watch', '--record', str(self.root/'missing'), '--record-sha256', self.intent_sha, + '--sdk-code', CODE, '--profile', str(self.root/'missing-profile'), + '--intent-json', window.canonical(self.record).decode(), '--profile-json', profile] + self.w.job = mock.Mock(return_value={'pid':os.getpid()}) + self.record['tools']['python']['path'] = str(pathlib.Path(sys.executable).resolve()) + captured = window.canonical(self.record).decode() + args[args.index('--intent-json')+1] = captured + args[args.index('--record-sha256')+1] = window.sha(captured.encode()) + with mock.patch.object(window.sys,'orig_argv',['python','-I','-S','-c',(HERE/'window.py').read_text()]), \ + mock.patch.object(window.sys,'platform','darwin'), mock.patch.object(window.os,'getppid',return_value=1), \ + mock.patch.object(window,'load_sdk',return_value=self.sdk), mock.patch.object(window,'Window',return_value=self.w), \ + mock.patch.object(window,'validate') as validate, mock.patch.object(self.w,'run') as run: + window.main(args) + validate.assert_called_once();run.assert_called_once_with(SOURCE,json.loads(profile)) + + def test_late_task_start_refuses_before_popen(self): + self.record['sdk_intent']['started_unix'] = int(time.time()) - 10 + with mock.patch.object(window.subprocess,'Popen') as popen, self.assertRaises(SystemExit):self.w.start_task({}) + popen.assert_not_called() + + def test_restoration_fault_notifies_without_retry_or_pass(self): + call=self.call + attempts=[] + def broken(argv,cutoff,**kwargs): + if 'bootstrap' in argv: + attempts.append(argv) + raise SystemExit('restoration bootstrap failed') + return call(argv,cutoff,**kwargs) + self.w.call=broken + result=self.run_fake_task() + self.assertEqual(result['outcome'],'fail');self.assertTrue(result['operator_notified']) + self.assertFalse(self.loaded);self.assertFalse(self.controller) + self.assertEqual(len([x for x in self.calls if 'bootout' in x]),1) + self.assertEqual(len(attempts),1) + self.assertFalse(self.bootstraps) + + def test_original_file_symlink_hardlink_and_writable_input_refuse(self): + path=self.temp/'input';path.write_bytes(b'original') + binding=self.bind(path) + link=self.temp/'hardlink';os.link(path,link) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,binding) + link.unlink();path.chmod(0o666) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,binding) + path.chmod(0o600);link.symlink_to(path) + with self.assertRaises(SystemExit):window.verify_file(self.sdk,dict(binding,path=str(link))) + + def test_permission_error_cannot_be_hidden_in_absence_diagnostic(self): + label=self.record['service']['label'] + error=('permission denied\nBad request.\nCould not find service "'+label+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',error) + with self.assertRaises(SystemExit):self.w.job(label) + + def test_only_exact_native_absence_status_and_diagnostics_are_accepted(self): + label=self.record['service']['label'] + exact=('Bad request.\nCould not find service "'+label+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',exact) + self.assertIsNone(self.w.job(label)) + self.assertEqual(self.w.native_fault_count,0) + cases=[(1,b'',exact),(5,b'',exact),(113,b'unexpected stdout',exact), + (113,b'',b'permission denied\n'+exact),(113,b'',exact+b'permission denied\n'), + (113,b'',b'Could not find service'),(113,b'',exact.replace(label.encode(),b'other-service')), + (113,b'',exact.replace(str(self.record['uid']).encode(),b'99999')), + (113,b'',exact.replace(b'\n',b'\r\n')),(0,b'',b'permission denied')] + for response in cases: + with self.subTest(response=response): + self.w.call=lambda *args,**kwargs:response + with self.assertRaises(SystemExit):self.w.job(label) + self.assertEqual(self.w.native_fault_count,len(cases)) + + def test_mixed_absence_cannot_bootstrap_restore_or_authorize_final_audit(self): + self.completed() + exact=('Bad request.\nCould not find service "'+window.job_label(self.record)+'" in domain for user gui: '+str(self.record['uid'])+'\n').encode() + self.w.call=lambda *args,**kwargs:(113,b'',b'permission denied\n'+exact) + with self.assertRaises(SystemExit):window.audit(self.w,SOURCE) + with self.assertRaises(SystemExit):self.w.restore(self.original_plist) + self.assertFalse((self.root/'window-audit.json').exists()) + self.assertEqual(self.bootstraps,[self.original_plist]) + + def test_ps_must_be_bound_and_reverified_before_effective_argument_readback(self): + candidate=copy.deepcopy(self.record);candidate['tools'].pop('ps') + with self.assertRaises(SystemExit):window.validate(self.sdk,candidate,SOURCE) + self.original_files['/bin/ps']=b'different native ps' + with self.assertRaises(SystemExit):self.w.service_matches() + self.assertFalse(any(pathlib.Path(call[0]).name=='ps' for call in self.calls)) + self.assertEqual(self.w.native_faults[-1]['tool'],'ps') + + def test_native_status_and_safe_diagnostic_digests_survive_stop_failure(self): + original=self.call + stdout=b'private diagnostic payload' + stderr=b'permission denied token=never-echo-this-value' + def stop_fault(argv,cutoff,**kwargs): + if pathlib.Path(argv[0]).name=='colima':return 77,stdout,stderr + return original(argv,cutoff,**kwargs) + self.w.call=stop_fault + result=self.run_fake_task() + self.assertIn('failed_stage',result) + self.assertEqual(result['failed_stage'],'stop-default') + fault=result['native_faults'][0] + self.assertEqual((fault['tool'],fault['operation'],fault['exit_status']),('colima','stop',77)) + self.assertEqual(fault['stdout']['sha256'],window.sha(stdout)) + self.assertEqual(fault['stderr']['sha256'],window.sha(stderr)) + self.assertEqual(fault['stderr']['safe_excerpt'],['permission denied']) + self.assertNotIn('never-echo-this-value',json.dumps(result)) + self.assertNotIn('private diagnostic payload',json.dumps(result)) + self.assertTrue(result['original_engine_unchanged']) + + def test_failed_recovery_restoration_retains_both_stages_and_diagnostics(self): + original=self.call + def faults(argv,cutoff,**kwargs): + if pathlib.Path(argv[0]).name=='colima':return 77,b'first private failure',b'permission denied' + if 'bootstrap' in argv:return 88,b'restore private failure',b'input/output error' + return original(argv,cutoff,**kwargs) + self.w.call=faults + result=self.run_fake_task() + self.assertIn('failed_stage',result) + self.assertEqual(result['failed_stage'],'stop-default') + self.assertEqual(result['restoration_failure']['failed_stage'],'restore-after-fault') + self.assertEqual([(item['stage'],item['exit_status']) for item in result['native_faults']], + [('stop-default',77),('restore-after-fault',88)]) + self.assertTrue(result['operator_notified']) + + def test_normal_restoration_native_failure_is_terminal_and_recoverable(self): + original=self.call;attempts=[] + def faults(argv,cutoff,**kwargs): + if 'bootstrap' in argv: + attempts.append(argv);return 88,b'',b'input/output error' + return original(argv,cutoff,**kwargs) + self.w.call=faults + result=self.run_fake_task() + self.assertEqual(len(attempts),1) + self.assertEqual(result['failed_stage'],'restore-original') + self.assertEqual(result['native_faults'][-1]['exit_status'],88) + self.assertTrue(result['operator_notified']) + + def test_fault_evidence_has_fixed_size_and_retains_early_and_late_failures(self): + self.w.call=lambda *args,**kwargs:(77,b'private stdout',b'permission denied') + for number in range(40): + self.w.stage='restore-original' + with self.assertRaises(SystemExit):self.w.native('launchctl','kickstart','ignored') + self.w.native_faults[-1]['sequence_for_test']=number + self.assertEqual(self.w.native_fault_count,40) + self.assertEqual(len(self.w.native_faults),16) + self.assertEqual([item['sequence_for_test'] for item in self.w.native_faults],list(range(8))+list(range(32,40))) + self.assertLess(len(window.canonical(self.w.failure_details(SystemExit()))),16000) + + def test_bounded_call_retains_actual_nonzero_status_and_captured_bytes(self): + class FakeNativeChild: + returncode=91 + def __init__(self): + read,write=os.pipe();os.write(write,b'private stdout');os.close(write);self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.write(write,b'permission denied private token');os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + self.w.stage='stop-default' + with mock.patch.object(window.subprocess,'Popen',return_value=FakeNativeChild()), self.assertRaises(window.NativeFailure): + self.w.native('colima','--profile','default','stop') + fault=self.w.native_faults[-1] + self.assertEqual(fault['exit_status'],91) + self.assertEqual(fault['stdout']['sha256'],window.sha(b'private stdout')) + self.assertEqual(fault['stderr']['sha256'],window.sha(b'permission denied private token')) + self.assertNotIn('private token',json.dumps(fault)) + + def test_owner_bootstrap_failure_retains_private_recovery_evidence(self): + proposal=copy.deepcopy(self.record);proposal['owner_process']=None + proposal['tools']['python']['path']=str(pathlib.Path(sys.executable).resolve()) + (self.root/'go.json').unlink() + args=type('Args',(),{'sdk_code':CODE,'profile':str(self.root/'profile')})() + profile=(HERE.parent/'condition-history/guest-profile.json').read_bytes() + code=(HERE/'window.py').read_text() + self.w.call=lambda *args,**kwargs:(5,b'private bootstrap payload',b'permission denied') + with mock.patch.object(window.os,'isatty',return_value=True), mock.patch.object(window.sys,'platform','darwin'), \ + mock.patch.object(window,'validate'), mock.patch.object(window,'Window',return_value=self.w), \ + self.assertRaises(window.NativeFailure): + window.owner(self.sdk,proposal,'a'*64,args,code,profile) + result=json.loads((self.root/'owner-failure.json').read_bytes()) + self.assertEqual(result['failed_stage'],'owner-bootstrap') + self.assertEqual(result['native_faults'][0]['exit_status'],5) + self.assertEqual(result['native_faults'][0]['tool'],'plutil') + self.assertFalse((self.root/'go.json').exists()) + self.assertNotIn('private bootstrap payload',json.dumps(result)) + + def test_timeout_fault_retains_killed_status_without_a_native_process(self): + class WaitingChild: + returncode=None + def __init__(self): + read,self.writer=os.pipe();self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def kill(self):self.returncode=-9;os.close(self.writer) + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + self.w.stage='restore-original' + cutoff=time.monotonic()+0.03 + with mock.patch.object(window.subprocess,'Popen',return_value=WaitingChild()), self.assertRaises(window.NativeFailure): + self.w.native('launchctl','kickstart','ignored',cutoff=cutoff) + fault=self.w.native_faults[-1] + self.assertEqual((fault['reason'],fault['exit_status']),('timeout',-9)) + self.assertEqual(fault['stage'],'restore-original') + self.assertEqual(fault['stdout']['bytes']+fault['stderr']['bytes'],0) + + def test_oversize_diagnostic_binds_only_the_bounded_capture(self): + class FakeNativeChild: + returncode=91 + def __init__(self): + read,write=os.pipe();os.write(write,b'x');os.close(write);self.stdout=os.fdopen(read,'rb') + read,write=os.pipe();os.close(write);self.stderr=os.fdopen(read,'rb') + def poll(self):return self.returncode + def wait(self,timeout):return self.returncode + self.w.call=window.Window.call.__get__(self.w,window.Window) + child=FakeNativeChild() + with mock.patch.object(window.subprocess,'Popen',return_value=child), \ + mock.patch.object(window.os,'read',return_value=b'x'*65536), self.assertRaises(window.NativeFailure): + self.w.native('launchctl','print','ignored') + fault=self.w.native_faults[-1] + self.assertEqual(fault['reason'],'output-limit') + self.assertEqual(fault['stdout']['bytes']+fault['stderr']['bytes'],2*1024**2) + self.assertLess(len(window.canonical(fault)),2000) + + def test_minimal_environment_excludes_forwarded_credentials_and_default_lima_home(self): + with mock.patch.dict(os.environ,{'SSH_AUTH_SOCK':'secret','DOCKER_AUTH_CONFIG':'secret','LIMA_HOME':'foreign'}): + env=self.w.env();stop_env=self.w.env(stop=True) + self.assertEqual(set(env),{'PATH','HOME','TMPDIR','DOCKER_CONFIG'}) + self.assertEqual(stop_env['COLIMA_HOME'],self.record['profile_home']) + self.assertEqual(stop_env['PATH'],'/original/bin:/usr/bin:/bin') + self.assertNotIn('LIMA_HOME',stop_env) + + +if __name__=='__main__':unittest.main() diff --git a/scripts/acceptance/colima-window/window.py b/scripts/acceptance/colima-window/window.py new file mode 100644 index 0000000..267169a --- /dev/null +++ b/scripts/acceptance/colima-window/window.py @@ -0,0 +1,634 @@ +"""One-shot shared-Colima window proposal. No native operation is admitted. + +The separately frozen task controller remains df617fb. This outer subject owns +the original service handoff, restoration, and its own launchd registration. +""" +import argparse +import hashlib +import json +import os +import pathlib +import plistlib +import re +import selectors +import shlex +import signal +import stat +import subprocess +import sys +import tempfile +import time +import types + +SDK_R = 'df617fb681958801182e67ca63e886041bec6de0' +SDK_M = '9ce029dcfb64220ff233e9f42605400ea4a73aceb598272a4034b3e8a65d6e38' +SDK_CODE_SHA = 'bf9dbd8166c71dd1b87bc5761640ce26d07e624d56b19d26e40e5ff02eeab9a1' +KEYS = {'schema', 'token', 'source_sha256', 'host_boot_id', 'uid', 'scope', 'owner_process', + 'started_unix', 'sdk_intent', 'tools', 'service', 'config_files', 'disks', + 'docker_host', 'engine', 'profile_home'} +SAFE_DIAGNOSTICS = (b'permission denied', b'operation not permitted', b'input/output error', + b'could not find service', b'bad request', b'connection refused', + b'cannot connect to the docker daemon', b'no such file or directory') + + +class NativeFailure(SystemExit): + def __init__(self, reason, status, output=b'', errors=b'', os_errno=None): + super().__init__(reason) + self.reason, self.status, self.output, self.errors = reason, status, output, errors + self.os_errno = os_errno + + +def diagnostic(data): + # No arbitrary diagnostic bytes enter the receipt or terminal. Only these + # fixed phrases are exposed; the complete bounded captured bytes are hashed. + lower = data.lower() + return {'bytes': len(data), 'sha256': sha(data), + 'safe_excerpt': [phrase.decode() for phrase in SAFE_DIAGNOSTICS if phrase in lower]} + + +def require(ok, message): + if not ok: + raise SystemExit(message) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def load_sdk(code): + require(sha(code.encode()) == SDK_CODE_SHA, 'wrong frozen task controller') + module = types.ModuleType('frozen_df617fb_guest') + exec(compile(code, 'git:' + SDK_R + ':guest-host.py', 'exec'), module.__dict__) + return module + + +def root_of(record): + return pathlib.Path(record['scope']) + + +def job_label(record): + return 'sdk-rust-colima-window-' + record['token'] + + +def target(record, label=None): + return 'gui/' + str(record['uid']) + '/' + (label or job_label(record)) + + +def verify_file(sdk, binding): + require(set(binding) == {'path', 'bytes', 'sha256'}, 'wrong file binding') + path = pathlib.Path(binding['path']) + require(path.is_absolute() and path.resolve() == path, 'noncanonical original input') + require(type(binding['bytes']) is int and 0 < binding['bytes'] <= 64 * 1024**2, 'invalid original input bound') + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_uid in (0, os.getuid()) + and before.st_nlink == 1 and not before.st_mode & 0o022, 'unsafe original input') + sdk.input_bytes(path, binding['bytes'], binding['sha256']) + + +def verify_original(sdk, record): + for binding in [*record['tools'].values(), *record['config_files']]: + verify_file(sdk, binding) + for disk in record['disks']: + require(set(disk) == {'path', 'device', 'inode'}, 'wrong original disk binding') + path = pathlib.Path(disk['path']) + info = path.lstat() + require(path.is_absolute() and path.resolve() == path and stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 and info.st_uid == record['uid'] + and (info.st_dev, info.st_ino) == (disk['device'], disk['inode']), 'original disk identity changed') + service = record['service'] + verify_file(sdk, service['plist']) + raw = sdk.regular(pathlib.Path(service['plist']['path']), 64 * 1024) + require(sha(raw) == service['plist']['sha256'], 'original LaunchAgent changed') + value = plistlib.loads(raw) + require(value['Label'] == service['label'] and value['RunAtLoad'] is True + and value['KeepAlive'] == service['keep_alive'] + and value['ProgramArguments'] == service['arguments'] + and pathlib.Path(value['ProgramArguments'][0]).resolve() == pathlib.Path(record['tools']['colima']['path']), + 'original LaunchAgent configuration changed') + actual_home = value.get('EnvironmentVariables', {}).get('COLIMA_HOME', str(pathlib.Path.home() / '.colima')) + require(actual_home == record['profile_home'], 'original default profile changed') + arguments = value['ProgramArguments'] + require(all(isinstance(item, str) and not any(ch.isspace() for ch in item) + for item in value['ProgramArguments']), 'unsupported service argument shape') + expected = [arguments[0], 'start', '-f', '--cpu', '4', '--memory', '6', '--disk', '60', + '--vm-type', 'vz', '--mount-type', 'virtiofs'] + require(arguments == expected, 'unsupported original default-service invocation') + environment = value.get('EnvironmentVariables', {}) + require(set(environment) <= {'PATH', 'HOME', 'COLIMA_HOME'} + and environment.get('HOME', str(pathlib.Path.home())) == str(pathlib.Path.home()), + 'unexpected original service environment') + return raw + + +def validate(sdk, record, source_sha): + require(set(record) == KEYS and record['schema'] == 'durable-workflow.colima-window/v1', 'wrong window schema') + require(record['source_sha256'] == source_sha and re.fullmatch('[0-9a-f]{16}', record['token']), 'wrong window source/token') + require(type(record['uid']) is int and record['uid'] == os.getuid() and record['uid'] > 0 + and record['host_boot_id'] == sdk.boot_identity(), 'wrong window host/UID') + root = root_of(record) + info = root.lstat() + require(root.is_absolute() and root.resolve() == root and root.name == 'sdk-rust-window-' + record['token'] + and stat.S_ISDIR(info.st_mode) and info.st_uid == record['uid'] + and stat.S_IMODE(info.st_mode) == 0o700, 'wrong private window scope') + require(type(record['started_unix']) is int, 'missing absolute window epoch') + owner = record['owner_process'] + require(set(owner) == {'pid', 'pgid', 'uid', 'start_seconds', 'start_microseconds', 'boot_id'} + and all(type(owner[key]) is int for key in ('pid', 'pgid', 'uid', 'start_seconds', 'start_microseconds')) + and owner['pid'] > 1 and owner['pgid'] > 0 and owner['uid'] == record['uid'] + and owner['start_seconds'] > 0 and 0 <= owner['start_microseconds'] < 1_000_000 + and owner['boot_id'] == record['host_boot_id'], 'wrong native owner epoch') + task = record['sdk_intent'] + require(set(task) == sdk.INTENT_KEYS and task['runner_commit'] == SDK_R + and task['manifest_sha256'] == SDK_M and task['guardian_sha256'] == SDK_CODE_SHA + and task['host_boot_id'] == record['host_boot_id'] and task['token'] == record['token'] + and task['started_unix'] == record['started_unix'] + 120 + and task['active_unix'] == task['started_unix'] + 1740 + and task['cleanup_unix'] == task['started_unix'] + 1800, 'wrong frozen task/window cutoffs') + sdk.paths(task['parent'], task['token']) + require(record['docker_host'].startswith('unix:///') and pathlib.Path(record['profile_home']).is_absolute(), 'local original profile required') + home = pathlib.Path(record['profile_home']) + require(home.resolve() == home and record['docker_host'] == 'unix://' + str(home / 'default/docker.sock'), 'wrong default engine endpoint') + require(set(record['tools']) == {'python', 'launchctl', 'plutil', 'colima', 'docker', 'ps', 'dm'}, 'wrong window tool set') + require(record['tools']['launchctl']['path'] == '/bin/launchctl' + and record['tools']['plutil']['path'] == '/usr/bin/plutil' + and record['tools']['ps']['path'] == '/bin/ps', 'native service tools required') + service = record['service'] + require(set(service) == {'label', 'plist', 'arguments', 'keep_alive', 'properties', 'vm_processes'} + and service['label'] != job_label(record) and not service['label'].startswith('sdk-rust-') + and re.fullmatch('[A-Za-z0-9_.-]+', service['label']) + and service['keep_alive'] == {'SuccessfulExit': False}, 'unsupported original service policy') + require(service['vm_processes'] and all(process['boot_id'] == record['host_boot_id'] + and process['uid'] == record['uid'] for process in service['vm_processes']), 'original VM process authority required') + require(record['config_files'] and record['disks'], 'original profile/disk inventory required') + require(len(record['config_files']) <= 8 and len(record['disks']) <= 8, 'oversize original inventory') + require({str(home / 'default/colima.yaml'), str(home / '_lima/colima/lima.yaml')} + <= {binding['path'] for binding in record['config_files']}, 'original profile configurations are missing') + engine = record['engine'] + require(set(engine) == {'info', 'memory_ceiling_bytes', 'images', 'volumes'} + and set(engine['info']) == {'ID', 'ServerVersion', 'Architecture', 'OSType', 'NCPU', 'DockerRootDir'} + and engine['info']['NCPU'] == 4 and engine['info']['OSType'] == 'linux' + and engine['info']['Architecture'] in ('x86_64', 'amd64') + and engine['memory_ceiling_bytes'] == 6 * 1024**3 + and all(isinstance(item, str) for key in ('images', 'volumes') for item in engine[key]) + and all(engine[key] == sorted(set(engine[key])) for key in ('images', 'volumes')), + 'wrong original engine fingerprint') + verify_original(sdk, record) + + +class Window: + def __init__(self, sdk, record, intent_sha, sdk_code): + self.sdk, self.record, self.intent_sha, self.sdk_code = sdk, record, intent_sha, sdk_code + self.root = root_of(record) + now, tick = time.time(), time.monotonic() + self.stop_cutoff = tick + max(0, record['started_unix'] + 120 - now) + self.task_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] - now) + self.restore_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] + 300 - now) + self.cleanup_cutoff = tick + max(0, record['sdk_intent']['cleanup_unix'] + 360 - now) + self.child = None + self.task_record = None + self.task_record_sha = None + self.launched = False + self.stop_attempted = False + self.original_plist = None + self.stage = 'created' + self.native_faults = [] + self.native_fault_count = 0 + self.last_native = None + + def evidence(self, tool, argv, status, output, errors, reason): + operation = argv[2] if tool == 'docker' and len(argv) > 2 else (argv[0] if argv else '') + if tool == 'colima':operation = 'stop' + if tool == 'ps':operation = 'command' + if tool == 'dm':operation = 'notify' + if operation not in {'print', 'bootout', 'bootstrap', 'kickstart', 'remove', '-lint', + 'info', 'ps', 'image', 'volume', 'stop', 'command', 'notify'}:operation = 'other' + return {'stage': self.stage, 'tool': tool, 'operation': operation, 'exit_status': status, + 'reason': reason, 'stdout': diagnostic(output), 'stderr': diagnostic(errors)} + + def remember_fault(self, evidence): + self.native_fault_count += 1 + # Keep the first eight failures and latest eight, including late restore + # or notification faults, within a fixed receipt/storage bound. + if len(self.native_faults) == 16:del self.native_faults[8] + self.native_faults.append(evidence) + + def failure_details(self, error): + return {'failed_stage': self.stage, 'error_type': type(error).__name__, + 'native_fault_count': self.native_fault_count, 'native_faults': list(self.native_faults), + 'last_native_result': self.last_native} + + def env(self, stop=False): + result = {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), + 'TMPDIR': str(self.root), 'DOCKER_CONFIG': str(self.root / 'docker-config')} + if stop: + raw = self.original_plist or verify_original(self.sdk, self.record) + result['PATH'] = plistlib.loads(raw).get('EnvironmentVariables', {}).get('PATH', result['PATH']) + result['COLIMA_HOME'] = self.record['profile_home'] + return result + + def call(self, argv, cutoff, *, pass_fds=(), stop=False, check=True): + remaining = cutoff - time.monotonic() + if remaining <= 0:raise NativeFailure('deadline', None) + child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, pass_fds=pass_fds, env=self.env(stop), close_fds=True) + output, errors = bytearray(), bytearray() + end = time.monotonic() + min(120 if stop else 20, remaining) + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ, output) + selector.register(child.stderr, selectors.EVENT_READ, errors) + while selector.get_map(): + if time.monotonic() >= end: + raise NativeFailure('timeout', None, bytes(output), bytes(errors)) + for key, _ in selector.select(min(0.1, max(0, end - time.monotonic()))): + data = os.read(key.fileobj.fileno(), 65536) + if not data:selector.unregister(key.fileobj);continue + if len(output) + len(errors) + len(data) > 2 * 1024**2: + key.data.extend(data[:2 * 1024**2 - len(output) - len(errors)]) + raise NativeFailure('output-limit', None, bytes(output), bytes(errors)) + key.data.extend(data) + child.wait(timeout=max(0.001, end - time.monotonic())) + if check and child.returncode != 0: + raise NativeFailure('exit-status', child.returncode, bytes(output), bytes(errors)) + return child.returncode, bytes(output), bytes(errors) + except (SystemExit, OSError, subprocess.TimeoutExpired) as error: + if child.poll() is None:child.kill();child.wait(timeout=2) + if isinstance(error, NativeFailure): + if error.status is None:error.status = child.returncode + raise + raise NativeFailure(type(error).__name__, child.returncode, bytes(output), bytes(errors), + getattr(error, 'errno', None)) from None + finally: + if child.poll() is None:child.kill();child.wait(timeout=2) + child.stdout.close();child.stderr.close() + + def native(self, name, *argv, cutoff=None, check=True, **kwargs): + try: + verify_file(self.sdk, self.record['tools'][name]) + result = self.call([self.record['tools'][name]['path'], *argv], + cutoff or self.restore_cutoff, check=False, **kwargs) + except NativeFailure as error: + self.last_native = dict(self.evidence(name, argv, error.status, error.output, error.errors, error.reason), + os_errno=error.os_errno) + self.remember_fault(self.last_native) + raise + except (OSError, SystemExit) as error: + self.last_native = dict(self.evidence(name, argv, None, b'', b'', type(error).__name__), + os_errno=getattr(error, 'errno', None)) + self.remember_fault(self.last_native) + raise + self.last_native = self.evidence(name, argv, *result, 'returned') + if check and result[0] != 0: + self.remember_fault(dict(self.last_native, reason='exit-status')) + raise NativeFailure('exit-status', *result) + return result + + def job(self, label, cutoff=None): + code, output, error = self.native('launchctl', 'print', target(self.record, label), cutoff=cutoff, check=False) + if code: + expected = ('Bad request.\nCould not find service "' + label + + '" in domain for user gui: ' + str(self.record['uid']) + '\n').encode() + if code == 113 and output == b'' and error == expected:return None + self.remember_fault(dict(self.last_native, reason='uncertain-absence')) + raise SystemExit('uncertain launchd query') + if error: + self.remember_fault(dict(self.last_native, reason='unexpected-print-diagnostic')) + raise SystemExit('unexpected launchd diagnostic') + text = output.decode() + def one(name, optional=False): + # Only top-level fields, never a similarly named line inside the + # controller's multiline -c argument or an environment block. + matches = re.findall(r'^\t' + name + r' = (.+)$', text, re.MULTILINE) + if optional and not matches:return None + require(len(matches) == 1, 'unexpected launchd effective schema') + return matches[0].strip() + pid = one('pid', optional=True) + return {'pid': int(pid) if pid else None, 'program': one('program'), 'properties': one('properties')} + + def engine(self): + def docker(*args):return self.native('docker', '--host', self.record['docker_host'], *args)[1].decode().strip() + value = json.loads(docker('info', '--format', '{{json .}}')) + require(not docker('ps', '-aq'), 'original engine is no longer idle') + # Usable Linux MemTotal can vary slightly across a boot; the captured + # original service/config fixes its 6-GiB allocation, not that page count. + require(5 * 1024**3 <= value['MemTotal'] <= 6 * 1024**3, 'original engine memory is outside its profile') + return {'info': {key: value[key] for key in ('ID', 'ServerVersion', 'Architecture', 'OSType', 'NCPU', 'DockerRootDir')}, + 'memory_ceiling_bytes': 6 * 1024**3, + 'images': sorted(set(docker('image', 'ls', '--no-trunc', '--format', '{{.ID}}').splitlines())), + 'volumes': sorted(docker('volume', 'ls', '--format', '{{.Name}}').splitlines())} + + def service_matches(self): + job = self.job(self.record['service']['label']) + require(job and job['pid'] and pathlib.Path(job['program']).resolve() == pathlib.Path(self.record['tools']['colima']['path']) + and job['properties'] == self.record['service']['properties'], 'effective original service changed') + command = self.native('ps', '-ww', '-p', str(job['pid']), '-o', 'command=')[1] + require(shlex.split(command.decode()) == self.record['service']['arguments'], 'effective original service arguments changed') + return job + + def bootstrap_fd(self, raw): + # An unlinked descriptor holds the captured plist; no edited pathname + # can replace the LaunchAgent configuration consumed by launchctl. + with tempfile.TemporaryFile(dir=self.root) as stream: + stream.write(raw);stream.flush();stream.seek(0) + descriptor = '/dev/fd/' + str(stream.fileno()) + self.native('plutil', '-lint', descriptor, pass_fds=(stream.fileno(),)) + self.native('launchctl', 'bootstrap', 'gui/' + str(self.record['uid']), descriptor, pass_fds=(stream.fileno(),)) + + def owner_alive(self): + expected = self.record['owner_process'] + return self.sdk.process_identity(expected['pid']) == expected + + def stop_default(self): + require(self.owner_alive(), 'owner disappeared before stop') + ready = json.loads(self.sdk.regular(self.root / 'window-ready.json')) + require(ready['intent_sha256'] == self.intent_sha + and ready['source_sha256'] == self.record['source_sha256'] + and ready['process'] == self.sdk.process_identity(os.getpid()), 'outside-agent readiness changed') + verify_original(self.sdk, self.record) + self.service_matches() + require(self.engine() == self.record['engine'], 'original engine inventory changed') + require(all(self.sdk.process_identity(process['pid']) == process + for process in self.record['service']['vm_processes']), 'original VM process changed') + require(time.monotonic() < self.stop_cutoff, 'stop window expired before mutation') + self.stop_attempted = True + self.native('launchctl', 'bootout', target(self.record, self.record['service']['label']), cutoff=self.stop_cutoff) + self.native('colima', '--profile', 'default', 'stop', cutoff=self.stop_cutoff, stop=True) + require(self.job(self.record['service']['label'], self.stop_cutoff) is None, 'original service survived stop') + require(all(self.sdk.process_identity(process['pid']) is None + for process in self.record['service']['vm_processes']), 'original VM survived stop') + self.sdk.exclusive(self.root / 'default-stopped.json', {'intent_sha256': self.intent_sha}) + + def start_task(self, profile): + while time.time() < self.record['sdk_intent']['started_unix']: + require(self.owner_alive() and time.monotonic() < self.stop_cutoff, 'owner/start window lost') + time.sleep(0.1) + require(self.owner_alive() and time.time() < self.record['sdk_intent']['started_unix'] + 5, + 'owner/start window lost before task') + sdk_intent = self.record['sdk_intent'] + intent_sha = self.sdk.exclusive(self.root / 'sdk-intent.json', sdk_intent) + self.sdk.exclusive(self.root / 'guest-profile.json', profile) + argv = [self.record['tools']['python']['path'], '-I', '-S', '-c', self.sdk_code, 'prepare', + '--record', str(self.root / 'sdk-intent.json'), '--record-sha256', intent_sha, + '--profile', str(self.root / 'guest-profile.json')] + self.child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, env=self.env()) + + def capture_ready(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + ready_path = scoped['parent'] / 'host-ready.json' + if not ready_path.exists():return False + ready = json.loads(self.sdk.regular(ready_path)) + require(self.child and ready['process']['pid'] == self.child.pid + and self.sdk.process_identity(self.child.pid) == ready['process'] + and ready['source_sha256'] == SDK_CODE_SHA, 'task readiness is not the owned child') + self.task_record_sha = ready['record_sha256'] + self.task_record = self.sdk.authority(scoped['parent'] / 'host-record.json', self.task_record_sha, SDK_CODE_SHA) + require(self.task_record['runner_commit'] == SDK_R and self.task_record['manifest_sha256'] == SDK_M, + 'task readiness tuple changed') + require(self.task_record['parent'] == self.record['sdk_intent']['parent'] + and self.task_record['token'] == self.record['token'], 'task readiness scope changed') + return True + + def task_absent(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + if self.child and self.child.poll() is None:return False + if os.path.lexists(scoped['image']) or os.path.ismount(scoped['mount']) or os.path.lexists(scoped['instance']):return False + for filename, key in (('vm-process.json', None), ('vm-started.json', 'host_process'), ('host-ready.json', 'process')): + path = scoped['parent'] / filename + if path.exists(): + process = json.loads(self.sdk.regular(path)) + if key:process = process[key] + if self.sdk.process_identity(process['pid']) is not None:return False + return True + + def monitor_task(self): + scoped = self.sdk.paths(self.record['sdk_intent']['parent'], self.record['token']) + while self.child.poll() is None and time.monotonic() < self.task_cutoff: + ready = self.task_record is not None or self.capture_ready() + if ready and not self.launched and self.owner_alive(): + self.sdk.exclusive(scoped['parent'] / 'launch.json', {'record_sha256': self.task_record_sha, + 'manifest_sha256': SDK_M, 'action': 'create-and-start'}) + self.launched = True + # No owner is needed for the frozen child's absolute-cutoff cleanup. + # Before VM launch, a complete ready record also permits early abort. + if ready and not self.launched and not self.owner_alive() and time.monotonic() + 30 < self.task_cutoff: + self.child.terminate();self.child.wait(timeout=5) + self.sdk.cleanup(self.task_record) + break + time.sleep(0.25) + require(self.task_absent(), 'task cleanup is uncertain; original service must remain stopped') + + def restore(self, original_plist): + require(self.task_absent(), 'task survivor blocks restoration') + verify_original(self.sdk, self.record) + if self.job(self.record['service']['label']) is None:self.bootstrap_fd(original_plist) + self.native('launchctl', 'kickstart', target(self.record, self.record['service']['label'])) + while time.monotonic() < self.restore_cutoff: + try: + self.service_matches() + require(self.engine() == self.record['engine'], 'restored engine differs from original') + verify_original(self.sdk, self.record) + return {'original_engine_unchanged': True, 'task_absent': True} + except (SystemExit, OSError, ValueError, KeyError): + time.sleep(0.5) + raise SystemExit('original service restoration is uncertain') + + def notify_failure(self): + # The host's existing local dm cache is used only for a real fault. + # No credential or vault value is copied into the task guest or a log. + previous, self.stage = self.stage, 'fault-notify' + try: + self.native('dm', 'sdk-rust: default Colima window has unresolved task/restoration custody; inspect the private window receipt.', cutoff=self.cleanup_cutoff) + return True + except BaseException:return False + finally:self.stage = previous + + def run(self, source_sha, profile): + original = None + restoration_attempted = False + result = {'outcome': 'fail', 'intent_sha256': self.intent_sha, 'source_sha256': source_sha} + try: + self.stage = 'preflight' + original = verify_original(self.sdk, self.record) + self.original_plist = original + self.sdk.exclusive(self.root / 'window-ready.json', {'intent_sha256': self.intent_sha, + 'source_sha256': source_sha, 'process': self.sdk.process_identity(os.getpid())}) + self.stage = 'wait-go' + while not (self.root / 'go.json').exists(): + require(self.owner_alive() and time.monotonic() < self.stop_cutoff, 'owner/go window lost') + time.sleep(0.1) + require(json.loads(self.sdk.regular(self.root / 'go.json')) == {'intent_sha256': self.intent_sha, 'action': 'one-window'}, 'wrong window request') + self.stage = 'stop-default' + self.stop_default() + self.stage = 'start-task' + self.start_task(profile) + self.stage = 'monitor-task' + self.monitor_task() + restoration_attempted = True + self.stage = 'restore-original' + result.update(self.restore(original), outcome='pending_independent_audit') + except BaseException as error: + result.update(self.failure_details(error)) + if self.stop_attempted: + if restoration_attempted: + result['operator_notified'] = self.notify_failure() + else: + try: + self.stage = 'restore-after-fault' + result.update(self.restore(original)) + except BaseException as restoration_error: + result['restoration_failure'] = self.failure_details(restoration_error) + result['operator_notified'] = self.notify_failure() + finally: + try: + result.update(native_fault_count=self.native_fault_count, native_faults=list(self.native_faults), + last_native_result=self.last_native) + result['controller_process'] = self.sdk.process_identity(os.getpid()) + self.sdk.exclusive(self.root / 'window-result.json', result) + finally: + self.remove_controller() + + def remove_controller(self): + # Receipt precedes self-removal. A separate invocation must prove that + # this job/process is gone; this process cannot certify its own absence. + self.stage = 'controller-remove' + try: + self.native('launchctl', 'remove', job_label(self.record), cutoff=self.cleanup_cutoff) + except BaseException as error: + detail = self.failure_details(error) + notified = self.notify_failure() + detail.update(native_fault_count=self.native_fault_count, native_faults=list(self.native_faults), + last_native_result=self.last_native) + self.sdk.exclusive(self.root / 'controller-cleanup-failure.json', { + **detail, 'outcome': 'fail', 'intent_sha256': self.intent_sha, 'operator_notified': notified}) + raise + + +def audit(window, source_sha): + sdk, record = window.sdk, window.record + result = json.loads(sdk.regular(root_of(record) / 'window-result.json')) + require(result['intent_sha256'] == window.intent_sha and result['source_sha256'] == source_sha + and result['outcome'] == 'pending_independent_audit' and result['task_absent'] is True + and result['original_engine_unchanged'] is True, 'window did not complete cleanly') + require(not (root_of(record) / 'controller-cleanup-failure.json').exists(), 'controller cleanup failed') + require(sdk.process_identity(result['controller_process']['pid']) is None + and window.job(job_label(record)) is None and window.task_absent(), 'window/controller survivor') + window.service_matches() + require(window.engine() == record['engine'], 'final original engine drift') + verify_original(sdk, record) + sdk.exclusive(root_of(record) / 'window-audit.json', {'outcome': 'pass', 'intent_sha256': window.intent_sha, + 'source_sha256': source_sha, 'controller_absent': True, 'task_absent': True, 'original_engine_unchanged': True}) + + +def owner(sdk, proposal, proposal_sha, args, code, profile_raw): + require(os.isatty(0), 'foreground owner TTY required') + require(sys.platform == 'darwin', 'native macOS owner required') + require(proposal['owner_process'] is None, 'owner epoch must be filled by the native bootstrap') + record = dict(proposal, owner_process=sdk.process_identity(os.getpid())) + source_sha = sha(code.encode()) + validate(sdk, record, source_sha) + require(pathlib.Path(sys.executable).resolve() == pathlib.Path(record['tools']['python']['path']), + 'owner Python differs from the captured launcher') + require(record['started_unix'] <= time.time() < record['started_unix'] + 30, 'stale owner launch') + root = root_of(record) + intent_sha = sdk.exclusive(root / 'window-intent.json', record) + (root / 'docker-config').mkdir(mode=0o700) + sdk.exclusive(root / 'docker-config/config.json', {'auths': {}}) + window = Window(sdk, record, intent_sha, args.sdk_code) + job = {'Label': job_label(record), 'ProgramArguments': [record['tools']['python']['path'], '-I', '-S', '-c', code, + 'watch', '--record', str(root / 'window-intent.json'), '--record-sha256', intent_sha, + '--intent-json', canonical(record).decode(), '--profile-json', profile_raw.decode(), + '--sdk-code', args.sdk_code, '--profile', args.profile], 'RunAtLoad': True, 'KeepAlive': False, + 'LaunchOnlyOnce': True, 'AbandonProcessGroup': False, 'WorkingDirectory': '/', + 'EnvironmentVariables': window.env(), 'StandardOutPath': '/dev/null', 'StandardErrorPath': '/dev/null'} + try: + window.stage = 'owner-bootstrap' + window.bootstrap_fd(plistlib.dumps(job)) + window.stage = 'owner-readiness' + while time.monotonic() < window.stop_cutoff: + ready_path = root / 'window-ready.json' + if ready_path.exists(): + ready = json.loads(sdk.regular(ready_path)) + effective = window.job(job_label(record)) + require(ready['intent_sha256'] == intent_sha and ready['source_sha256'] == source_sha + and effective and ready['process']['pid'] == effective['pid'] + and sdk.process_identity(effective['pid']) == ready['process'], 'independent controller readiness mismatch') + break + time.sleep(0.1) + else:raise SystemExit('outside-agent controller did not become ready') + except BaseException as error: + sdk.exclusive(root / 'owner-failure.json', {'intent_sha256': intent_sha, + 'source_sha256': source_sha, **window.failure_details(error)}) + raise + sdk.exclusive(root / 'go.json', {'intent_sha256': intent_sha, 'action': 'one-window'}) + print(json.dumps({'window_intent_sha256': intent_sha, 'source_sha256': source_sha, + 'proposal_sha256': proposal_sha, 'owner_process': record['owner_process']}), flush=True) + with selectors.DefaultSelector() as selector: + selector.register(sys.stdin, selectors.EVENT_READ) + while time.monotonic() < window.cleanup_cutoff: + result_path = root / 'window-result.json' + if result_path.exists() and window.job(job_label(record)) is None and \ + sdk.process_identity(json.loads(sdk.regular(result_path))['controller_process']['pid']) is None: + audit(window, source_sha) + print(json.dumps({'operational_window': 'pass', 'sdk_acceptance': 'separate'}), flush=True) + return + for key, _ in selector.select(0.25): + if not os.read(key.fileobj.fileno(), 1024): + raise SystemExit('owner input closed; independent controller retains cleanup custody') + raise SystemExit('owner final-audit cutoff') + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('owner', 'watch', 'audit')) + parser.add_argument('--record', required=True) + parser.add_argument('--record-sha256', required=True) + parser.add_argument('--sdk-code', required=True) + parser.add_argument('--profile', required=True) + parser.add_argument('--intent-json') + parser.add_argument('--profile-json') + args = parser.parse_args(argv) + require('-I' in sys.orig_argv and '-S' in sys.orig_argv and '-c' in sys.orig_argv, 'captured outer source required') + source_sha = sha(sys.orig_argv[sys.orig_argv.index('-c') + 1].encode()) + sdk = load_sdk(args.sdk_code) + if args.mode == 'watch': + require(args.intent_json is not None and args.profile_json is not None, 'captured watch inputs required') + raw = args.intent_json.encode() + profile_raw = args.profile_json.encode() + else: + require(args.intent_json is None and args.profile_json is None, 'unexpected watch inputs') + raw = sdk.regular(pathlib.Path(args.record), 512 * 1024) + profile_raw = sdk.regular(pathlib.Path(args.profile), 64 * 1024) + require(sha(raw) == args.record_sha256, 'window intent digest mismatch') + record = json.loads(raw) + require(sha(profile_raw) == sdk.PROFILE_SHA, 'wrong frozen SDK profile') + if args.mode == 'owner': + owner(sdk, record, args.record_sha256, args, sys.orig_argv[sys.orig_argv.index('-c') + 1], profile_raw) + return + window = Window(sdk, record, args.record_sha256, args.sdk_code) + if args.mode == 'watch': + require(sys.platform == 'darwin' and os.getppid() == 1, 'outside-agent launchd parent required') + require(record['uid'] == os.getuid() and re.fullmatch('[0-9a-f]{16}', record['token']), 'wrong controller identity') + job = window.job(job_label(record)) + require(job and job['pid'] == os.getpid(), 'controller does not own its launchd registration') + try: + validate(sdk, record, source_sha) + require(pathlib.Path(sys.executable).resolve() == pathlib.Path(record['tools']['python']['path']), + 'controller Python differs from captured launcher') + require(record['started_unix'] <= time.time() < record['started_unix'] + 30, 'stale outer launch') + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + except BaseException: + window.remove_controller() + raise + window.run(source_sha, json.loads(profile_raw)) + else: + validate(sdk, record, source_sha) + audit(window, source_sha) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/README.md b/scripts/acceptance/condition-history/README.md new file mode 100644 index 0000000..2fe3df7 --- /dev/null +++ b/scripts/acceptance/condition-history/README.md @@ -0,0 +1,210 @@ +# Published condition-history acceptance + +This packet prepares the selected acceptance for [#191](https://github.com/durable-workflow/sdk-rust/issues/191) +and [#192](https://github.com/durable-workflow/sdk-rust/issues/192). +It is source under review. It has not built its consumer or run a container. +Server history review and Security executable review precede a separate General +admission of the exact runner commit, command, fresh capacity and absolute deadline. + +The package is crates.io `durable-workflow =4.12.0`, archive SHA-256 +`620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4`, +source `4b2440a5618ccc1c98960ea2e1e62127042c3590` (H). Its published tag, +archive and approval records are inputs and remain unchanged. + +`run.py` exports H's official replay consumer and four existing history fixtures +byte for byte, with fixed SHA-256 checks. `fixture/tests/conditions.rs` includes +that consumer and adds the public Worker port of H's four 300/180-second +orphan/tail variants and genuine-timeout refusal. Each case runs with fresh +workers. The typed replay error is observed in the handler before returning the +same error to the Worker; the ordinary published task-failure report is also +retained. The consumer then runs one new live two-member false-wake/reopen case +against unchanged PHP Server 2.11.8, requiring typed `both` completion. + +The original raw proxy artifact for #191 is unavailable. These existing fixtures +are retained evidence, without a claim that they reconstruct that missing full +capture. The live case produces a new capture. #192's original duplicate producer +was the PHP Laravel bridge on native Server 3.0.5. This packet proves installed +Rust replay and does not claim Rust duplicate emission or Server admission behavior. + +All executable image references select immutable `linux/amd64` manifests: + +| Role | Image digest | +| --- | --- | +| Rust 1.86.0 build/consumer | `rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081` | +| PHP Server 2.11.8 | `ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07` | +| MySQL | `mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995` | +| Redis | `redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2` | + +Provenance is limited to those executed child manifests and their effective +`linux/amd64` content readbacks. The Server source label must equal +`fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff`. This packet makes no registry-index +membership claim. + +The closed subject manifest binds Git R, H and every executable member's Git +blob, SHA-256, mode and size. `run.py` loads the subject directly from Git R. +The subject captures the engine, comparator, Rust fixture, inherited helper and +supervisor, new role supervisor and recovery program from allowlisted Git blobs. +It materializes a private read-only snapshot and refuses extra, missing, linked +or nonregular members. Imports, fixture exports and recovery launch use the +captured bytes. Python runs with `-I -S`; no checkout file is imported or reopened +as executable code after preflight. The inherited launch methods are overridden +by the literal role launcher in this packet. + +The build has two sequential phases with two Cargo jobs, 2 CPU, 3 GiB memory, +equal total memory-swap, 1024 PIDs, non-root user, read-only root, all capabilities +dropped and no-new-privileges. Cargo uses a fresh home, lock, public archive and +binary. The public sparse index, lock, archive and VCS commit must agree. Fetch +alone has the default network. Compilation is offline, with source mounted +read-only and only Cargo cache and target output writable. No prior acceptance +binary, lock or registry cache is reused. + +Runtime starts after all build containers are absent. Loopback replay runs alone. +The PHP case has at most five roles: + +| Role | CPU | Memory and equal total swap | PIDs | User | +| --- | ---: | ---: | ---: | --- | +| MySQL | 1 | 1280 MiB | 512 | 0:0, image drops to service user | +| Redis | 0.25 | 128 MiB | 256 | 0:0, image drops to service user | +| HTTP | 0.75 | 768 MiB | 256 | 0:0, image starts its service processes | +| Queue | 0.5 | 512 MiB | 256 | 1000:1000 | +| Worker and driver | 0.5 | 768 MiB | 256 | Non-root owner UID:GID | + +Bootstrap uses the HTTP CPU/memory/PID profile and is removed before HTTP and +queue startup. Total runtime caps are 3 CPU/3.375 GiB. `resources.PROFILES` +literally defines capability additions for each service; all roles drop ALL +before those additions. Worker has a read-only root and read-only fixture/binary +mounts, with only task state writable. Database/cache data and PHP cached +configuration use bounded tmpfs. Image volume declarations must be fully covered. + +Each role is created and started while its inline captured supervisor waits for +a private ready marker. Before releasing that gate, the owner independently reads +back image/content/platform, resource and swap/PID caps, user, root mode, +capabilities, init/security settings, exact command/deadline, environment, +network/internal ownership and mount destinations/types/access. Tokens and the +Server key arrive through private stdin into role tmpfs only after readback. +No host credential, Docker socket, published port or host network is admitted. +The owner uses a fixed anonymous Docker configuration and an explicit local Unix +daemon. Before work creation or registry requests, read-only preflight refuses +any existing owner/run container, network or volume and every colliding exact name. +An uncertain query is a refusal. + +One 30-minute absolute owner window includes setup, fetch, compilation, execution +and cleanup. Active work ends 60 seconds before the final ceiling. The Cell's +roles share one maximum 600-second deadline plus a 10-second TERM/KILL grace. +`run.json` is an exclusive owner-only regular single-link record binding R, H, +manifest and recovery digests, token, local daemon, root, UID/GID, isolated owner +PID/group/session, Linux boot ID and kernel start ticks, and all three deadlines. +Execution requires Linux proc authority. Identity observations must agree across +two kernel-stat reads and boot reads; recovery revalidates that tuple immediately +before each TERM or KILL. A missing or changed authority refuses the signal. +Its digest is captured outside +the work root in the private launch handoff. + +Before any pull, the owner starts a detached recovery process from captured +inline bytes and requires its exact readiness receipt. At the active cutoff it +uses its captured complete authority to stop the exact owner and remove only +matching immutable container/network IDs. A later source/record drift fails the +result but cannot replace the already-captured cleanup authority. Foreign process +epochs, labels or names are refused. Interrupted Cell evidence is purged because +credential values are deliberately absent from the recovery record. Cleanup is +bounded by the final ceiling. No edit, retry, fallback or extension is included. + +After source review and a separate exact execution admission, use the reviewed +inline collector below. `R`, `M`, `CHECKOUT`, fresh `W`, token `T`, local daemon +`D`, and captured `S`/`ACTIVE`/`CLEANUP` belong in the private proposal. `ACTIVE` +is `S + 1740` and `CLEANUP` is `S + 1800`. Do not substitute a mutable script path +for the captured entry point. + +```sh +python3 -I -S -c ' +import pathlib, re, subprocess, sys +checkout, revision = sys.argv[1:3] +if not re.fullmatch("[0-9a-f]{40}", revision): + raise SystemExit("invalid reviewed runner") +entry = subprocess.run(["git", "--no-replace-objects", "-C", checkout, "show", + revision + ":scripts/acceptance/condition-history/run.py"], + check=True, capture_output=True, timeout=20).stdout.decode() +raise SystemExit(subprocess.run([sys.executable, "-I", "-S", "-c", entry, + "run", "--checkout", checkout, "--runner-commit", revision, *sys.argv[3:]], + start_new_session=True).returncode) +' CHECKOUT R --manifest-sha256 M \ + --package-commit 4b2440a5618ccc1c98960ea2e1e62127042c3590 \ + --work-dir W --run-token T --docker-endpoint D \ + --guest-boot-id B \ + --started-unix S --active-unix ACTIVE --cleanup-unix CLEANUP +``` + +The collector preserves the exact Git blob's trailing newline and starts a new +owner session. A `manifest` invocation of the same captured entry reads Git only +and prints the deterministic manifest and its digest. It does not contact Docker +or registries. The run refuses wrong H, dirty/wrong R, wrong M, existing work, +stale deadlines or a changed entry before importing the engine or creating roles. +Host aliases, live capacity samples and private paths stay out of public records. + +For immediate recovery or the mandatory independent final audit, capture +`recover.py` directly from the reviewed Git R and invoke those bytes with +`python3 -I -S -c`. Supply `recover` or `audit` and all of `--work-dir`, +`--record-sha256`, `--manifest-sha256`, `--runner-commit`, `--package-commit`, +`--run-token`. Audit also requires the externally captured `--summary-sha256`. +Standalone recovery does not depend on a mutable checkout entry point. + +The owner writes `summary.json` with `O_EXCL|O_NOFOLLOW`, mode 0600, and returns +its immutable SHA-256. It records package/source/lock/binary inputs, exact role +readbacks and fresh typed replay/live results. The fetch transport is the public +SDK raw control-plane method, not an independent second HTTP client. Owner checks +remain `pending_independent_audit`. Only after owner exit can a separately invoked +captured audit bind that summary digest, verify the original record/snapshot and +recovery-process receipt, and confirm every matching container, network, volume, +owner and recovery process group absent. It writes its own exclusive handoff. +The guest audit returns `pending_external_guest_teardown`, not acceptance. +Sanitize and retain both immutable results before the external guardian removes +the guest and its enclosing store. Independent host absence/custody evidence +remains required before any acceptance or closure. + +The task network has an explicit local bridge/IPv4-only IPAM profile and exact +masquerade, inter-container and loopback-binding options. Driver, scope, ingress, +attachable/configuration flags, labels, IPAM and endpoints are read back before +releasing each role. This does not claim isolation from the daemon host itself. + +The current shared-daemon execution proposal is held by Security's storage +finding on #210. A subsequent reviewed proposal must select a disposable +task-owned daemon/work store with a hard byte ceiling and externally owned +teardown. Capacity samples do not establish that ceiling. Neither the process +nor network source correction admits a pull, build or runtime from this packet. + +The guest entry now additionally binds the externally observed boot ID B, a +block-backed ext4 root of at most 20 GiB, and the same device for its fresh work +parent and Docker data root. It refuses host/network filesystem shares, a +nonempty daemon or build-cache baseline, and a later device/geometry drift. +The run record binds that storage identity. Image/cache inventories remain in +external-guardian custody until whole-guest/store teardown; the guest never +guesses which layers to delete from a shared daemon. The concrete +[guest/guardian source proposal](guest-proposal.md) selects cached native Lima/VZ +inputs and a fixed enclosing host disk. The captured external guardian also owns +bounded setup before allocation. Exact source review, private handoff and fresh +capacity remain prerequisites; no VM execution is admitted. After an exact +private request, that guardian owns create from captured stdin, rechecks copied +image identity, records effective configuration, starts the guest and verifies +the running guest-agent executable before any consumer. These are source +assertions under review, not observed guest results. + +The comparator scans every opened wait, condition outcome and timer row, binding +the authored group, member key/fingerprint/occurrence, wait ID and integer timeout. +Additional reopens must keep that member lineage and cannot extend its recorded +timeout. Present timer terminals require a matching earlier schedule; a superseded +orphan's timer is distinct from the current tail. Current-tail timeouts contradict +this case's required satisfied result. H's minimal two-member fixture has no timer +rows, and its direct condition outcomes do not imply an invented timer schedule. +The live consumer also checks both public parallel results are `Satisfied` before +returning `both`. These are source assertions for review, not observed live results. + +Inert source checks, which do not invoke Docker or Cargo: + +```sh +python3 -m unittest discover -s scripts/acceptance/condition-history -p 'test_*.py' +rustfmt --edition 2021 --check scripts/acceptance/condition-history/fixture/tests/conditions.rs +``` + +These check source guards, fixed fixture export and the history comparator. They +do not prove that the new Rust consumer compiles or that either live acceptance +criterion passes. Those remain subject to the reviewed bounded execution. diff --git a/scripts/acceptance/condition-history/check_history.py b/scripts/acceptance/condition-history/check_history.py new file mode 100644 index 0000000..28b6ef9 --- /dev/null +++ b/scripts/acceptance/condition-history/check_history.py @@ -0,0 +1,134 @@ +"""Offline comparator for the newly captured two-member PHP history. + +The declared identities follow unchanged H's two-member fixture. That minimal +fixture contains no timer rows, so a timer schedule is not mandatory. Every +timer row actually present must bind to an opened wait, and every timer terminal +must bind to its preceding schedule. Direct condition outcomes need no invented +timer row (H also supports that history shape). +""" + +MEMBERS = [('votes', 'sha256:votes-v1'), ('approvals', 'sha256:approvals-v1')] +GROUP = {'parallel_group_id': 'parallel-calls:1:2', 'parallel_group_kind': 'condition', + 'parallel_group_base_sequence': 1, 'parallel_group_size': 2} + + +def require(condition, label): + if not condition: + raise ValueError(label) + + +def member(payload): + require(isinstance(payload, dict), 'condition-payload') + require(all(type(payload.get(field)) is int for field in + ('sequence', 'parallel_group_index', 'parallel_group_base_sequence', 'parallel_group_size')), + 'typed-group-numbers') + index = payload['parallel_group_index'] + require(index in (0, 1) and all(payload.get(k) == v for k, v in GROUP.items()), 'group-identity') + key, fingerprint = MEMBERS[index] + require(payload.get('condition_key') == key + and payload.get('condition_definition_fingerprint') == fingerprint + and payload.get('condition_wait_occurrence_id') == 'rust:condition-wait:' + str(index), + 'authored-member-identity') + return index + + +def opened_wait(payload, waits): + sequence = payload.get('sequence') + require(type(sequence) is int and sequence in waits, 'outcome-before-open') + wait = waits[sequence] + require(payload.get('condition_wait_id') == wait['payload']['condition_wait_id'], 'wait-identity') + return wait + + +def open_wait(payload, waits, tails): + index = member(payload) + sequence, timeout = payload['sequence'], payload.get('timeout_seconds') + require(sequence == len(waits) + 1 and payload.get('condition_wait_id') == 'condition:' + str(sequence), + 'open-sequence-identity') + require(type(timeout) is int and 0 <= timeout <= 300, 'typed-timeout-bound') + if sequence <= 4: + require(index == (sequence - 1) % 2, 'initial-reopen-members') + previous = tails.get(index) + if previous is None: + require(sequence in (1, 2) and timeout == 300, 'original-timeout') + else: + require(timeout <= waits[previous]['payload']['timeout_seconds'], 'extended-timeout') + waits[sequence] = {'payload': payload, 'member': index, 'terminal': None} + tails[index] = sequence + + +def condition_outcome(kind, payload, waits, tails): + index = member(payload) + wait = opened_wait(payload, waits) + require(wait['member'] == index and wait['terminal'] is None, 'condition-terminal-identity') + if 'timeout_seconds' in payload: + require(type(payload['timeout_seconds']) is int + and payload['timeout_seconds'] == wait['payload']['timeout_seconds'], 'outcome-timeout') + if kind == 'ConditionWaitTimedOut': + require(tails[index] != payload['sequence'], 'current-tail-timeout') + wait['terminal'] = kind + + +def timer_row(kind, payload, waits, tails, timers): + require(isinstance(payload, dict), 'timer-payload') + wait = opened_wait(payload, waits) + timer_id = payload.get('timer_id') + require(isinstance(timer_id, str) and timer_id and payload.get('timer_kind') == 'condition_timeout', 'timer-identity') + require(type(payload.get('delay_seconds')) is int + and payload['delay_seconds'] == wait['payload']['timeout_seconds'], 'timer-delay') + identity = (payload['sequence'], payload['condition_wait_id'], payload['delay_seconds']) + if kind == 'TimerScheduled': + require(timer_id not in timers + and not any(t['identity'][0] == identity[0] for t in timers.values()), 'duplicate-timer-schedule') + timers[timer_id] = {'identity': identity, 'terminal': None} + else: + require(timer_id in timers and timers[timer_id]['identity'] == identity + and timers[timer_id]['terminal'] is None, 'unresolved-timer-terminal') + if kind == 'TimerFired': + require(tails[wait['member']] != payload['sequence'], 'current-tail-timer-fired') + timers[timer_id]['terminal'] = kind + + +def scan(events): + waits, tails, timers = {}, {}, {} + completed = False + for event in events: + require(isinstance(event, dict), 'history-row') + kind, payload = event.get('event_type'), event.get('payload') + if kind == 'WorkflowCompleted': + require(not completed, 'duplicate-completion') + completed = True + elif kind == 'ConditionWaitOpened': + require(not completed, 'open-after-completion') + open_wait(payload, waits, tails) + elif kind in ('ConditionWaitSatisfied', 'ConditionWaitTimedOut'): + require(not completed, 'condition-outcome-after-completion') + condition_outcome(kind, payload, waits, tails) + elif kind in ('TimerScheduled', 'TimerFired', 'TimerCancelled'): + require(not completed or kind == 'TimerCancelled', 'timer-after-completion') + timer_row(kind, payload, waits, tails, timers) + require(completed and len(waits) >= 4, 'completed-group-history') + return waits + + +def check(report): + require(report.get('outcome') == 'pass' and report.get('result') == 'both', 'typed-result') + snapshots = [report.get(key) for key in ('initial', 'reopened', 'history')] + require(all(isinstance(events, list) for events in snapshots), 'history-arrays') + initial, reopened, final = snapshots + require(reopened[:len(initial)] == initial and final[:len(reopened)] == reopened, 'append-only-history') + waits = scan(final) + require(sum(e.get('event_type') == 'ConditionWaitOpened' for e in initial) == 2, 'initial-two-members') + types = [e.get('event_type') for e in final] + require(not any(t in ('WorkflowFailed', 'ReplayFailure', 'WorkflowTimedOut') for t in types), 'no-replay-failure') + signal_rows = [(i, e.get('payload', {}).get('signal_name')) for i, e in enumerate(final) + if e.get('event_type') == 'SignalReceived'] + require([name for _, name in signal_rows] == ['vote', 'vote', 'approve'], 'signal-order') + open_rows = [i for i, e in enumerate(final) if e.get('event_type') == 'ConditionWaitOpened'] + require(open_rows[1] < signal_rows[0][0] < open_rows[2] < open_rows[3] < signal_rows[1][0], 'false-wake-row-order') + early_satisfied = [e.get('payload', {}).get('sequence') for e in reopened + if e.get('event_type') == 'ConditionWaitSatisfied'] + require(2 in early_satisfied and 1 not in early_satisfied, 'unresolved-first-member') + require(types.index('WorkflowCompleted') > signal_rows[-1][0], 'completion-after-approval') + return {'outcome': 'pass', 'first_open_sequences': [1, 2], 'reopened_sequences': [3, 4], + 'all_open_sequences': list(waits), 'captured_events': len(final), 'result': report['result']} diff --git a/scripts/acceptance/condition-history/engine.py b/scripts/acceptance/condition-history/engine.py new file mode 100644 index 0000000..3eea9b8 --- /dev/null +++ b/scripts/acceptance/condition-history/engine.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""One explicitly admitted public-package condition-history acceptance. + +Source preparation and stubbed tests do not authorize invoking this runner. +See README.md for its fixed tuple, command and remaining review gate. +""" +import json +import os +import pathlib +import shutil +import subprocess +import sys +import time +import tomllib +import types +import urllib.request +import sealed_subject as subject + +HERE = subject.CURRENT.root / 'scripts/acceptance/condition-history' +REPO = pathlib.Path(subject.ARGS.checkout) +native = subject.CURRENT.module('native') +comparator = subject.CURRENT.module('comparator') +resources = subject.CURRENT.module('resources') +recovery = subject.CURRENT.module('recovery') + +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +CRATE_SHA = '620a8287af73c363dac61f0a91f20971a39f3b8cdc9513fbea85cdbd95a74ba4' +RUST = 'rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081' +SERVER = 'ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07' +SERVER_SOURCE = 'fa3fc2f2f6b8935ea4a6eea47f6a9221e2adafff' +MYSQL = 'mysql@sha256:80f4933e3835f9dc4d35a28ec500d7986cb4414e6c6821c5461239cb7beb8995' +REDIS = 'redis@sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2' +INPUTS = { + 'tests/replay_regression_corpus.rs': 'b54be69c5b82ec6ad32bd2871c3451f182185fa01ca3316a1c3aee935c48c9cc', + 'tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json': + '815526c761ce314960335b0f7cafaa0aa60fa519ddd08252451a7a5ebdca821d', + 'tests/fixtures/replay-regressions/timer-separated-condition-group-server-reopen.json': + 'f86ecb0cc57634a3eb4c87ffbd4fa55555b82c2794a16cd7b4ce9d33e009d0f6', + 'tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json': + '50a630596b5d85a392a33cc595fe58a513f59f6ebdbba928fa22be4ea0fb9075', + 'tests/fixtures/replay-regressions/condition-orphaned-duplicate-timeout-keeps-wait.json': + '012790d1fc23fe11ef9208d3a8243dc282d160903f7df98396c77e1f8fd898f7', +} +CELL = 'condition-history' +PREFIX = 'sdk-rust-condition' +LIMITS = {'mysql': ('1', '1280m'), 'redis': ('0.25', '128m'), + 'bootstrap': ('0.75', '768m'), 'runtime': ('0.75', '768m'), + 'queue': ('0.5', '512m'), 'consumer': ('0.5', '768m')} + + +def require(condition, label): + if not condition: + raise SystemExit(label) + + +def command(args, argv, **kwargs): + subject.CURRENT.verify() + return resources.call(args, argv, **kwargs) + + +def export_consumer(fixture): + fixture.mkdir(mode=0o700) + subject.CURRENT.export(subject.PREFIX + 'fixture/', fixture) + for source, expected in INPUTS.items(): + raw = command(subject.ARGS, ['git', '--no-replace-objects', '-C', str(REPO), + 'show', H + ':' + source], text=False).stdout + require(subject.digest(raw) == expected, 'fixed history input differs') + target = fixture / ('tests/official.rs' if source == 'tests/replay_regression_corpus.rs' else source) + target.parent.mkdir(parents=True, exist_ok=True) + subject.exclusive(target, raw) + + +def build(args, fixture, controller, script, phase, network=None): + user = '%d:%d' % (os.getuid(), os.getgid()) + spec = controller.profile(phase, RUST, ['sh', '-ec', script], user, 'none', fixture, + env={'CARGO_HOME': '/work/.cargo', 'CARGO_BUILD_JOBS': '2', + 'CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS': 'fallback'}, + deadline=args.active_unix - 10) + name = controller.start(spec) + try: + code = command(args, ['docker', 'wait', name], timeout=max(1, args.active_unix - time.time())).stdout.strip() + logs = command(args, ['docker', 'logs', '--tail', '2000', name], check=False) + require(code == '0', 'bounded build phase failed') + return logs + finally: + object_id = recovery.owned(controller.record, 'container', name) + if object_id: + recovery.docker(controller.record, ['rm', '-fv', object_id]) + + +def verify_crate(fixture): + lock = tomllib.loads((fixture / 'Cargo.lock').read_text()) + packages = [p for p in lock['package'] if p['name'] == 'durable-workflow'] + require(len(packages) == 1 and packages[0]['version'] == '4.12.0' + and packages[0]['source'] == native.CRATES_IO_SOURCE + and packages[0]['checksum'] == CRATE_SHA, 'wrong public package lock') + archives = list((fixture / '.cargo/registry/cache').glob('*/durable-workflow-4.12.0.crate')) + require(len(archives) == 1 and native.sha256(archives[0]) == CRATE_SHA, 'wrong downloaded archive') + with native.tarfile.open(archives[0]) as archive: + vcs = json.load(archive.extractfile('durable-workflow-4.12.0/.cargo_vcs_info.json')) + require(vcs['git']['sha1'] == H and not vcs['git'].get('dirty'), 'wrong public package VCS') + require(not (fixture / '.cargo/config.toml').exists(), 'registry replacement is forbidden') + return {'version': '4.12.0', 'source': H, 'archive_sha256': CRATE_SHA, + 'lock_sha256': native.sha256(fixture / 'Cargo.lock')} + + +def registry_identity(): + request = urllib.request.Request('https://index.crates.io/du/ra/durable-workflow', + headers={'User-Agent': 'durable-workflow-condition-history-acceptance'}) + with urllib.request.urlopen(request, timeout=30) as response: + versions = [json.loads(line) for line in response.read(8 * 1024 * 1024).splitlines()] + exact = [version for version in versions if version.get('vers') == '4.12.0'] + require(len(exact) == 1 and exact[0].get('cksum') == CRATE_SHA + and exact[0].get('yanked') is False, 'public registry identity refused') + return {'version': '4.12.0', 'sha256': CRATE_SHA, 'yanked': False} + + +class ConditionCell(native.Cell): + def __init__(self, *args): + super().__init__(*args) + self.receipts = [] + + def launch(self, line, name, image, child, user, credentials): + role = name.removeprefix(self.task + '-') + require(role in LIMITS and len(self.containers) < 5, 'unknown role or runtime ceiling') + env = dict(line[i + 1].split('=', 1) for i, value in enumerate(line) if value == '-e') + alias = line[line.index('--network-alias') + 1] if '--network-alias' in line else None + spec = self.controller.profile(role, image, child, user, credentials, self.fixture, self.state, + env=env, alias=alias, deadline=self.deadline) + self.containers.append(name) + return self.controller.start(spec, + self.token if credentials in ('auth', 'both') else '', + self.server_key if credentials in ('server', 'both') else '') + + def remove_docker_objects(self): + errors_before = len(self.harness_run.cleanup_errors) + self.harness_run.best_effort('logs-dir', CELL, lambda: self.cell_path('logs').mkdir(exist_ok=True)) + for name in self.containers: + try: + object_id = recovery.owned(self.controller.record, 'container', name) + if object_id: + self.harness_run.best_effort('container-logs', name, self.save_logs, self.dir / 'logs', name, object_id) + except BaseException: + self.harness_run.cleanup_errors.append({'step': 'ownership', 'object': name, 'error': 'Refused'}) + result = recovery.cleanup(self.controller.record) + if not result['pass']: + self.harness_run.cleanup_errors.append({'step': 'cleanup', 'object': CELL, 'error': 'Uncertain'}) + return result['pass'] and not any(e['step'] == 'ownership' for e in self.harness_run.cleanup_errors[errors_before:]) + + def start_php(self, token): + env = {'APP_ENV': 'testing', 'APP_DEBUG': 'false', 'DB_CONNECTION': 'mysql', + 'DB_HOST': 'mysql', 'DB_DATABASE': 'durable_workflow', 'DB_USERNAME': 'workflow', + 'DB_PASSWORD': 'workflow', 'REDIS_HOST': 'redis', 'QUEUE_CONNECTION': 'redis', + 'CACHE_STORE': 'redis', 'DW_AUTH_DRIVER': 'token', 'DW_WORKER_PROTOCOL_VERSION': '1.20'} + mysql = self.plain('mysql', MYSQL, [], alias='mysql', tmpfs='/var/lib/mysql', pids=512, + caps=('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'), + env={'MYSQL_DATABASE': 'durable_workflow', 'MYSQL_USER': 'workflow', + 'MYSQL_PASSWORD': 'workflow', 'MYSQL_ROOT_PASSWORD': 'root'}) + self.plain('redis', REDIS, [], alias='redis', tmpfs='/data', caps=('CHOWN', 'SETGID', 'SETUID')) + until = min(time.time() + 120, self.deadline - 15) + while native.call(['docker', 'exec', mysql, 'mysqladmin', 'ping', '-h', 'localhost', '-uworkflow', '-pworkflow'], + check=False).returncode: + require(time.time() < until, 'mysql readiness deadline') + time.sleep(1) + bootstrap = self.plain('bootstrap', SERVER, ['server-bootstrap'], env=env, + caps=('DAC_OVERRIDE',), credentials='server') + require(native.call(['docker', 'wait', bootstrap], timeout=max(1, self.deadline - time.time())).stdout.strip() == '0', + 'PHP bootstrap failed') + object_id = recovery.owned(self.controller.record, 'container', bootstrap) + require(object_id, 'bootstrap ownership changed') + recovery.docker(self.controller.record, ['rm', '-fv', object_id]) + self.containers.remove(bootstrap) + self.plain('runtime', SERVER, [], env=env, alias='qualification-origin', credentials='both', + caps=('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')) + self.plain('queue', SERVER, ['php', 'artisan', 'queue:work', '--sleep=1', '--tries=3', '--max-time=600'], + env=env, user='1000:1000', credentials='server') + + def execute(self, args): + outcome = {'outcome': 'fail', 'deadline_unix': self.deadline} + try: + self.cell_path('state').mkdir(parents=True) + self.dir.chmod(0o777) + self.state.chmod(0o777) + command(args, self.controller.network_arguments()) + # No service roles exist during loopback replay; then exactly four + # service roles plus one combined driver/Worker. No proxy/repair. + replay = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_history_replay', '--nocapture']) + require(command(args, ['docker', 'wait', replay], timeout=300).stdout.strip() == '0', 'installed replay failed') + object_id = recovery.owned(self.controller.record, 'container', replay) + require(object_id, 'replay ownership changed') + recovery.docker(self.controller.record, ['rm', '-fv', object_id]) + self.containers.remove(replay) + self.start_php(self.token) + consumer = self.start('consumer', RUST, ['/fixture/condition-history', '--exact', + 'installed_conditions::installed_live_group', '--ignored', '--nocapture'], credentials='auth') + require(command(args, ['docker', 'wait', consumer], timeout=180).stdout.strip() == '0', 'live group failed') + finally: + stopped = self.remove_docker_objects() + try: + require(stopped, 'writer cleanup uncertain') + self.fence_artifacts() + require(not self.harness_run.cleanup_errors, 'evidence fence failed') + replay = json.loads(self.artifact_bytes(self.state / 'replay.json')) + live = json.loads(self.artifact_bytes(self.state / 'live.json')) + require(replay.get('outcome') == 'pass' and len(replay.get('cases', {})) == 9, 'replay report incomplete') + outcome.update(outcome='pass', replay=replay, live=live, history_check=comparator.check(live), + role_receipts=self.controller.receipts) + except BaseException: + outcome['outcome'] = 'fail' + self.harness_run.best_effort('evidence-purge', CELL, self.purge_evidence) + raise + return outcome + + +def start_recovery(args, record): + program = subject.CURRENT.read(subject.PREFIX + 'recover.py').decode() + require(subject.digest(program.encode()) == record['recovery_sha256'], 'captured recovery source mismatch') + recovery_argv = [sys.executable, '-I', '-S', '-c', program, 'recover', '--work-dir', args.work_dir, + '--record-sha256', subject.RECORD_SHA, '--manifest-sha256', subject.CURRENT.sha256, + '--runner-commit', args.runner_commit, '--package-commit', H, '--run-token', args.run_token, + '--at', str(args.active_unix)] + watchdog = subprocess.Popen(recovery_argv, start_new_session=True, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=resources.environment(args)) + try: + process = recovery.process_identity(watchdog.pid) + require(process is not None, 'recovery did not start') + recovery_process_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'recovery-process.json', subject.canonical({ + 'source_sha256': record['recovery_sha256'], 'process': process})) + ready_path = pathlib.Path(args.work_dir) / 'recovery-ready.json' + until = min(time.time() + 5, args.active_unix - 10) + while not ready_path.exists(): + require(watchdog.poll() is None and time.time() < until, 'captured recovery not ready') + time.sleep(0.05) + ready = json.loads(recovery.read(ready_path)) + require(ready == {'record_sha256': subject.RECORD_SHA, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_sha256': record['recovery_sha256'], 'process': process}, 'wrong recovery-ready authority') + except BaseException: + if watchdog.poll() is None: + watchdog.terminate() + try: + watchdog.wait(timeout=5) + except subprocess.TimeoutExpired: + watchdog.kill() + watchdog.wait(timeout=5) + raise + return watchdog, recovery_process_sha + + +def execute(args): + # The captured bootstrap has already verified R/H/manifest and refused all + # collisions before creating this root or contacting a registry. + record = recovery.validate_record(args.work_dir, subject.RECORD_SHA, args.runner_commit, + subject.CURRENT.sha256, subject.RECORD['recovery_sha256'], args.run_token) + owner = native.Run(PREFIX, token=args.run_token) + owner.base = PREFIX + '-' + args.run_token + controller = resources.Docker(args, subject.CURRENT, record) + native.call = lambda line, **kwargs: command(args, line, **kwargs) + watchdog, recovery_process_sha = start_recovery(args, record) + result = {'outcome': 'fail', 'checks_outcome': 'fail', 'record_sha256': subject.RECORD_SHA, + 'runner_commit': args.runner_commit, 'package_commit': H, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_process_sha256': recovery_process_sha, 'storage': record['storage']} + try: + registry = registry_identity() + images = [RUST, SERVER, MYSQL, REDIS] + for image in images: + command(args, ['docker', 'pull', '--platform', 'linux/amd64', image], timeout=600) + identities = {image: {'kind': 'executed-child-manifest', 'reference': image, + 'content_sha256': controller.image(image)['sha256'], 'os': 'linux', 'architecture': 'amd64'} for image in images} + labels = controller.image(SERVER)['raw']['Config'].get('Labels') or {} + require(labels.get('org.opencontainers.image.revision') == SERVER_SOURCE, 'wrong PHP Server source') + fixture = pathlib.Path(args.work_dir) / 'fixture' + export_consumer(fixture) + sources = {p.relative_to(fixture).as_posix(): native.sha256(p) for p in fixture.rglob('*') if p.is_file()} + fetch = build(args, fixture, controller, 'cargo generate-lockfile && cargo fetch --locked', 'fetch') + subject.exclusive(fixture / 'fetch.stdout', fetch.stdout.encode()) + require(all(native.sha256(fixture / path) == sha for path, sha in sources.items()), 'fetch changed consumer source') + package = verify_crate(fixture) + (fixture / 'target').mkdir(exist_ok=True) + compiled = build(args, fixture, controller, + 'rustc -V && cargo -V && cargo test --locked --offline --test conditions --no-run --message-format=json', 'build') + subject.exclusive(fixture / 'build.stdout', compiled.stdout.encode()) + subject.exclusive(fixture / 'build.stderr', compiled.stderr.encode()) + toolchain = compiled.stdout.splitlines()[:2] + require(len(toolchain) == 2 and toolchain[0].startswith('rustc 1.86.0 ') + and toolchain[1].startswith('cargo 1.86.0 '), 'wrong actual Rust/Cargo toolchain') + artifacts = [json.loads(line) for line in compiled.stdout.splitlines() if line.startswith('{')] + executables = [p['executable'] for p in artifacts if p.get('reason') == 'compiler-artifact' + and p.get('target', {}).get('name') == 'conditions' and p.get('executable')] + require(len(executables) == 1 and executables[0].startswith('/work/target/debug/deps/conditions-'), 'wrong consumer executable') + binary = native.confined_child(fixture, executables[0].removeprefix('/work/')) + output = fixture / 'target/debug/condition-history' + # Exclusive binary handoff, never an overwrite of a pre-existing name. + subject.exclusive(output, subject.regular(binary, limit=256 * 1024 * 1024), 0o500) + binary_sha = native.sha256(output) + require(not recovery.objects(record, 'container'), 'build/runtime overlap') + cell = ConditionCell(types.SimpleNamespace(rust_image=RUST), CELL, pathlib.Path(args.work_dir), + {'executed_manifest': SERVER}, fixture, owner) + cell.controller = controller + cell.deadline = min(int(time.time()) + 600, args.active_unix - 10) + outcome = cell.execute(args) + require(native.sha256(output) == binary_sha and verify_crate(fixture) == package, 'consumer/package changed') + require(all(native.sha256(fixture / path) == sha for path, sha in sources.items()), 'consumer source changed') + result.update(outcome='pending_independent_audit', checks_outcome='pass', package=package, + binary_sha256=binary_sha, input_sha256=INPUTS, consumer_source_sha256=sources, + toolchain=toolchain, registry=registry, images=identities, cell=outcome) + except BaseException as error: + result['error_type'] = type(error).__name__ + finally: + cleanup = recovery.cleanup(record) + result['cleanup'] = cleanup + result['cleanup_errors'] = owner.cleanup_errors + try: + result['storage_custody'] = {'disposition': 'pending_external_guest_teardown', + 'image_ids': recovery.docker(record, ['image', 'ls', '-q', '--no-trunc']).stdout.split(), + 'image_cache_inventory': [json.loads(line) for line in recovery.docker(record, + ['system', 'df', '--format', '{{json .}}']).stdout.splitlines() if line]} + except BaseException: + result.update(outcome='fail', checks_outcome='fail', storage_custody={'disposition': 'unverified'}) + if not cleanup['pass'] or owner.cleanup_errors: + result.update(outcome='fail', checks_outcome='fail') + subject.CURRENT.verify() + data = subject.canonical(result) + clean, leaked = native.redact_credentials(data, owner.secret_values) + if leaked: + result = {'outcome': 'fail', 'checks_outcome': 'fail', 'error_type': 'CredentialOutput', + 'record_sha256': subject.RECORD_SHA} + clean = subject.canonical(result) + recovery.purge(args.work_dir) + summary_sha = subject.exclusive(pathlib.Path(args.work_dir) / 'summary.json', clean) + if cleanup['pass']: + watchdog.terminate() + watchdog.wait(timeout=5) + print(json.dumps({'summary_sha256': summary_sha, 'record_sha256': subject.RECORD_SHA, + 'manifest_sha256': subject.CURRENT.sha256, 'checks_outcome': result['checks_outcome'], + 'acceptance': 'requires independent captured-source audit after owner exit'}), flush=True) + require(result['checks_outcome'] == 'pass', 'checks failed; no acceptance') diff --git a/scripts/acceptance/condition-history/fixture/Cargo.toml b/scripts/acceptance/condition-history/fixture/Cargo.toml new file mode 100644 index 0000000..ae960e9 --- /dev/null +++ b/scripts/acceptance/condition-history/fixture/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "installed-condition-history" +version = "0.0.0" +edition = "2021" +publish = false +rust-version = "1.86" + +[workspace] + +[dependencies] +durable-workflow = "=4.12.0" +chrono = { version = "0.4", default-features = false, features = ["alloc"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } + +[profile.dev] +debug = 0 +incremental = false diff --git a/scripts/acceptance/condition-history/fixture/tests/conditions.rs b/scripts/acceptance/condition-history/fixture/tests/conditions.rs new file mode 100644 index 0000000..c64bafa --- /dev/null +++ b/scripts/acceptance/condition-history/fixture/tests/conditions.rs @@ -0,0 +1,407 @@ +// run.py exports this file byte-for-byte from the immutable package source H. +// It supplies the official Worker/loopback transport, not SDK implementation. +include!("official.rs"); + +mod installed_conditions { + use super::*; + use durable_workflow::{RawRequest, WorkflowResultOptions}; + + type Outcome = Result>; + + fn check(ok: bool, message: &str) -> Outcome<()> { + if ok { + Ok(()) + } else { + Err(message.into()) + } + } + + fn event(kind: &str, at: &str, payload: Value) -> Value { + json!({"event_type": kind, "timestamp": at, "payload": payload}) + } + + // Faithful public-Worker port of H's decision_history: sequences, row order, + // occurrence identity, timestamps and timeout values are kept unchanged. + fn decision_history(reopen: u64, genuine_timeout: bool, tail_fire: bool) -> Vec { + let wait = |seq: u64, at: &str, timeout: u64| { + event( + "ConditionWaitOpened", + at, + json!({ + "sequence": seq, "condition_wait_id": format!("condition:{seq}"), + "condition_wait_occurrence_id": "rust:condition-wait:0", + "condition_key": "decision-1", "condition_definition_fingerprint": "sha256:decision-v1", + "timeout_seconds": timeout + }), + ) + }; + let timer = |kind: &str, seq: u64, at: &str, delay: u64| { + event( + kind, + at, + json!({ + "sequence": seq, "timer_id": format!("condition-timer:{seq}"), + "timer_kind": "condition_timeout", "condition_wait_id": format!("condition:{seq}"), + "delay_seconds": delay + }), + ) + }; + let orphan = timer("TimerFired", 1, "2026-11-04T04:12:36.956000+00:00", 300); + let mut history = vec![ + wait(1, "2026-11-04T04:07:36.890000+00:00", 300), + timer("TimerScheduled", 1, "2026-11-04T04:07:36.891000+00:00", 300), + ]; + if genuine_timeout { + history.push(orphan.clone()); + } + history.extend([ + wait(2, "2026-11-04T04:07:36.908000+00:00", 300), + timer("TimerScheduled", 2, "2026-11-04T04:07:36.909000+00:00", 300), + event("SignalReceived", "2026-11-04T04:09:36.979000+00:00", json!({ + "signal_name": "decision", "signal_id": "s1", "workflow_command_id": "command-s1", + "arguments": encode_payload(&json!(["proposal-1"]), DEFAULT_CODEC).unwrap() + })), + event("ConditionWaitSatisfied", "2026-11-04T04:09:37.008000+00:00", json!({ + "sequence": 2, "condition_wait_id": "condition:2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "decision-1", + "condition_definition_fingerprint": "sha256:decision-v1" + })), + timer("TimerCancelled", 2, "2026-11-04T04:09:37.009000+00:00", 300), + wait(3, "2026-11-04T04:09:37.010000+00:00", reopen), + timer("TimerScheduled", 3, "2026-11-04T04:09:37.011000+00:00", reopen), + ]); + if !genuine_timeout { + history.push(orphan); + } + if tail_fire { + history.push(timer( + "TimerFired", + 3, + "2026-11-04T04:12:37.216000+00:00", + reopen, + )); + } + history + } + + async fn decision_replay(history: Vec, delivery: &str) -> Outcome { + let id = format!("installed-orphan-{delivery}"); + let task = json!({"task_id": id, "workflow_id": "installed-orphan", "run_id": "installed-run", + "workflow_type": "installed.decision", "arguments": encode_payload(&json!([]), DEFAULT_CODEC)?, + "payload_codec": DEFAULT_CODEC, "history_events": history, "workflow_task_attempt": 1, + "lease_owner": "regression-corpus-worker"}); + let server = FixtureServer::start(task, 0, None); + let client = Client::builder(server.base_url()) + .timeout(Duration::from_secs(2)) + .build()?; + let typed_error = Arc::new(Mutex::new(None::)); + let captured_error = Arc::clone(&typed_error); + let mut worker = + Worker::new(client, "regression-corpus").worker_id("regression-corpus-worker"); + worker.register_workflow("installed.decision", move |ctx, _| { + let captured_error = Arc::clone(&captured_error); + async move { + let outcome = match ctx + .wait_condition( + ConditionWaitOptions::new("decision-1", "sha256:decision-v1") + .timeout(Duration::from_secs(300)), + || Ok(false), + ) + .await + { + Err(Error::NonDeterministicReplay(failure)) => { + *captured_error.lock().unwrap() = Some(json!({"reason": failure.reason, + "sequence": failure.sequence})); + return Err(Error::NonDeterministicReplay(failure)); + } + result => result?, + }; + Ok(json!(if outcome.is_satisfied() { + "satisfied" + } else { + "timed_out" + })) + } + }); + check( + worker.run_once().await? == 1, + "one history task must settle", + )?; + let completion = server.request_body(&format!("/api/worker/workflow-tasks/{id}/complete")); + let failure = server.request_body(&format!("/api/worker/workflow-tasks/{id}/fail")); + check( + completion.is_some() != failure.is_some(), + "exactly one task report is required", + )?; + let captured = typed_error.lock().unwrap().clone(); + Ok(json!({"completion": completion, "failure": failure, "typed_error": captured})) + } + + #[tokio::test] + async fn installed_history_replay() -> Outcome<()> { + let mut results = serde_json::Map::new(); + for name in [ + "two-member-condition-group-server-reopen.json", + "timer-separated-condition-group-server-reopen.json", + "cooperative-grouped-condition-reopen.json", + "condition-orphaned-duplicate-timeout-keeps-wait.json", + ] { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/replay-regressions") + .join(name); + let fixture: Value = serde_json::from_slice(&fs::read(path)?)?; + let a = execute_fixture_delivery(&fixture, "installed-a").await?; + let b = execute_fixture_delivery(&fixture, "installed-b").await?; + check( + a == b, + "fresh workers must produce identical canonical replay results", + )?; + results.insert(name.to_string(), a); + } + for reopen in [300, 180] { + for tail in [false, true] { + let label = format!("reopen-{reopen}-tail-{tail}"); + let history = decision_history(reopen, false, tail); + let a = decision_replay(history.clone(), "a").await?; + let b = decision_replay(history.clone(), "b").await?; + check(a == b, "fresh workers must preserve orphan/tail results")?; + check( + a["failure"].is_null(), + "valid orphan history must not fail replay", + )?; + let commands = a["completion"]["commands"] + .as_array() + .ok_or("missing command list")?; + check( + commands.len() == 1, + "orphan/tail replay must produce exactly one command", + )?; + if tail { + check( + commands[0]["type"] == "complete_workflow", + "tail timer must complete as timed out", + )?; + let envelope: PayloadEnvelope = + serde_json::from_value(commands[0]["result"].clone())?; + let value: Value = decode_payload(&envelope)?; + check( + value == "timed_out", + "tail result must be the typed timed-out value", + )?; + } else { + check( + commands[0]["type"] == "open_condition_wait" + && commands[0]["timeout_seconds"] == 0, + "late orphan must keep logical wait open with zero remaining time", + )?; + } + results.insert(label, json!({"input_history": history, "observed": a})); + } + } + let history = decision_history(300, true, false); + let negative = decision_replay(history.clone(), "negative-a").await?; + let repeat = decision_replay(history.clone(), "negative-b").await?; + check( + negative == repeat, + "genuine-timeout refusal must survive fresh workers", + )?; + check( + negative["completion"].is_null(), + "genuine timeout must not publish completion", + )?; + check( + negative["typed_error"]["reason"] == "condition_wait_reopened_after_timeout" + && negative["typed_error"]["sequence"] == 2 + && negative["failure"]["failure"]["type"] == "RustWorkflowTaskFailure", + "genuine-timeout reopen must retain its typed refusal reason", + )?; + results.insert( + "genuine-timeout".into(), + json!({"input_history": history, "observed": negative}), + ); + fs::write( + "/state/replay.json", + serde_json::to_vec_pretty(&json!({"outcome": "pass", "cases": results}))?, + )?; + Ok(()) + } + + async fn history(client: &Client, workflow: &str, run: &str) -> Outcome> { + let mut events = Vec::new(); + let mut next = None; + for _ in 0..50 { + let mut request = RawRequest::get(["workflows", workflow, "runs", run, "history"]); + if let Some(token) = next.take() { + request = request.query("next_page_token", token); + } + let response = client.control_plane_request(request).await?; + check(response.status() == 200, "history read must succeed")?; + let page = response.json()?; + events.extend( + page["events"] + .as_array() + .ok_or("history has no event list")? + .iter() + .cloned(), + ); + next = page["next_page_token"] + .as_str() + .filter(|s| !s.is_empty()) + .map(str::to_owned); + if next.is_none() { + return Ok(events); + } + } + Err("history pagination did not end".into()) + } + + async fn wait_history( + client: &Client, + workflow: &str, + run: &str, + opens: usize, + ) -> Outcome> { + for _ in 0..300 { + let events = history(client, workflow, run).await?; + check( + !events.iter().any(|e| { + matches!( + e["event_type"].as_str(), + Some("WorkflowFailed" | "ReplayFailure") + ) + }), + "live workflow must not record a replay failure", + )?; + if events + .iter() + .filter(|e| e["event_type"] == "ConditionWaitOpened") + .count() + >= opens + { + return Ok(events); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err("condition opens did not arrive within the bounded wait".into()) + } + + #[tokio::test] + #[ignore = "requires separately admitted unchanged published PHP 2.11.8 runtime"] + async fn installed_live_group() -> Outcome<()> { + let token = Some(fs::read_to_string("/credentials/token")?); + let operator = Client::builder("http://qualification-origin:8080") + .control_token(token.clone()) + .build()?; + let mut ready = false; + for _ in 0..120 { + if let Ok(response) = operator + .control_plane_request(RawRequest::get(["health"]).timeout(Duration::from_secs(2))) + .await + { + if response.status() == 200 { + ready = true; + break; + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + check(ready, "PHP runtime readiness deadline")?; + let cluster = operator + .control_plane_request(RawRequest::get(["cluster", "info"])) + .await?; + check( + cluster.status() == 200, + "cluster identity read must succeed", + )?; + check( + cluster.json()?["version"] == "2.11.8", + "wrong PHP Server version", + )?; + let client = Client::builder("http://qualification-origin:8080") + .worker_token(token) + .build()?; + let mut worker = Worker::new(client, "installed-condition-history") + .worker_id("installed-condition-history") + .max_concurrent_workflow_tasks(1) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow("installed.two-member", |ctx, _| async move { + let votes = ctx.clone(); + let approvals = ctx.clone(); + let outcomes = ctx + .parallel(vec![ + ParallelOperation::condition( + ConditionWaitOptions::new("votes", "sha256:votes-v1") + .timeout(Duration::from_secs(300)), + move || Ok(votes.signals("vote")?.len() >= 2), + ), + ParallelOperation::condition( + ConditionWaitOptions::new("approvals", "sha256:approvals-v1") + .timeout(Duration::from_secs(300)), + move || Ok(!approvals.signals("approve")?.is_empty()), + ), + ]) + .await?; + if !matches!( + outcomes.as_slice(), + [ + ParallelResult::Condition(ConditionWaitResult::Satisfied), + ParallelResult::Condition(ConditionWaitResult::Satisfied), + ] + ) { + return Err(Error::WorkerLoop( + "both live conditions must be satisfied".into(), + )); + } + Ok(json!("both")) + }); + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + let execution = tokio::spawn(async move { + worker + .run_until(async { + let _ = stopped.await; + }) + .await + }); + let driver = async { + let handle = operator + .start_workflow( + "installed.two-member", + "installed-condition-history", + "installed-two-member", + json!([]), + ) + .await?; + let run = handle + .run_id + .as_deref() + .ok_or("start omitted selected run")?; + let initial = wait_history(&operator, &handle.workflow_id, run, 2).await?; + handle.signal("vote", json!(["one"])).await?; + let reopened = wait_history(&operator, &handle.workflow_id, run, 4).await?; + handle.signal("vote", json!(["two"])).await?; + handle.signal("approve", json!(["yes"])).await?; + let value = handle + .result(WorkflowResultOptions { + poll_interval: Duration::from_millis(100), + timeout: Duration::from_secs(30), + }) + .await?; + check( + value == "both", + "two-member workflow must return typed both", + )?; + let final_history = history(&operator, &handle.workflow_id, run).await?; + fs::write( + "/state/live.json", + serde_json::to_vec_pretty(&json!({"outcome": "pass", + "workflow_id": handle.workflow_id, "run_id": run, "result": value, + "initial": initial, "reopened": reopened, "history": final_history}))?, + )?; + Ok::<_, Box>(()) + } + .await; + let _ = stop.send(()); + execution.await??; + driver + } +} diff --git a/scripts/acceptance/condition-history/guest-host.py b/scripts/acceptance/condition-history/guest-host.py new file mode 100644 index 0000000..cdd4df6 --- /dev/null +++ b/scripts/acceptance/condition-history/guest-host.py @@ -0,0 +1,758 @@ +"""Source proposal for an external, task-scoped Lima/VZ storage guardian. + +The plan is inert. Prepare/guard may allocate storage and consume an exact +private guest-launch request only after separate admission; audit is read-only. +""" +import argparse +import ctypes +import errno +import gzip +import hashlib +import json +import os +import pathlib +import plistlib +import re +import selectors +import shlex +import stat +import subprocess +import sys +import time + +LIMA_SHA = '216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49' +AGENT_SHA = '4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959' +AGENT_BINARY_SHA = 'd799765623d5fa3b52ac30dd437b8f867968aeac594933c8ddc3bf1ac78cb26a' +IMAGE_SHA = '4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6' +IMAGE_BYTES = 358_298_593 +STORE_BYTES = 32 * 1024**3 +GUEST_BYTES = 20 * 1024**3 +MEMORY_BYTES = 6 * 1024**3 +PROFILE_SHA = 'fd12240c0a04fbe0614901038586a9cc411f154e11a56d6d1fcd3f4f045f0516' +AUTHORITY_KEYS = {'schema', 'token', 'runner_commit', 'manifest_sha256', 'guardian_sha256', + 'started_unix', 'active_unix', 'cleanup_unix', 'parent', 'limactl', + 'guest_agent', 'host_boot_id', 'image_dev', 'image_ino', 'whole_device', + 'volume_uuid', 'profile_sha256', 'profile', 'guest_image', 'guest_agent_binary_sha256'} +INTENT_KEYS = (AUTHORITY_KEYS - {'image_dev', 'image_ino', 'whole_device', 'volume_uuid', + 'profile_sha256', 'profile', 'guest_image', 'guest_agent_binary_sha256'}) | {'cached_image'} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def regular(path, limit=8 * 1024**2): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), 'unsafe host handoff') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'host handoff replaced') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and after.st_nlink == 1 + and (opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), 'host handoff changed') + return data + + +def exclusive(path, value): + data = canonical(value) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), 0o600) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + require(regular(path) == data, 'host handoff changed after write') + return digest(data) + + +def paths(parent, token): + root = pathlib.Path(parent) + require(root.is_absolute() and root.resolve() == root and re.fullmatch('[0-9a-f]{16}', token), 'invalid host scope') + require(root.name == 'sr-' + token and not any(ch.isspace() for ch in str(root)), 'wrong host scope name') + home = root / 'v/l' + name = 'sdk-rust-' + token + require(len(str(home / name / 'ssh.sock.1234567890123456').encode()) < 104, 'Lima socket path is too long') + return {'parent': root, 'mount': root / 'v', 'image': root / 'store.dmg', + 'home': home, 'instance': home / name, 'name': name} + + +def environment(record): + scoped = paths(record['parent'], record['token']) + return {'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'HOME': str(pathlib.Path.home()), + 'LIMA_HOME': str(scoped['home']), + 'LIMA_SHELLENV_BLOCK': '*', 'TMPDIR': record.get('_setup_tmp', str(scoped['mount'] / 'tmp'))} + + +def plan(parent, token, limactl, image_path, profile): + scoped = paths(parent, token) + image = pathlib.Path(image_path) + require(image == scoped['mount'] / 'a/guest.raw.gz', 'guest image is outside bounded store') + value = json.loads(json.dumps(profile)) + # Pin the entire JSON document, including provision scripts and networking. + expected = json.dumps(value, sort_keys=True, indent=2) + '\n' + require(digest(expected.encode()) == PROFILE_SHA, 'unreviewed guest template') + require(pathlib.Path(limactl).is_absolute(), 'absolute VM launcher required') + require(value['images'] == [{'location': '__TASK_IMAGE__', 'arch': 'x86_64', 'digest': 'sha256:' + IMAGE_SHA}], 'wrong guest image profile') + require(value['cpus'] == 4 and value['memory'] == '6GiB' and value['disk'] == '20GiB' + and value['vmType'] == 'vz' and value['arch'] == 'x86_64', 'wrong guest resource profile') + require(value['base'] == [] and value['mounts'] == [] and value['additionalDisks'] == [] + and value['copyToHost'] == [] and value['env'] == {} and value['propagateProxyEnv'] is False, + 'inherited inputs or host shares refused') + require(all(value['ssh'][key] is False for key in ('loadDotSSHPubKeys', 'forwardAgent', 'forwardX11', 'forwardX11Trusted')) + and value['containerd'] == {'system': False, 'user': False}, 'forwarded credentials or implicit runtime refused') + require(value['portForwards'] == [{'guestIP': '0.0.0.0', 'guestIPMustBeZero': False, + 'guestPortRange': [1, 65535], 'proto': 'any', 'ignore': True}], 'guest service forwarding refused') + value['images'][0]['location'] = str(image) + return {'schema': 'durable-workflow.condition-guest-plan/v1', 'profile': value, + 'profile_sha256': digest(canonical(value)), 'guest_disk_bytes': GUEST_BYTES, + 'host_store_bytes': STORE_BYTES, 'guest_memory_bytes': MEMORY_BYTES, 'vcpus': 4, + 'minimum_host_available_bytes': 10 * 1024**3, 'minimum_host_free_disk_bytes': 48 * 1024**3, + 'setup_seconds': 300, 'active_seconds': 1740, 'cleanup_seconds': 1800, + 'create_store': ['/usr/bin/hdiutil', 'create', '-sectors', str(STORE_BYTES // 512), + '-type', 'UDIF', '-layout', 'GPTSPUD', '-fs', 'APFS', '-nospotlight', + '-volname', 'sr-' + token, str(scoped['image'])], + 'attach_store': ['/usr/bin/hdiutil', 'attach', '-nobrowse', '-owners', 'on', + '-mountpoint', str(scoped['mount']), '-plist', str(scoped['image'])], + 'create_vm': [limactl, '--tty=false', 'create', '--name', scoped['name'], '-'], + 'start_vm': [limactl, '--tty=false', 'start', '--timeout', '5m', scoped['name']], + 'stop_vm': [limactl, '--tty=false', 'stop', '--force', scoped['name']], + 'delete_vm': [limactl, '--tty=false', 'delete', '--force', scoped['name']], + 'environment': environment({'parent': parent, 'token': token})} + + +def call(record, argv, input_data=None): + remaining = min(record['_cleanup_mono'], record.get('_stage_mono', float('inf'))) - time.monotonic() + require(remaining > 0, 'external cleanup deadline exhausted') + # Read with a hard in-memory ceiling; native commands cannot fill host logs. + require(input_data is None or len(input_data) <= 8192, 'oversize native input') + child = subprocess.Popen(argv, stdin=subprocess.PIPE if input_data is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=environment(record), close_fds=True) + data = bytearray() + cutoff = time.monotonic() + min(record.get('_command_seconds', 20), remaining) + try: + if input_data is not None: + child.stdin.write(input_data) + child.stdin.close() + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while selector.get_map(): + require(time.monotonic() < cutoff, 'native scoped control timed out') + for key, _ in selector.select(min(0.1, max(0, cutoff - time.monotonic()))): + block = os.read(key.fileobj.fileno(), 65536) + if not block: + selector.unregister(key.fileobj) + continue + require(len(data) + len(block) <= 2 * 1024**2, 'oversize native control response') + data.extend(block) + child.wait(timeout=max(0.001, cutoff - time.monotonic())) + require(child.returncode == 0, 'native scoped control failed') + return bytes(data) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=2) + child.stdout.close() + + +def boot_identity(): + value = subprocess.run(['/usr/sbin/sysctl', '-n', 'kern.bootsessionuuid'], check=True, + capture_output=True, text=True, timeout=5).stdout.strip().lower() + require(re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value), 'missing Darwin boot authority') + return value + + +class BsdInfo(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint32) for name in ( + 'flags', 'status', 'xstatus', 'pid', 'ppid', 'uid', 'gid', 'ruid', 'rgid', 'svuid', 'svgid', 'reserved')] + _fields_ += [('comm', ctypes.c_char * 16), ('name', ctypes.c_char * 32)] + _fields_ += [(name, ctypes.c_uint32) for name in ('nfiles', 'pgid', 'pjobc', 'tdev', 'tpgid')] + _fields_ += [('nice', ctypes.c_int32), ('start_seconds', ctypes.c_uint64), ('start_microseconds', ctypes.c_uint64)] + + +def bsd_identity(pid): + require(sys.platform == 'darwin', 'Darwin host guardian required') + library = ctypes.CDLL('/usr/lib/libproc.dylib', use_errno=True) + library.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] + library.proc_pidinfo.restype = ctypes.c_int + info = BsdInfo() + count = library.proc_pidinfo(pid, 3, 0, ctypes.byref(info), ctypes.sizeof(info)) + if count == 0 and ctypes.get_errno() == errno.ESRCH: + return None + require(count == ctypes.sizeof(info) and info.pid == pid and info.uid == os.getuid() + and info.start_seconds > 0 and info.start_microseconds < 1_000_000, 'unavailable host kernel authority') + return {'pid': pid, 'pgid': info.pgid, 'uid': info.uid, + 'start_seconds': info.start_seconds, 'start_microseconds': info.start_microseconds} + + +def process_identity(pid): + boot = boot_identity() + first, second = bsd_identity(pid), bsd_identity(pid) + require(first == second and boot == boot_identity(), 'host process epoch changed during observation') + return dict(first, boot_id=boot) if first else None + + +def authority(path, expected_sha, source_sha): + raw = regular(path, 64 * 1024) + require(digest(raw) == expected_sha, 'host authority digest mismatch') + value = json.loads(raw) + require(set(value) == AUTHORITY_KEYS and value['schema'] == 'durable-workflow.condition-host/v1', 'wrong host authority schema') + require(value['guardian_sha256'] == source_sha and re.fullmatch('[0-9a-f]{40}', value['runner_commit']) + and re.fullmatch('[0-9a-f]{64}', value['manifest_sha256']), 'wrong captured host source') + require(all(type(value[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix', 'image_dev', 'image_ino')) + and value['active_unix'] == value['started_unix'] + 1740 + and value['cleanup_unix'] == value['started_unix'] + 1800, 'wrong host deadlines or inode authority') + require(value['host_boot_id'] == boot_identity(), 'host boot changed') + scoped = paths(value['parent'], value['token']) + info = scoped['parent'].lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and stat.S_IMODE(info.st_mode) == 0o700, + 'wrong host parent authority') + binary = pathlib.Path(value['limactl']).resolve(strict=True) + require(digest(binary.read_bytes()) == LIMA_SHA, 'reviewed VM launcher changed') + agent = pathlib.Path(value['guest_agent']).resolve(strict=True) + require(digest(agent.read_bytes()) == AGENT_SHA, 'reviewed guest agent changed') + selected_agent(value) + template = json.loads(json.dumps(value['profile'])) + template['images'][0]['location'] = '__TASK_IMAGE__' + expected_profile = plan(value['parent'], value['token'], value['limactl'], + str(scoped['mount'] / 'a/guest.raw.gz'), template)['profile'] + require(value['profile'] == expected_profile and digest(canonical(value['profile'])) == value['profile_sha256'] + and value['guest_agent_binary_sha256'] == AGENT_BINARY_SHA, 'captured profile or agent authority mismatch') + require(re.fullmatch('/dev/disk[0-9]+', value['whole_device']) + and re.fullmatch('[0-9A-Fa-f-]{36}', value['volume_uuid']) + and re.fullmatch('[0-9a-f]{64}', value['profile_sha256']), 'invalid native store authority') + value['_cleanup_mono'] = time.monotonic() + max(0, value['cleanup_unix'] - time.time()) + value['_active_mono'] = time.monotonic() + max(0, value['active_unix'] - time.time()) + return value + + +def verify_store(record, profile_required=True): + scoped = paths(record['parent'], record['token']) + info = scoped['image'].lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == os.getuid() + and (info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) + and 0 < info.st_size <= STORE_BYTES + 2 * 1024**2, 'fixed host image changed') + entries = plistlib.loads(call(record, ['/usr/bin/hdiutil', 'info', '-plist']))['images'] + matches = [entry for entry in entries if pathlib.Path(entry['image-path']).resolve() == scoped['image']] + require(len(matches) == 1, 'host image attachment is missing or ambiguous') + entities = matches[0]['system-entities'] + require(any(entry.get('dev-entry') == record['whole_device'] for entry in entities) + and any(entry.get('mount-point') == str(scoped['mount']) for entry in entities), 'foreign host image attachment') + whole = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', record['whole_device']])) + volume = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', str(scoped['mount'])])) + require(whole['TotalSize'] == STORE_BYTES and volume['VolumeUUID'].lower() == record['volume_uuid'].lower() + and volume['TotalSize'] <= STORE_BYTES and volume['MountPoint'] == str(scoped['mount']) + and volume['FilesystemType'] == 'apfs', + 'host store does not have its fixed byte boundary') + if profile_required: + require(scoped['home'].stat().st_dev == scoped['mount'].stat().st_dev + and digest(regular(scoped['mount'] / 'guest.yaml')) == record['profile_sha256'], + 'reviewed guest profile changed') + + +def input_bytes(path, expected_size, expected_sha, destination=None, deadline=None): + """Hash/copy only the pinned public inputs, streaming within the setup bound.""" + path = pathlib.Path(path).resolve(strict=True) + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size == expected_size, + 'wrong public input size or type') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + stream = os.fdopen(fd, 'rb') + target = None + state, count = hashlib.sha256(), 0 + try: + if destination: + target = os.fdopen(os.open(destination, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600), 'wb') + with stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'public input replaced') + while True: + if deadline is not None:require(time.monotonic() < deadline, 'public input setup cutoff') + block = stream.read(1024**2) + if not block:break + count += len(block) + require(count <= expected_size, 'public input grew') + state.update(block) + if target:target.write(block) + after = os.fstat(stream.fileno()) + require((opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns) and after.st_nlink == 1, + 'public input changed') + require(count == expected_size and state.hexdigest() == expected_sha, 'public input digest mismatch') + if target: + target.flush() + os.fsync(target.fileno()) + finally: + if not stream.closed:stream.close() + if target:target.close() + + +def bound_input(path, expected_size, expected_sha, deadline=None): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), + 'unsafe consumed guest input') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + state, count = hashlib.sha256(), 0 + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'consumed guest input replaced on open') + while True: + if deadline is not None:require(time.monotonic() < deadline, 'consumed input cutoff') + block = stream.read(1024**2) + if not block:break + count += len(block) + require(count <= expected_size, 'consumed guest input grew') + state.update(block) + after = os.fstat(stream.fileno()) + current = path.lstat() + identity = lambda info: (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns, info.st_nlink) + require(identity(before) == identity(opened) == identity(after) == identity(current), 'consumed guest input changed') + require(count == expected_size and state.hexdigest() == expected_sha, 'consumed guest input digest mismatch') + return {'device': opened.st_dev, 'inode': opened.st_ino, 'bytes': expected_size, 'sha256': expected_sha} + + +def selected_agent(record): + # Lima 2.2.1 usrlocal.ShareLima/GuestAgentBinary uses this executable-relative + # compressed path first; instance.Start passes it as --guestagent to hostagent. + launcher = pathlib.Path(record['limactl']).resolve() + expected = launcher.parent.parent / 'share/lima/lima-guestagent.Linux-x86_64.gz' + require(pathlib.Path(record['limactl']) == launcher and pathlib.Path(record['guest_agent']) == expected + and expected.resolve() == expected, 'pinned agent is not the executable-selected Lima agent') + return expected + + +def agent_binary_sha(path, deadline=None): + state, count = hashlib.sha256(), 0 + with gzip.open(path, 'rb') as stream: + while True: + if deadline is not None:require(time.monotonic() < deadline, 'guest agent input cutoff') + data = stream.read(1024**2) + if not data:break + count += len(data) + require(count <= 64 * 1024**2, 'oversize uncompressed guest agent') + state.update(data) + require(count > 0, 'empty guest agent') + require(state.hexdigest() == AGENT_BINARY_SHA, 'uncompressed pinned guest agent changed') + input_bytes(path, 8_257_812, AGENT_SHA, deadline=deadline) + return state.hexdigest() + + +def verify_guest_inputs(record): + scoped = paths(record['parent'], record['token']) + identity = bound_input(scoped['mount'] / 'a/guest.raw.gz', IMAGE_BYTES, IMAGE_SHA, record['_active_mono']) + require(identity == record['guest_image'], 'consumed image identity differs from copied destination') + selected_agent(record) + input_bytes(record['guest_agent'], 8_257_812, AGENT_SHA, deadline=record['_active_mono']) + require(agent_binary_sha(record['guest_agent'], record['_active_mono']) == record['guest_agent_binary_sha256'], 'consumed guest agent changed') + + +def setup_intent(path, expected_sha, source_sha): + require(stat.S_IMODE(path.lstat().st_mode) == 0o600, 'private setup intent required') + raw = regular(path, 64 * 1024) + require(digest(raw) == expected_sha, 'setup intent digest mismatch') + value = json.loads(raw) + require(set(value) == INTENT_KEYS and value['schema'] == 'durable-workflow.condition-host-setup/v1', + 'wrong setup intent schema') + require(value['guardian_sha256'] == source_sha and re.fullmatch('[0-9a-f]{40}', value['runner_commit']) + and re.fullmatch('[0-9a-f]{64}', value['manifest_sha256']), 'wrong setup source authority') + require(all(type(value[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix')) + and value['active_unix'] == value['started_unix'] + 1740 + and value['cleanup_unix'] == value['started_unix'] + 1800 + and value['started_unix'] <= time.time() < value['started_unix'] + 300, 'wrong setup deadlines') + require(value['host_boot_id'] == boot_identity(), 'setup host boot changed') + scoped = paths(value['parent'], value['token']) + require(not os.path.lexists(scoped['parent']), 'setup scope already exists') + for key in ('limactl', 'guest_agent', 'cached_image'): + require(pathlib.Path(value[key]).is_absolute(), 'absolute pinned input required') + now, monotonic = time.time(), time.monotonic() + value['_cleanup_mono'] = monotonic + max(0, value['cleanup_unix'] - now) + value['_active_mono'] = monotonic + max(0, value['active_unix'] - now) + value['_stage_mono'] = monotonic + max(0, value['started_unix'] + 300 - now) + return value + + +def capacity(record): + require(sys.platform == 'darwin' and os.uname().machine == 'x86_64', 'native Intel VZ host required') + version = call(record, ['/usr/bin/sw_vers', '-productVersion']).decode().strip() + build = call(record, ['/usr/bin/sw_vers', '-buildVersion']).decode().strip() + require((version, build) == ('15.8.2', '24H232'), 'reviewed native OS changed') + memory = call(record, ['/usr/bin/vm_stat']).decode() + page = re.search(r'page size of (\d+) bytes', memory) + require(page, 'missing host page geometry') + available = 0 + for label in ('Pages free', 'Pages inactive', 'Pages speculative'): + match = re.search(r'^' + label + r':\s+(\d+)\.', memory, re.MULTILINE) + require(match, 'missing host memory readback') + available += int(match[1]) * int(page[1]) + filesystem = os.statvfs(paths(record['parent'], record['token'])['parent'].parent) + free = filesystem.f_bavail * filesystem.f_frsize + require(available >= 10 * 1024**3 and free >= 48 * 1024**3, 'fresh host capacity gate refused') + return {'available_bytes': available, 'free_disk_bytes': free, 'os': version, 'build': build} + + +def observe_attachment(record): + scoped = paths(record['parent'], record['token']) + entries = plistlib.loads(call(record, ['/usr/bin/hdiutil', 'info', '-plist']))['images'] + matches = [entry for entry in entries if pathlib.Path(entry['image-path']).resolve() == scoped['image']] + require(len(matches) == 1, 'setup image attachment missing or ambiguous') + entities = matches[0]['system-entities'] + require(any(e.get('mount-point') == str(scoped['mount']) for e in entities), 'setup image mount missing') + devices = [] + for entity in entities: + device = entity.get('dev-entry', '') + if re.fullmatch('/dev/disk[0-9]+', device): + info = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', device])) + if info['TotalSize'] == STORE_BYTES:devices.append(device) + require(len(devices) == 1, 'fixed setup device is ambiguous') + volume = plistlib.loads(call(record, ['/usr/sbin/diskutil', 'info', '-plist', str(scoped['mount'])])) + require(volume['FilesystemType'] == 'apfs' and volume['MountPoint'] == str(scoped['mount']) + and 0 < volume['TotalSize'] <= STORE_BYTES + and re.fullmatch('[0-9A-Fa-f-]{36}', volume['VolumeUUID']), 'wrong setup volume readback') + return {'whole_device': devices[0], 'volume_uuid': volume['VolumeUUID']} + + +def remove_store(record): + scoped = paths(record['parent'], record['token']) + call(record, ['/usr/bin/hdiutil', 'detach', record['whole_device']]) + require(not os.path.ismount(scoped['mount']), 'owned host store remains mounted') + info = scoped['image'].lstat() + require((info.st_dev, info.st_ino) == (record['image_dev'], record['image_ino']) + and info.st_nlink == 1 and stat.S_ISREG(info.st_mode), 'host image changed before deletion') + scoped['image'].unlink() + + +def prepare(record, template): + """Resident watchdog owns setup before the first disk mutation. No VM start.""" + scoped = paths(record['parent'], record['token']) + proposal = plan(record['parent'], record['token'], record['limactl'], + str(scoped['mount'] / 'a/guest.raw.gz'), template) + observed = capacity(record) + selected_agent(record) + for key, size, sha in (('limactl', 35_175_712, LIMA_SHA), ('guest_agent', 8_257_812, AGENT_SHA), + ('cached_image', IMAGE_BYTES, IMAGE_SHA)): + input_bytes(record[key], size, sha, deadline=record['_stage_mono']) + scoped['parent'].mkdir(mode=0o700) + record['_setup_tmp'] = str(scoped['parent']) + exclusive(scoped['parent'] / 'setup-ready.json', {'guardian_sha256': record['guardian_sha256'], + 'runner_commit': record['runner_commit'], 'manifest_sha256': record['manifest_sha256'], + 'process': process_identity(os.getpid()), 'capacity': observed}) + attached = False + try: + record['_command_seconds'] = 180 + call(record, proposal['create_store']) + info = scoped['image'].lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == os.getuid() + and 0 < info.st_size <= STORE_BYTES + 2 * 1024**2, 'wrong newly created store') + record.update(image_dev=info.st_dev, image_ino=info.st_ino) + scoped['mount'].mkdir(mode=0o700) + call(record, proposal['attach_store']) + record.update(observe_attachment(record)) + attached = True + verify_store(record, profile_required=False) + for directory in ('a', 'l', 'tmp'):(scoped['mount'] / directory).mkdir(mode=0o700) + record.pop('_setup_tmp') + input_bytes(record['cached_image'], IMAGE_BYTES, IMAGE_SHA, scoped['mount'] / 'a/guest.raw.gz', record['_stage_mono']) + record['guest_image'] = bound_input(scoped['mount'] / 'a/guest.raw.gz', IMAGE_BYTES, IMAGE_SHA, record['_stage_mono']) + record['guest_agent_binary_sha256'] = agent_binary_sha(record['guest_agent'], record['_stage_mono']) + record['profile'] = proposal['profile'] + record['profile_sha256'] = exclusive(scoped['mount'] / 'guest.yaml', proposal['profile']) + require(record['profile_sha256'] == proposal['profile_sha256'], 'rendered setup template changed') + value = {key: record[key] for key in AUTHORITY_KEYS if key != 'schema'} + value['schema'] = 'durable-workflow.condition-host/v1' + record_sha = exclusive(scoped['parent'] / 'host-record.json', value) + verify_store(record) + require(time.monotonic() < record['_stage_mono'], 'setup window exhausted') + for key in ('_stage_mono', '_command_seconds'):record.pop(key, None) + return value, record_sha + except BaseException as error: + # Failed native create/attach may not return enough identity to authorize + # destructive recovery. Uncertain objects remain explicit external custody. + record.pop('_stage_mono', None) + record['_command_seconds'] = 20 + disposition = 'retained_uncertain_task_store' + if not attached and 'image_ino' in record: + try: + record.update(observe_attachment(record)) + attached = True + except BaseException: + pass + if attached: + try: + verify_store(record, profile_required=False) + remove_store(record) + disposition = 'owned_store_removed' + except BaseException: + pass + exclusive(scoped['parent'] / 'setup-failure.json', {'outcome': 'fail', + 'error_type': type(error).__name__, 'custody': disposition, + 'runner_commit': record['runner_commit'], 'manifest_sha256': record['manifest_sha256'], + 'store_authority': {key: record[key] for key in ('image_dev', 'image_ino', 'whole_device', 'volume_uuid') if key in record}}) + raise + + +def capture_agent(record): + scoped = paths(record['parent'], record['token']) + pid_path = scoped['instance'] / 'ha.pid' + if not pid_path.exists(): + return None + pid_text = regular(pid_path, 64).strip() + require(pid_text.isdigit() and int(pid_text) > 1, 'invalid owned VM PID file') + process = process_identity(int(pid_text)) + require(process and process['start_seconds'] >= record['started_unix'], 'stale VM process epoch') + command = call(record, ['/bin/ps', '-ww', '-p', pid_text.decode(), '-o', 'command=']).decode() + argv = shlex.split(command) + binding = '--pidfile=' + str(pid_path) + pairs = list(zip(argv, argv[1:])) + require(argv and pathlib.Path(argv[0]).resolve() == pathlib.Path(record['limactl']).resolve() + and 'hostagent' in argv and (binding in argv or ('--pidfile', str(pid_path)) in pairs) + and ('--guestagent', str(selected_agent(record))) in pairs + and scoped['name'] in argv and process_identity(process['pid']) == process, 'foreign VM host agent') + return process + + +def export_handoff(record, record_sha): + scoped = paths(record['parent'], record['token']) + raw = regular(scoped['parent'] / 'exports.json', 64 * 1024) + value = json.loads(raw) + require(set(value) == {'record_sha256', 'guest_boot_id', 'guest-summary.json', 'guest-audit.json'} + and value['record_sha256'] == record_sha + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value['guest_boot_id']), + 'wrong external export authority') + for name in ('guest-summary.json', 'guest-audit.json'): + require(digest(regular(scoped['parent'] / name)) == value[name], 'exported guest proof changed') + return value, digest(raw) + + +def finish_requested(record, record_sha): + scoped = paths(record['parent'], record['token']) + marker = scoped['parent'] / 'finish.json' + if not os.path.lexists(marker): + return False + value = json.loads(regular(marker, 64 * 1024)) + _, export_sha = export_handoff(record, record_sha) + require(value == {'record_sha256': record_sha, 'exports_sha256': export_sha}, 'wrong cleanup request authority') + return True + + +def effective_config(record, status): + scoped = paths(record['parent'], record['token']) + value = json.loads(call(record, [record['limactl'], 'list', '--json', '--all-fields', scoped['name']])) + require(value['name'] == scoped['name'] and value['dir'] == str(scoped['instance']) + and value['status'] == status and not value.get('errors') + and value['vmType'] == 'vz' and value['arch'] == 'x86_64' and value['cpus'] == 4 + and value['memory'] == MEMORY_BYTES and value['disk'] == GUEST_BYTES, + 'effective VM identity or resources changed') + def contains(actual, expected): + if expected == [] or expected == {}: + return actual is None or actual == expected + if isinstance(expected, dict): + return isinstance(actual, dict) and all(k in actual and contains(actual[k], v) for k, v in expected.items()) + if isinstance(expected, list): + return isinstance(actual, list) and len(actual) == len(expected) and all(contains(a, e) for a, e in zip(actual, expected)) + return type(actual) is type(expected) and actual == expected + # base/minimum version are template loader directives; every effective + # supplied field is checked, including mounts, images, ssh and provisioning. + expected = {k: v for k, v in record['profile'].items() if k not in ('base', 'minimumLimaVersion')} + config = dict(value['config']) + config.setdefault('env', {}) + require(contains(config, expected), 'effective VM profile differs from captured configuration') + return value + + +GUEST_AGENT_PROBE = '''import hashlib,json,pathlib +root=pathlib.Path('/proc');boot=(root/'sys/kernel/random/boot_id').read_text().strip();found=[] +for entry in root.iterdir(): + if not entry.name.isdigit():continue + try:comm=(entry/'comm').read_text().strip() + except FileNotFoundError:continue + if comm!='lima-guestagent':continue + try: + first=(entry/'stat').read_text();state=hashlib.sha256();count=0 + with (entry/'exe').open('rb') as stream: + while True: + data=stream.read(1048576) + if not data:break + count+=len(data) + if count>67108864:raise SystemExit('oversize running guest agent') + state.update(data) + second=(entry/'stat').read_text() + start=lambda value:int(value[value.rfind(') ')+2:].split()[19]) + if start(first)!=start(second):raise SystemExit('guest agent epoch changed') + found.append({'pid':int(entry.name),'start_ticks':start(second),'guest_boot_id':boot,'sha256':state.hexdigest()}) + except FileNotFoundError:raise SystemExit('guest process changed during probe') +if len(found)!=1 or (root/'sys/kernel/random/boot_id').read_text().strip()!=boot:raise SystemExit('ambiguous guest agent authority') +print(json.dumps(found[0],sort_keys=True)) +''' + + +def launch_guest(record, record_sha): + scoped = paths(record['parent'], record['token']) + request = json.loads(regular(scoped['parent'] / 'launch.json', 64 * 1024)) + require(request == {'record_sha256': record_sha, 'manifest_sha256': record['manifest_sha256'], 'action': 'create-and-start'}, + 'wrong exact guest launch request') + require(not scoped['instance'].exists(), 'guest instance already exists') + verify_store(record) + verify_guest_inputs(record) + # The bytes consumed by create come from captured authority, never the + # mutable guest.yaml pathname. No external process owns create/start. + profile = canonical(record['profile']) + require(digest(profile) == record['profile_sha256'], 'captured consumption profile changed') + record['_stage_mono'] = record['_active_mono'] + record['_command_seconds'] = 180 + try: + call(record, [record['limactl'], '--tty=false', 'create', '--name', scoped['name'], '-'], profile) + before = effective_config(record, 'Stopped') + exclusive(scoped['parent'] / 'vm-created.json', {'record_sha256': record_sha, + 'profile_sha256': digest(profile), 'guest_image': record['guest_image'], 'effective': before}) + verify_guest_inputs(record) + require(effective_config(record, 'Stopped') == before, 'VM config changed before guardian start') + record['_command_seconds'] = 300 + call(record, [record['limactl'], '--tty=false', 'start', '--timeout', '5m', scoped['name']]) + after = effective_config(record, 'Running') + process = capture_agent(record) + require(process, 'missing consumed VM hostagent authority') + proof = json.loads(call(record, [record['limactl'], 'shell', scoped['name'], '--', + 'sudo', '-n', 'python3', '-I', '-S', '-c', GUEST_AGENT_PROBE])) + require(set(proof) == {'pid', 'start_ticks', 'guest_boot_id', 'sha256'} + and type(proof['pid']) is int and proof['pid'] > 1 + and type(proof['start_ticks']) is int and proof['start_ticks'] > 0 + and proof['sha256'] == record['guest_agent_binary_sha256'] + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', proof['guest_boot_id']), + 'running guest agent does not match selected pinned agent') + exclusive(scoped['parent'] / 'vm-started.json', {'record_sha256': record_sha, + 'effective': after, 'host_process': process, 'agent': proof}) + finally: + for key in ('_stage_mono', '_command_seconds'):record.pop(key, None) + + +def cleanup(record): + scoped = paths(record['parent'], record['token']) + verify_store(record) + captured_path = scoped['parent'] / 'vm-process.json' + process = json.loads(regular(captured_path)) if captured_path.exists() else capture_agent(record) + if process: + current = process_identity(process['pid']) + if current: + require(current == process and capture_agent(record) == process, + 'VM process epoch changed before scoped stop') + require(process_identity(process['pid']) == process, 'VM process epoch changed at scoped stop') + call(record, [record['limactl'], '--tty=false', 'stop', '--force', scoped['name']]) + require(process_identity(process['pid']) is None, 'owned VM process survived') + require(not (scoped['instance'] / 'ha.pid').exists(), 'VM process custody is uncertain') + if scoped['instance'].exists(): + call(record, [record['limactl'], '--tty=false', 'delete', '--force', scoped['name']]) + require(not scoped['instance'].exists(), 'VM instance storage survived') + verify_store(record) + remove_store(record) + return {'guest_absent': True, 'store_unmounted': True, 'image_absent': True, + 'image_cache_disposition': 'whole_task_guest_and_store_removed'} + + +def audit(record, record_sha, source_sha): + scoped = paths(record['parent'], record['token']) + require(not os.path.lexists(scoped['image']) and not os.path.ismount(scoped['mount']) + and not os.path.lexists(scoped['instance']), 'external VM/store survivor') + ready = json.loads(regular(scoped['parent'] / 'host-ready.json')) + require(ready['record_sha256'] == record_sha and ready['source_sha256'] == source_sha + and process_identity(ready['process']['pid']) is None, 'host guardian still active or changed') + vm = json.loads(regular(scoped['parent'] / 'vm-process.json')) + require(process_identity(vm['pid']) is None, 'VM host agent survived') + result = json.loads(regular(scoped['parent'] / 'host-cleanup.json')) + require(result['record_sha256'] == record_sha and result['guardian_sha256'] == source_sha + and result['outcome'] == 'pending_independent_host_audit' + and result['guest_absent'] is True and result['store_unmounted'] is True and result['image_absent'] is True, + 'external cleanup did not pass') + exports, export_sha = export_handoff(record, record_sha) + require(finish_requested(record, record_sha), 'completed guest export request required') + guest = json.loads(regular(scoped['parent'] / 'guest-summary.json')) + guest_audit = json.loads(regular(scoped['parent'] / 'guest-audit.json')) + started = json.loads(regular(scoped['parent'] / 'vm-started.json')) + require(guest['runner_commit'] == record['runner_commit'] and guest['manifest_sha256'] == record['manifest_sha256'] + and guest['checks_outcome'] == 'pass' and guest_audit['outcome'] == 'pending_external_guest_teardown' + and guest['storage']['guest_boot_id'] == exports['guest_boot_id'] + and guest_audit['storage'] == guest['storage'] + and guest_audit['manifest_sha256'] == record['manifest_sha256'] + and started['record_sha256'] == record_sha + and started['agent']['guest_boot_id'] == exports['guest_boot_id'] + and started['agent']['sha256'] == record['guest_agent_binary_sha256'] + and guest_audit['summary_sha256'] == exports['guest-summary.json'], 'guest evidence does not authorize acceptance') + result.update(outcome='pass', exports_sha256=export_sha) + exclusive(scoped['parent'] / 'host-audit.json', result) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('plan', 'prepare', 'guard', 'audit')) + parser.add_argument('--record') + parser.add_argument('--record-sha256') + parser.add_argument('--plan-input') + parser.add_argument('--profile') + args = parser.parse_args(argv) + if args.mode == 'plan': + value = json.loads(pathlib.Path(args.plan_input).read_bytes()) + print(json.dumps(plan(**value), sort_keys=True)) + return + require('-I' in sys.orig_argv and '-S' in sys.orig_argv and '-c' in sys.orig_argv, 'captured host guardian required') + source_sha = digest(sys.orig_argv[sys.orig_argv.index('-c') + 1].encode()) + mode = args.mode + if mode == 'prepare': + initial = setup_intent(pathlib.Path(args.record), args.record_sha256, source_sha) + template = json.loads(regular(pathlib.Path(args.profile), 64 * 1024)) + _, record_sha = prepare(initial, template) + args.record = str(paths(initial['parent'], initial['token'])['parent'] / 'host-record.json') + args.record_sha256 = record_sha + mode = 'guard' + record = authority(pathlib.Path(args.record), args.record_sha256, source_sha) + scoped = paths(record['parent'], record['token']) + if mode == 'guard': + require(record['started_unix'] <= time.time() < record['started_unix'] + 300, + 'host setup cutoff exceeded') + verify_store(record) + exclusive(scoped['parent'] / 'host-ready.json', {'record_sha256': args.record_sha256, + 'source_sha256': source_sha, 'process': process_identity(os.getpid())}) + control_error = None + launched = False + try: + while time.monotonic() < record['_active_mono']: + if not launched and os.path.lexists(scoped['parent'] / 'launch.json'): + launch_guest(record, args.record_sha256) + launched = True + process = capture_agent(record) + receipt = scoped['parent'] / 'vm-process.json' + if process and not receipt.exists(): + exclusive(receipt, process) + if finish_requested(record, args.record_sha256): + break + time.sleep(0.25) + except BaseException as error: + # Bad handoffs or observations never bypass the cleanup attempt. + control_error = type(error).__name__ + try: + result = dict(cleanup(record), outcome='pending_independent_host_audit') + except BaseException as error: + result = {'outcome': 'fail', 'error_type': type(error).__name__, + 'custody': 'external_owner_must_resolve_retained_task_objects'} + if control_error: + result.update(outcome='fail', control_error_type=control_error) + result.update(record_sha256=args.record_sha256, guardian_sha256=source_sha) + exclusive(scoped['parent'] / 'host-cleanup.json', result) + require(result['outcome'] != 'fail', 'external cleanup incomplete') + else: + print(json.dumps(audit(record, args.record_sha256, source_sha))) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/guest-profile.json b/scripts/acceptance/condition-history/guest-profile.json new file mode 100644 index 0000000..6129427 --- /dev/null +++ b/scripts/acceptance/condition-history/guest-profile.json @@ -0,0 +1,77 @@ +{ + "additionalDisks": [], + "arch": "x86_64", + "base": [], + "caCerts": { + "certs": [], + "files": [], + "removeDefaults": false + }, + "containerd": { + "system": false, + "user": false + }, + "copyToHost": [], + "cpus": 4, + "disk": "20GiB", + "env": {}, + "hostResolver": { + "enabled": false, + "hosts": {} + }, + "images": [ + { + "arch": "x86_64", + "digest": "sha256:4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6", + "location": "__TASK_IMAGE__" + } + ], + "memory": "6GiB", + "minimumLimaVersion": "2.2.1", + "mountInotify": false, + "mountType": "virtiofs", + "mounts": [], + "nestedVirtualization": false, + "networks": [ + { + "vzNAT": true + } + ], + "portForwards": [ + { + "guestIP": "0.0.0.0", + "guestIPMustBeZero": false, + "guestPortRange": [ + 1, + 65535 + ], + "ignore": true, + "proto": "any" + } + ], + "propagateProxyEnv": false, + "provision": [ + { + "mode": "dependency", + "script": "#!/bin/sh\nset -eu\ncommand -v docker >/dev/null\ncommand -v python3 >/dev/null\ncommand -v git >/dev/null\n", + "skipDefaultDependencyResolution": true + } + ], + "ssh": { + "forwardAgent": false, + "forwardX11": false, + "forwardX11Trusted": false, + "loadDotSSHPubKeys": false, + "localPort": 0 + }, + "timezone": "UTC", + "upgradePackages": false, + "user": { + "home": "/home/fixture.guest", + "name": "fixture", + "passwordlessSudo": true, + "shell": "/bin/bash", + "uid": 1000 + }, + "vmType": "vz" +} diff --git a/scripts/acceptance/condition-history/guest-proposal.md b/scripts/acceptance/condition-history/guest-proposal.md new file mode 100644 index 0000000..3f08525 --- /dev/null +++ b/scripts/acceptance/condition-history/guest-proposal.md @@ -0,0 +1,213 @@ +# Disposable guest source proposal + +This is a source proposal for #210, not an execution admission. No VM, disk +image, pull or consumer build has been started. Security must review the exact +source and General must admit a private invocation with fresh capacity before +any of the commands below can run. The shared-daemon proposal is superseded. + +The selected backend is the already installed Lima 2.2.1 native Virtualization +framework backend on an Intel macOS 15.8.2 host. It avoids a host installation, +provider account or paid resource. An existing writable shared VM is unsuitable +because it shares the host home and owns unrelated images and caches. The two +Linux hosts have neither an installed VM launcher nor current-user KVM access. +The other native VZ host is arm64, while this acceptance pins amd64 manifests. +Private host names, cache paths, scope and capacity receipts belong in the +execution proposal rather than this repository. + +## Immutable inputs and cost + +| Input | Exact identity | +| --- | --- | +| `limactl` | Lima 2.2.1, 35,175,712 bytes, SHA-256 `216450fc8e3eeb7f50e3030de216c32efeef505b111ed3fb24912a36feebea49` | +| Linux amd64 guest agent gzip | 8,257,812 bytes, SHA-256 `4aa75eacc151cbf2b7aec1b9ebe3886a0fc3268977c108308c802ce267c76959` | +| Uncompressed guest agent | SHA-256 `d799765623d5fa3b52ac30dd437b8f867968aeac594933c8ddc3bf1ac78cb26a` | +| Cached guest image | [colima-core v0.10.4 Ubuntu 24.04 Docker amd64 raw gzip](https://github.com/abiosoft/colima-core/releases/download/v0.10.4/ubuntu-24.04-minimal-cloudimg-amd64-docker.raw.gz), 358,298,593 bytes, SHA-256 `4cd967d2c58aa6971621343d372255b14ffbfe28a185cda0e3c34ac8a1eb15e6` | +| Native tools | macOS 15.8.2 build 24H232 `hdiutil`, `diskutil`, `sysctl`, public `libproc` process information | +| Host guardian | Captured `guest-host.py` Git blob at admitted R, SHA-256 in the R manifest and private authority | +| Guest profile | Complete `guest-profile.json` at R, with its one image path replaced by the task-local path | + +The cache was hashed read-only and matches the public release asset digest and +size. No download was needed. The release source builds Docker 29.5.2, but that +is an expectation to verify inside the fresh guest, not an observed engine +receipt. Python 3, Git, Docker, a block-backed ext4 root and an empty daemon/cache +are required. Missing dependencies cause refusal without package installation. + +The bounded alternative costs **zero provider dollars**, one 4-vCPU / 6-GiB +guest and one fixed **32-GiB** read/write UDIF containing the complete fresh Lima +home, inputs, VM disk, generated SSH keys, temporary files and logs. The guest +disk is **20 GiB**, including its OS, Cargo cache, work, Docker/containerd stores, +images and logs. Its block-device and filesystem ceilings are checked before +registry requests and again before daemon mutations. Space exhaustion fails the +single attempt; there is no enlargement, retry, shared-cache reuse or pruning. +Only two sanitized guest results, each at most 8 MiB, plus bounded private host +authority/receipt files are exported outside the UDIF. Native control output is +held in memory with a combined 2-MiB ceiling and never streamed to unbounded logs. + +Admission requires at least **10 GiB available RAM** and **48 GiB free host disk**, +fresh metrics, and agreement from the owners of any existing shared VM. The +observed host had about 6.5 GiB available, so it did not pass. A coordinated idle +window could release the existing 6-GiB shared VM allocation; this proposal does +not authorize stopping it. The fixed disk is additional to that shared VM, which +is never an acceptance input. No global daemon setting or host package changes. + +The entire invocation starts at S, active work ends at S+1740 seconds, and all +cleanup ends by S+1800. Setup consumes the same window and must finish within +300 seconds. Host guardian timers use monotonic time once the externally captured +absolute cutoffs are bound. The existing guest Cell keeps its 600-second bound, +two Cargo jobs, sequential build/runtime phases and per-role caps. + +## Launch and custody stages + +`guest-host.py plan --plan-input PRIVATE_JSON` is inert. Its closed template check +produces the exact native argument vectors, environment, rendered profile digest +and resource/deadline budgets. It accepts private canonical parent P named +`sr-T`, where T is a fresh 16-hex token. Its Lima home is `P/v/l`, guest name +`sdk-rust-T`, gzip `P/v/a/guest.raw.gz` and fixed image `P/store.dmg`. The short +private path must satisfy Lima's Unix socket limit. No shell interpolation is +used for native controls. + +1. Before the first mutation, the external owner captures R/M, S and both + cutoffs, input/tool hashes, architecture, OS, native Python version, free space + and current capacity. Write an exclusive 0600 setup intent outside the fresh P, + schema `durable-workflow.condition-host-setup/v1`, with exactly `INTENT_KEYS` + from the captured source. It binds the guardian SHA, R/M, host boot UUID, + token, S/cutoffs, scope, launcher/agent and already cached public image paths. + Capture its digest externally. Start the captured guardian in a detached new + session with native Python `-I -S -c`, mode `prepare`, `--record INTENT`, + `--record-sha256 INTENT_SHA` and `--profile CAPTURED_PROFILE`. No mutable + checkout executable is loaded. This resident process is the setup watchdog; + it owns the first native disk mutation and remains the guardian afterward. +2. The watchdog verifies the closed intent, pinned template and all three public + input hashes, native Intel architecture, exact OS and fresh RAM/disk gates. + Insufficient capacity refuses before even creating P. It creates P exclusively + at 0700 and writes `setup-ready.json` with its kernel epoch and capacity. + Only its plan's `hdiutil create -sectors 67108864 -type UDIF -layout + GPTSPUD -fs APFS -nospotlight` and `attach -nobrowse -owners on -mountpoint P/v + -plist` commands run. Native command output has the hard in-memory ceiling; + create/attach are each additionally bounded by the remaining 300-second setup + cutoff. Setup temporary files stay in P until the fixed filesystem is mounted, + then all VM/input/cache/tmp/log writes use that filesystem. Capture image + device/inode, the whole `/dev/diskN`, APFS volume + UUID and the 32-GiB whole-device readback. Refuse an existing image, attachment, + mount, Lima home or instance. Copy the already verified public gzip into the + store without networking, reopen/hash the destination through a no-follow + single-link owner file descriptor, and bind its device/inode/size/SHA. Recheck + destination identity and bytes at consumption. Write the + rendered JSON-as-YAML profile exclusively. No private host source or credential + tree is copied. The installed public guest agent remains a pinned read-only + input. Local image input and disabled containerd downloads avoid Lima's global + downloader cache. Native readbacks must identify exactly one 32-GiB device, + one matching image attachment and an APFS volume at the exact mount point. +3. The resident watchdog writes an exclusive 0600 authority record with schema + `durable-workflow.condition-host/v1`. Its exact fields are `AUTHORITY_KEYS` in + `guest-host.py`: R/M, guardian digest, token, host boot UUID, S/cutoffs, private + scope and launcher/agent paths, image device/inode, whole device, volume UUID + rendered profile/digest, copied image identity and uncompressed guest-agent + digest. Its SHA-256 is captured outside the store. +4. The same process verifies its new record and becomes the active guardian + without a process or custody gap. Standalone `guard` mode can only consume an + already admitted complete authority record. The guardian verifies + launcher/agent digests, host boot, authority, image inode/geometry, attachment + and profile. Its exclusive `host-ready.json` binds authority/source and the + guardian's kernel process epoch. Only after independent readiness verification + may the external owner write an exclusive private `launch.json` with exactly + `record_sha256`, `manifest_sha256` and `action: "create-and-start"`. This request + is part of the separately admitted invocation. The resident guardian owns + both create and start. It revalidates the copied image and selected agent, + then passes the captured profile bytes on stdin to `limactl create --name NAME + -`; no mutable YAML pathname supplies executable settings. It records and + checks `limactl list --json --all-fields NAME` before start, including image, + resource, network, mount, SSH, forwarding, environment and provision settings. + Immediately before its own start call it repeats the input/config checks. + Setup/template file edits cannot replace the captured create input. +5. Lima receives a fresh `LIMA_HOME`, a minimal environment, and no SSH agent, + Docker context, proxy or cloud environment. Its host HOME remains the actual + account home, with host SSH-key loading disabled. The complete profile has no + base template, host mount, copied CA, copied host file, package install, service + forward or nested virtualization. VZ NAT permits public package/image fetches. + Only Lima's generated guest-control SSH connection is permitted. It does not + forward host credentials. `hostResolver` and proxy propagation are disabled. +6. The guardian captures the task instance's `ha.pid` only after binding its + scoped hostagent command and native kernel process epoch. Public + `proc_pidinfo(PROC_PIDTBSDINFO)` provides start seconds **and microseconds**, + UID, PID and group; consistent double reads plus host boot UUID are required. + There is no `ps lstart` signal authority. Revalidate immediately before scoped + `limactl stop --force`. The guardian never sends a numeric host kill signal to + a recovered PID. Missing, changed or foreign authority is a failed result + requiring the external owner, not permission to stop another instance. + The supported Lima 2.2.1 selection is causal: the canonical launcher resolves + `/share/lima/lima-guestagent.Linux-x86_64.gz` through + `usrlocal.GuestAgentBinary`, and `instance.Start` passes it as `--guestagent` + to hostagent. The source refuses another correctly hashed path, and the + independent hostagent command readback must name this exact selected path. + Primary source: [agent selection](https://github.com/lima-vm/lima/blob/v2.2.1/pkg/usrlocal/usrlocal.go) + and [start argument construction](https://github.com/lima-vm/lima/blob/v2.2.1/pkg/instance/start.go). +7. After start, the guardian repeats effective configuration readback and runs + a captured read-only probe over Lima's generated SSH control connection. In + the disposable guest only, passwordless sudo lets the probe hash the unique + running guest agent's `/proc/PID/exe`, with consistent Linux boot/start ticks. + That kernel executable hash must match the pinned uncompressed agent above. + A path declaration alone cannot authorize the receipt. Exclusive + `vm-started.json` binds the actual guest boot, agent hash, hostagent epoch and + effective configuration; no SDK consumer is released before this receipt. + These probes are source under review and have not run in a guest. + Independently obtain B over the generated control path and match that receipt. + Transfer a clean public R checkout with the immutable H tag and invoke + the README's captured entry with that B and the same S/cutoffs. No host source + mount or credential is used. The guest requires an empty daemon/build cache + and ext4 root/work/Docker data on its fixed block device before pulling. + Existing Linux boot/start-tick recovery and full internal-network readbacks + are preserved. Fetch is public/anonymous; credential tests use synthetic values. +8. After guest owner and recovery exit, independently run the captured guest + audit, require the sanitized summary and its exact SHA, and export only the + two bounded JSON results to P. An external exclusive `exports.json` binds the + host record digest, independently observed `guest_boot_id` and both file + digests. An exclusive `finish.json` binds the host record and exact exports + digest. A missing, invalid or changed handoff cannot produce acceptance. +9. At a valid finish request or the unchanged active cutoff, the guardian stops + only the bound scoped instance, confirms the hostagent absent, deletes the + scoped instance, rechecks the enclosing image, detaches its bound whole device, + confirms the mount absent and unlinks only the original image inode. Images, + caches, work, logs and ephemeral keys disappear together. Cleanup never prunes + a shared daemon. It writes exclusive `host-cleanup.json` outside the disk. +10. After guardian exit, run the same captured source in independent `audit` + mode. Require image/mount/instance absence, guardian and VM hostagent absence, + exact host custody/source receipts, R/M/B agreement, guest audit and summary + digest agreement. Its exclusive `host-audit.json` may then record pass. A + guest pass alone remains `pending_external_guest_teardown`. + +Failure at any stage is terminal for this attempt. The resident setup watchdog +handles partial create/attach/copy failures within the unchanged cleanup cutoff. +After a lost attach response, it attempts a read-only recovery of the complete +attachment identity before authorizing detach. A failed create or uncertain +identity retains the task image explicitly; `setup-failure.json` records the +captured inode/device/volume authority it actually obtained. It never guesses +ownership or declares absence after an uncertain native result. The guardian +attempts cleanup after malformed exports or failed process observation. If a +device, inode, PID epoch or mount changed, it refuses destructive guesses and +records retained custody for the external owner, who must resolve it before any +capacity release. Exhausted time, a failed detach or a surviving process/store +is failure, even when product checks passed. A hard host reboot is an external +recovery case, not an unattended-success claim. + +## Remaining review gates + +The packet requires both separate execution admission and the exact private +launch request before it can create/start a guest. Setup watchdog, input +consumption and teardown source are included in the same sealed guardian. Read-only native +inspection confirmed the supported `diskutil` property names and Darwin epoch +structure. Native CLI help also confirms stdin create and effective Config +readback support; a read-only public-agent probe confirms executable-relative +selection and its uncompressed digest. Fake native controls bind +physical/synthesized devices, APFS geometry and mount/UUID authority. Actual held +source countercontrols expose destination corruption, post-ready YAML +edit/restore and an unrelated hashed agent path; candidate controls refuse them. +No new attachment, VM or guest-kernel readback is claimed. +Before execution, supply the exact private intent/command/handoff and fresh +sufficient capacity. The guest dependency assertions and all effective native +readbacks must pass on the sole admitted attempt; a missing tool or unexpected +schema fails rather than authorizing installation or a fallback. Security reviews +the full executable/storage packet; +General admits its exact R/M/command/window. Any admitted source change requires +new exact review. The old missing #191 raw capture and PHP-only #192 producer +provenance remain explicit; this guest does not alter either acceptance boundary. diff --git a/scripts/acceptance/condition-history/recover.py b/scripts/acceptance/condition-history/recover.py new file mode 100644 index 0000000..baf07a0 --- /dev/null +++ b/scripts/acceptance/condition-history/recover.py @@ -0,0 +1,446 @@ +"""Captured, stdlib-only cleanup and independent final audit. No checkout import.""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import signal +import stat +import subprocess +import sys +import time + +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +PREFIX = 'sdk-rust-condition' +CELL = 'condition-history' +RUN = 'dev.durable-workflow.acceptance.run' +OWNER = 'dev.durable-workflow.acceptance.owner' +SUBJECT = 'dev.durable-workflow.acceptance.subject' +ROLE = 'dev.durable-workflow.acceptance.role' +RECORD_KEYS = {'schema', 'owner', 'token', 'cells', 'runner_commit', 'package_commit', 'manifest_sha256', + 'recovery_sha256', 'started_unix', 'active_unix', 'cleanup_unix', 'work_dir', + 'docker_endpoint', 'uid', 'gid', 'owner_process', 'storage'} +STORAGE_KEYS = {'schema', 'guest_boot_id', 'device', 'block_bytes', 'filesystem_bytes', 'docker_root', 'daemon_id'} +GUEST_DISK_BYTES = 20 * 1024**3 +ROLES = ('fetch', 'build', 'mysql', 'redis', 'bootstrap', 'runtime', 'queue', 'consumer') +PUBLIC_CONFIG = {'auths': {'https://index.docker.io/v1/': {}, 'ghcr.io': {}}} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def read(path, mode=0o600, limit=8 * 1024 * 1024): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 + and stat.S_IMODE(before.st_mode) == mode and before.st_uid == os.getuid(), 'unsafe authority/artifact file') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + info = os.fstat(stream.fileno()) + require((info.st_dev, info.st_ino) == (before.st_dev, before.st_ino) and info.st_nlink == 1, 'authority file replaced') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and after.st_nlink == 1 + and (info.st_size, info.st_mtime_ns, info.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), 'authority file changed') + return data + + +def exclusive(path, data): + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), 0o600) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + info = os.fstat(stream.fileno()) + require(info.st_nlink == 1 and stat.S_ISREG(info.st_mode) and info.st_size == len(data), 'unsafe final handoff') + require(read(path) == data, 'handoff changed') + return digest(data) + + +def process_identity(pid): + require(sys.platform == 'linux' and type(pid) is int and pid > 1, 'Linux proc process authority required') + boot = boot_identity() + try: + first = process_stat(pid) + second = process_stat(pid) + except FileNotFoundError: + return None + require(first == second and boot == boot_identity(), 'process epoch changed during observation') + return dict(first, boot_id=boot) + + +def boot_identity(): + value = pathlib.Path('/proc/sys/kernel/random/boot_id').read_text().strip() + require(re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value), 'invalid Linux boot authority') + return value + + +def process_stat(pid): + fd = os.open('/proc/%d/stat' % pid, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + raw = stream.read(4097) + require(len(raw) <= 4096 and raw.startswith(('%d (' % pid).encode()), 'invalid proc process authority') + # comm may contain spaces or parentheses. Fields after its final ')' start + # at state (field 3); starttime is field 22 in the kernel's proc ABI. + head, separator, tail = raw.rpartition(b') ') + fields = tail.split() + require(separator and head and len(fields) >= 20, 'incomplete proc process authority') + return {'pid': pid, 'pgid': int(fields[2]), 'sid': int(fields[3]), 'start_ticks': int(fields[19])} + + +def validate_process(process, label): + require(set(process) == {'pid', 'pgid', 'sid', 'boot_id', 'start_ticks'} + and all(type(process[k]) is int for k in ('pid', 'pgid', 'sid', 'start_ticks')) + and process['pid'] > 1 and process['pid'] == process['pgid'] == process['sid'] + and process['start_ticks'] > 0 + and isinstance(process['boot_id'], str) + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', process['boot_id']), label) + + +def filesystem_storage(work_dir, docker_root, expected_boot): + require(sys.platform == 'linux' and boot_identity() == expected_boot, 'wrong disposable guest boot identity') + root = pathlib.Path('/') + device = root.stat().st_dev + require(os.major(device) != 0, 'block-backed disposable guest root required') + for name in (pathlib.Path(work_dir).parent, pathlib.Path(docker_root)): + require(name.is_absolute() and name.resolve(strict=True) == name and name.stat().st_dev == device, + 'work/daemon storage escapes guest root device') + block_path = pathlib.Path('/sys/dev/block/%d:%d/size' % (os.major(device), os.minor(device))) + block_bytes = int(block_path.read_text().strip()) * 512 + fs = os.statvfs(root) + filesystem_bytes = fs.f_blocks * fs.f_frsize + require(0 < filesystem_bytes <= block_bytes <= GUEST_DISK_BYTES, 'guest storage exceeds hard block ceiling') + mounts = pathlib.Path('/proc/self/mountinfo').read_text().splitlines() + root_rows = [] + for line in mounts: + before, separator, after = line.partition(' - ') + require(separator, 'invalid guest mount authority') + fields, tail = before.split(), after.split() + require(len(fields) >= 6 and len(tail) >= 3, 'invalid guest mount authority') + require(tail[0] not in ('9p', 'virtiofs', 'nfs', 'nfs4', 'cifs', 'fuse.sshfs'), 'host/network filesystem share refused') + if fields[4] == '/': + root_rows.append((fields, tail)) + require(len(root_rows) == 1 and root_rows[0][1][0] == 'ext4' + and root_rows[0][0][2] == '%d:%d' % (os.major(device), os.minor(device)), 'wrong guest root filesystem') + return {'guest_boot_id': expected_boot, 'device': '%d:%d' % (os.major(device), os.minor(device)), + 'block_bytes': block_bytes, 'filesystem_bytes': filesystem_bytes, 'docker_root': docker_root} + + +def validate_storage(value, work_dir, observe=True): + require(set(value) == STORAGE_KEYS and value['schema'] == 'durable-workflow.condition-guest-storage/v1' + and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', value['guest_boot_id']) + and re.fullmatch('[0-9]+:[0-9]+', value['device']) + and type(value['block_bytes']) is int and type(value['filesystem_bytes']) is int + and 0 < value['filesystem_bytes'] <= value['block_bytes'] <= GUEST_DISK_BYTES + and isinstance(value['docker_root'], str) and value['docker_root'].startswith('/') + and isinstance(value['daemon_id'], str) and re.fullmatch('[A-Za-z0-9:-]{12,128}', value['daemon_id']), + 'wrong bounded guest storage authority') + if observe: + current = filesystem_storage(work_dir, value['docker_root'], value['guest_boot_id']) + require(all(value[k] == v for k, v in current.items()), 'guest storage authority changed') + + +def storage_preflight(args, command): + info = json.loads(command(['docker', 'info', '--format', '{{json .}}']).stdout) + frame = filesystem_storage(args.work_dir, info['DockerRootDir'], args.guest_boot_id) + value = dict(frame, schema='durable-workflow.condition-guest-storage/v1', daemon_id=info['ID']) + validate_storage(value, args.work_dir) + # The public base image contains tools only. Nothing in this daemon may be + # inherited from a previous SDK experiment or another owner. + for line in (['docker', 'ps', '-aq'], ['docker', 'image', 'ls', '-q', '--no-trunc'], ['docker', 'volume', 'ls', '-q']): + require(not command(line).stdout.split(), 'disposable guest daemon is not empty') + rows = [json.loads(line) for line in command(['docker', 'system', 'df', '--format', '{{json .}}']).stdout.splitlines() if line] + require(len(rows) == 4 and {row['Type'] for row in rows} == {'Images', 'Containers', 'Local Volumes', 'Build Cache'} + and all((type(row['TotalCount']) is int and row['TotalCount'] == 0) + or (type(row['TotalCount']) is str and row['TotalCount'] == '0') for row in rows), + 'guest image/cache baseline is not empty') + return value + + +def group_members(pgid): + result = subprocess.run(['ps', '-axo', 'pid=,pgid='], capture_output=True, text=True, check=True, timeout=5) + return [int(fields[0]) for line in result.stdout.splitlines() + if len(fields := line.split()) == 2 and int(fields[1]) == pgid] + + +def make_record(args, manifest_sha, recovery_sha): + owner = process_identity(os.getpid()) + require(owner and owner['pid'] == owner['pgid'] == owner['sid'], 'isolated owner session required') + root = pathlib.Path(args.work_dir) + require(root.is_absolute() and root.resolve() == root, 'canonical work root required') + return {'schema': 'durable-workflow.condition-run/v4', 'owner': PREFIX, 'token': args.run_token, + 'cells': [CELL], 'runner_commit': args.runner_commit, 'package_commit': H, + 'manifest_sha256': manifest_sha, 'recovery_sha256': recovery_sha, + 'started_unix': args.started_unix, 'active_unix': args.active_unix, + 'cleanup_unix': args.cleanup_unix, 'work_dir': str(root), + 'docker_endpoint': args.docker_endpoint, 'uid': os.getuid(), 'gid': os.getgid(), + 'owner_process': owner, 'storage': args.guest_storage} + + +def validate_record(root, expected_record_sha, runner, manifest_sha, recovery_sha, token): + root = pathlib.Path(root) + require(root.is_absolute() and root.resolve() == root and not root.is_symlink(), 'redirected work root') + info = root.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and stat.S_IMODE(info.st_mode) == 0o700, 'wrong work root authority') + raw = read(root / 'run.json') + require(digest(raw) == expected_record_sha, 'run record digest mismatch') + record = json.loads(raw) + require(set(record) == RECORD_KEYS and record['schema'] == 'durable-workflow.condition-run/v4', 'wrong run record schema') + require(record['owner'] == PREFIX and record['token'] == token and re.fullmatch('[0-9a-f]{16}', token) + and record['cells'] == [CELL], 'wrong recovery ownership') + require(record['runner_commit'] == runner and re.fullmatch('[0-9a-f]{40}', runner) + and record['package_commit'] == H and record['manifest_sha256'] == manifest_sha + and record['recovery_sha256'] == recovery_sha, 'wrong recovery source subject') + require(record['work_dir'] == str(root) and record['uid'] == os.getuid() and record['gid'] == os.getgid() + and record['docker_endpoint'].startswith('unix:///'), 'wrong recovery placement') + require(all(type(record[k]) is int for k in ('started_unix', 'active_unix', 'cleanup_unix')) + and record['cleanup_unix'] == record['started_unix'] + 1800 + and record['active_unix'] == record['cleanup_unix'] - 60, 'wrong recovery deadlines') + process = record['owner_process'] + validate_process(process, 'wrong captured owner process') + validate_storage(record['storage'], root, observe=False) + require(process['boot_id'] == record['storage']['guest_boot_id'], 'owner/storage guest identity mismatch') + snapshot = root / 'snapshot' + require(not snapshot.is_symlink() and stat.S_ISDIR(snapshot.lstat().st_mode), 'unsafe recovery snapshot') + raw_manifest = read(snapshot / 'manifest.json', mode=0o400) + require(digest(raw_manifest) == manifest_sha, 'recovery manifest changed') + value = json.loads(raw_manifest) + require(set(value) == {'schema', 'runner_commit', 'package_commit', 'files'} + and value['schema'] == 'durable-workflow.condition-subject/v1' + and value['runner_commit'] == runner and value['package_commit'] == H, 'wrong sealed manifest authority') + expected = value['files'] + require(expected['scripts/acceptance/condition-history/recover.py']['sha256'] == recovery_sha, 'recovery source drift') + seen = set() + allowed_dirs = {p.as_posix() for name in expected for p in pathlib.PurePosixPath(name).parents if str(p) != '.'} + def unreadable(error): + raise SystemExit('unreadable recovery snapshot') + for parent, dirs, files in os.walk(snapshot, followlinks=False, onerror=unreadable): + for name in dirs: + path = pathlib.Path(parent) / name + require(not path.is_symlink() and stat.S_ISDIR(path.lstat().st_mode) + and path.relative_to(snapshot).as_posix() in allowed_dirs, 'redirected or extra snapshot directory') + for name in files: + path = pathlib.Path(parent) / name + relative = path.relative_to(snapshot).as_posix() + seen.add(relative) + if relative == 'manifest.json': + continue + require(relative in expected, 'unexpected recovery source member') + data = read(path, mode=0o400) + require(len(data) == expected[relative]['bytes'] and digest(data) == expected[relative]['sha256'], 'sealed recovery source changed') + require(seen == set(expected) | {'manifest.json'}, 'missing recovery source member') + return record + + +def docker(record, command, check=True): + validate_storage(record['storage'], record['work_dir']) + root = pathlib.Path(record['work_dir']) + require(json.loads(read(root / 'recovery-docker/config.json')) == PUBLIC_CONFIG, 'anonymous Docker configuration changed') + env = {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(root / 'public-home'), + 'DOCKER_CONFIG': str(root / 'recovery-docker'), 'DOCKER_HOST': record['docker_endpoint']} + remaining = record['cleanup_unix'] - time.time() + require(remaining > 0, 'cleanup deadline exhausted') + result = subprocess.run(['docker', *command], capture_output=True, text=True, timeout=min(5, remaining), env=env) + require(not check or result.returncode == 0, 'recovery Docker query/removal failed') + return result + + +def object_names(record): + base = PREFIX + '-' + record['token'] + task = base + '-' + CELL + # The only names this invocation can own, including partial build creates. + return task, {r: base + '-' + r if r in ('fetch', 'build') else task + '-' + r for r in ROLES} + + +def objects(record, kind): + command = {'container': ['ps', '-aq'], 'network': ['network', 'ls', '-q'], 'volume': ['volume', 'ls', '-q']}[kind] + return docker(record, command + ['--filter', 'label=' + OWNER + '=' + PREFIX, + '--filter', 'label=' + RUN + '=' + record['token']]).stdout.split() + + +def owned(record, kind, identifier): + labels = '.Config.Labels' if kind == 'container' else '.Labels' + template = '{"labels":{{json %s}},"name":{{json .Name}},"id":{{json .Id}}}' % labels + result = docker(record, [kind, 'inspect', '--format', template, identifier], check=False) + if result.returncode: + require(any(x in result.stderr.lower() for x in ('no such object', 'no such container', 'no such network')), 'uncertain object ownership') + return None + frame = json.loads(result.stdout) + labels, name = frame['labels'], frame['name'].removeprefix('/') + task, names_ = object_names(record) + role = labels.get(ROLE) + require(labels.get(OWNER) == PREFIX and labels.get(RUN) == record['token'] + and labels.get(SUBJECT) == record['manifest_sha256'], 'foreign object authority') + require((kind == 'network' and role == 'network' and name == task) + or (kind == 'container' and role in names_ and name == names_[role]), 'unknown owned object name') + require(isinstance(frame['id'], str) and re.fullmatch('[0-9a-f]{64}', frame['id']), 'invalid immutable object ID') + return frame['id'] + + +def cleanup(record): + errors = [] + for kind in ('container', 'network'): + try: + identifiers = objects(record, kind) + except BaseException: + errors.append('query-' + kind) + continue + for identifier in identifiers: + try: + object_id = owned(record, kind, identifier) + if object_id: + docker(record, ['rm', '-fv', object_id] if kind == 'container' else ['network', 'rm', object_id]) + except BaseException: + errors.append('remove-' + kind) + remaining = {} + for kind in ('container', 'network', 'volume'): + try: + remaining[kind] = objects(record, kind) + except BaseException: + remaining[kind] = None + errors.append('verify-' + kind) + return {'pass': not errors and all(v == [] for v in remaining.values()), 'errors': errors, 'remaining': remaining} + + +def purge(root): + import shutil + parent = pathlib.Path(root) / 'cells' + require(not parent.is_symlink(), 'redirected Cell parent') + target = parent / CELL + if target.is_symlink(): + target.unlink() + elif target.exists(): + shutil.rmtree(target) + require(not os.path.lexists(target), 'Cell purge survivor') + + +def stop_owner(record): + expected = record['owner_process'] + current = process_identity(expected['pid']) + if current is not None: + require(current == expected, 'owner PID epoch changed; refusing foreign process') + signal_owner(expected, signal.SIGTERM) + until = time.monotonic() + 5 + while group_members(expected['pgid']) and time.monotonic() < until: + time.sleep(0.1) + if group_members(expected['pgid']): + signal_owner(expected, signal.SIGKILL) + require(not group_members(expected['pgid']), 'owner group survived or cannot be attributed') + + +def signal_owner(expected, signum): + require(process_identity(expected['pid']) == expected, 'owner process epoch changed before signal') + os.killpg(expected['pgid'], signum) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('recover', 'audit')) + for key in ('work-dir', 'record-sha256', 'manifest-sha256', 'runner-commit', 'package-commit', 'run-token'): + parser.add_argument('--' + key, required=True) + parser.add_argument('--summary-sha256') + parser.add_argument('--at', type=int) + args = parser.parse_args(argv) + require(args.package_commit == H, 'wrong recovery package source') + original = sys.orig_argv + require('-I' in original and '-S' in original and '-c' in original, 'captured recovery program required') + source_sha = digest(original[original.index('-c') + 1].encode()) + record = validate_record(args.work_dir, args.record_sha256, args.runner_commit, + args.manifest_sha256, source_sha, args.run_token) + config_dir = pathlib.Path(args.work_dir) / 'recovery-docker' + if not config_dir.exists(): + config_dir.mkdir(mode=0o700) + exclusive(config_dir / 'config.json', canonical(PUBLIC_CONFIG)) + require(not config_dir.is_symlink() and json.loads(read(config_dir / 'config.json')) == PUBLIC_CONFIG, + 'wrong captured recovery configuration') + if args.mode == 'recover': + if args.at is not None: + exclusive(pathlib.Path(args.work_dir) / 'recovery-ready.json', canonical({ + 'record_sha256': args.record_sha256, 'manifest_sha256': args.manifest_sha256, + 'recovery_sha256': source_sha, 'process': process_identity(os.getpid())})) + root_fd = os.open(args.work_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + errors = [] + try: + if args.at is not None: + require(args.at == record['active_unix'], 'wrong recovery wake deadline') + time.sleep(max(0, args.at - time.time())) + # A later file drift fails evidence but cannot replace the captured + # authority/code or prevent its already-authorized exact cleanup. + try: + validate_record(args.work_dir, args.record_sha256, args.runner_commit, + args.manifest_sha256, source_sha, args.run_token) + except BaseException: + errors.append('source-or-record-drift') + try: + stop_owner(record) + except BaseException: + errors.append('owner-process-uncertain') + result = cleanup(record) + try: + import shutil + parent = os.open('cells', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_fd) + try: + info = os.stat(CELL, dir_fd=parent, follow_symlinks=False) + if stat.S_ISLNK(info.st_mode):os.unlink(CELL, dir_fd=parent) + else:shutil.rmtree(CELL, dir_fd=parent) + except FileNotFoundError: + pass + finally: + os.close(parent) + except FileNotFoundError: + pass + except BaseException: + errors.append('cell-purge-uncertain') + result['errors'] += errors + result['pass'] = result['pass'] and not errors + exclusive(pathlib.Path(args.work_dir) / 'recovery-audit.json', canonical(result)) + require(result['pass'], 'recovery incomplete') + finally: + os.close(root_fd) + else: + require(args.summary_sha256 and re.fullmatch('[0-9a-f]{64}', args.summary_sha256), 'expected immutable summary digest required') + raw = read(pathlib.Path(args.work_dir) / 'summary.json') + require(digest(raw) == args.summary_sha256, 'summary handoff digest mismatch') + summary = json.loads(raw) + require(summary.get('record_sha256') == args.record_sha256 + and summary.get('runner_commit') == args.runner_commit and summary.get('package_commit') == H + and summary.get('manifest_sha256') == args.manifest_sha256, 'summary subject mismatch') + remaining = {kind: objects(record, kind) for kind in ('container', 'network', 'volume')} + require(all(v == [] for v in remaining.values()), 'independent object survivor') + require(not group_members(record['owner_process']['pgid']), 'independent owner process survivor') + watcher_raw = read(pathlib.Path(args.work_dir) / 'recovery-process.json') + require(digest(watcher_raw) == summary.get('recovery_process_sha256'), 'recovery-process handoff changed') + watcher = json.loads(watcher_raw) + require(set(watcher) == {'source_sha256', 'process'} and watcher['source_sha256'] == source_sha, + 'wrong captured recovery-process receipt') + process = watcher['process'] + validate_process(process, 'wrong recovery-process identity') + require(not group_members(watcher['process']['pgid']), 'independent recovery process survivor') + require(summary.get('checks_outcome') == 'pass' + and summary.get('outcome') == 'pending_independent_audit' + and summary.get('cleanup') == {'pass': True, 'errors': [], + 'remaining': {'container': [], 'network': [], 'volume': []}} + and summary.get('cleanup_errors') == [], 'owner checks or cleanup did not pass') + result = {'outcome': 'pending_external_guest_teardown', 'record_sha256': args.record_sha256, + 'manifest_sha256': args.manifest_sha256, 'summary_sha256': args.summary_sha256, + 'remaining': remaining, 'owner_process_absent': True, 'recovery_process_absent': True, + 'storage': record['storage']} + exclusive(pathlib.Path(args.work_dir) / 'independent-audit.json', canonical(result)) + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/resources.py b/scripts/acceptance/condition-history/resources.py new file mode 100644 index 0000000..8c15164 --- /dev/null +++ b/scripts/acceptance/condition-history/resources.py @@ -0,0 +1,252 @@ +"""Literal Docker role profiles and effective readbacks. Tests supply a fake CLI.""" +import hashlib +import json +import os +import pathlib +import re +import subprocess +import time + +RUN = 'dev.durable-workflow.acceptance.run' +OWNER = 'dev.durable-workflow.acceptance.owner' +SUBJECT = 'dev.durable-workflow.acceptance.subject' +ROLE = 'dev.durable-workflow.acceptance.role' +PREFIX = 'sdk-rust-condition' +CELL = 'condition-history' +NETWORK_OPTIONS = {'com.docker.network.bridge.enable_ip_masquerade': 'false', + 'com.docker.network.bridge.enable_icc': 'true', + 'com.docker.network.bridge.host_binding_ipv4': '127.0.0.1'} +NETWORK_IPAM = {'Driver': 'default', 'Options': {}, + 'Config': [{'Subnet': '172.30.254.0/28', 'Gateway': '172.30.254.1'}]} +PROFILES = { + 'fetch': (2_000_000_000, 3 * 1024**3, 1024, True, ()), + 'build': (2_000_000_000, 3 * 1024**3, 1024, True, ()), + 'mysql': (1_000_000_000, 1280 * 1024**2, 512, False, ('CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID')), + 'redis': (250_000_000, 128 * 1024**2, 256, False, ('CHOWN', 'SETGID', 'SETUID')), + 'bootstrap': (750_000_000, 768 * 1024**2, 256, False, ('DAC_OVERRIDE',)), + 'runtime': (750_000_000, 768 * 1024**2, 256, False, ('CHOWN', 'DAC_OVERRIDE', 'KILL', 'SETGID', 'SETUID')), + 'queue': (500_000_000, 512 * 1024**2, 256, False, ()), + 'consumer': (500_000_000, 768 * 1024**2, 256, True, ()), +} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def names(token): + base = PREFIX + '-' + token + task = base + '-' + CELL + return task, {r: base + '-' + r if r in ('fetch', 'build') else task + '-' + r for r in PROFILES} + + +def environment(args): + root = pathlib.Path(args.work_dir) + return {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(root / 'public-home'), + 'DOCKER_HOST': args.docker_endpoint, 'DOCKER_CONFIG': str(root / 'public-docker')} + + +def call(args, line, timeout=120, check=True, **kwargs): + if hasattr(args, 'storage_guard'): + args.storage_guard() + remaining = int(args.active_unix - time.time()) + require(remaining > 0, 'active deadline exhausted') + result = subprocess.run(line, capture_output=True, timeout=min(timeout, remaining), + env=environment(args), **dict({'text': True}, **kwargs)) + require(not check or result.returncode == 0, 'bounded command failed') + return result + + +def preflight_collisions(args): + """Every query is read-only and precedes work/snapshot/registry creation.""" + task, roles = names(args.run_token) + filters = ['--filter', 'label=' + RUN + '=' + args.run_token, '--filter', 'label=' + OWNER + '=' + PREFIX] + for command in (['docker', 'ps', '-aq'], ['docker', 'network', 'ls', '-q'], ['docker', 'volume', 'ls', '-q']): + require(not call(args, command + filters).stdout.split(), 'preexisting owner/run object') + for kind, candidates in [('container', roles.values()), ('network', [task])]: + for name in candidates: + result = call(args, ['docker', kind, 'inspect', '--format', '{{.Id}}', name], check=False) + if result.returncode == 0: + raise SystemExit('preexisting exact object name') + require(any(s in result.stderr.lower() for s in ('no such object', 'no such container', 'no such network')), + 'collision query is uncertain') + + +def caps(values): + return sorted(v.removeprefix('CAP_').upper() for v in values or []) + + +def env_map(values): + pairs = [v.split('=', 1) for v in values or []] + require(all(len(p) == 2 for p in pairs), 'malformed environment readback') + return dict(pairs) + + +class Docker: + def __init__(self, args, subject, record): + self.args, self.subject, self.record = args, subject, record + self.task, self.names = names(args.run_token) + self.images, self.receipts, self.created = {}, [], [] + + def call(self, line, **kwargs): + self.subject.verify() + return call(self.args, line, **kwargs) + + def labels(self, role): + return {RUN: self.args.run_token, OWNER: PREFIX, SUBJECT: self.subject.sha256, ROLE: role} + + def network_arguments(self): + line = ['docker', 'network', 'create', '--driver', 'bridge', '--scope', 'local', '--internal', + '--ipv4=true', '--ipv6=false', '--ipam-driver', 'default', + '--subnet', NETWORK_IPAM['Config'][0]['Subnet'], '--gateway', NETWORK_IPAM['Config'][0]['Gateway']] + for key, value in NETWORK_OPTIONS.items(): + line += ['--opt', key + '=' + value] + for key, value in self.labels('network').items(): + line += ['--label', key + '=' + value] + return line + [self.task] + + def validate_network(self, network, container_id): + require(re.fullmatch('[0-9a-f]{64}', network['Id']) and network['Name'] == self.task + and network['Driver'] == 'bridge' and network['Scope'] == 'local' + and network['Internal'] is True and network['Attachable'] is False and network['Ingress'] is False + and network['EnableIPv4'] is True and network['EnableIPv6'] is False + and network['ConfigOnly'] is False and network['ConfigFrom'] == {'Network': ''}, 'wrong effective network profile') + require(network['Labels'] == self.labels('network') and network['Options'] == NETWORK_OPTIONS, + 'wrong internal network authority/options') + ipam = dict(network['IPAM'], Options=network['IPAM'].get('Options') or {}) + require(ipam == NETWORK_IPAM, 'wrong effective network IPAM') + allowed = {r['container_id'] for r in self.receipts if r['network'] == self.task} | {container_id} + require(set(network['Containers']) <= allowed and container_id in network['Containers'], 'foreign or missing network endpoint') + return {'id': network['Id'], 'driver': 'bridge', 'scope': 'local', 'internal': True, + 'attachable': False, 'ingress': False, 'ipv4': True, 'ipv6': False, + 'options': NETWORK_OPTIONS, 'ipam': NETWORK_IPAM} + + def image(self, reference): + if reference not in self.images: + data = json.loads(self.call(['docker', 'image', 'inspect', reference]).stdout)[0] + require(data['Os'] == 'linux' and data['Architecture'] == 'amd64', 'wrong image platform') + content = json.dumps({'layers': data['RootFS']['Layers'], 'config': data['Config']}, sort_keys=True).encode() + self.images[reference] = {'raw': data, 'sha256': hashlib.sha256(content).hexdigest()} + return self.images[reference] + + def profile(self, role, image, command, user, credentials, fixture, state=None, env=None, alias=None, deadline=None): + require(role in PROFILES and credentials in ('none', 'auth', 'server', 'both'), 'unknown role/credential profile') + require(deadline is not None and deadline + 10 <= self.args.active_unix, 'role exceeds active deadline') + cpus, memory, pids, readonly, added = PROFILES[role] + metadata = self.image(image)['raw']['Config'] + child = (metadata.get('Entrypoint') or []) + (command or metadata.get('Cmd') or []) + require(child and all(isinstance(v, str) for v in child), 'invalid image entrypoint') + uid, gid = user.split(':') + require(uid.isdecimal() and gid.isdecimal() and (role not in ('fetch', 'build', 'consumer') or int(uid) != 0), 'unsafe role user') + tmpfs = {'/credentials': 'mode=0700,uid=%s,gid=%s,size=64k,noexec,nosuid,nodev' % (uid, gid), + '/tmp': 'mode=1777,size=64m,nosuid,nodev'} + binds = [] + if role in ('fetch', 'build'): + binds.append((str(fixture), '/work', role == 'fetch')) + if role == 'build': + binds += [(str(fixture / '.cargo'), '/work/.cargo', True), + (str(fixture / 'target'), '/work/target', True)] + elif role == 'consumer': + binds += [(str(fixture), '/work', False), (str(fixture / 'target/debug'), '/fixture', False), + (str(state), '/state', True)] + elif role == 'mysql': + tmpfs['/var/lib/mysql'] = 'mode=0750,size=512m,nosuid,nodev,noexec' + elif role == 'redis': + tmpfs['/data'] = 'mode=0750,size=64m,nosuid,nodev,noexec' + if role in ('bootstrap', 'runtime', 'queue'): + tmpfs['/app/bootstrap/cache'] = 'mode=0777,size=8m,noexec,nosuid,nodev' + declared = set((metadata.get('Volumes') or {}).keys()) + require(declared <= set(tmpfs) | {b[1] for b in binds}, 'uncovered image volume') + network = 'bridge' if role == 'fetch' else 'none' if role == 'build' else self.task + program = self.subject.read('scripts/acceptance/condition-history/role-supervisor.sh').decode() + argv = ['-c', program, '--', program, str(deadline), '10', credentials] + child + expected_env = env_map(metadata.get('Env')) + expected_env.update(env or {}) + return {'role': role, 'name': self.names[role], 'image': image, 'user': user, 'credentials': credentials, + 'cpus': cpus, 'memory': memory, 'pids': pids, 'readonly': readonly, 'cap_add': list(added), + 'tmpfs': tmpfs, 'binds': binds, 'network': network, 'alias': alias, 'deadline': deadline, + 'argv': argv, 'env': expected_env, 'explicit_env': env or {}} + + def create_arguments(self, spec): + line = ['docker', 'create', '--init', '--name', spec['name'], '--platform', 'linux/amd64', + '--user', spec['user'], '--cpus', str(spec['cpus'] / 1_000_000_000), + '--memory', str(spec['memory']), '--memory-swap', str(spec['memory']), '--pids-limit', str(spec['pids']), + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--network', spec['network'], + '--entrypoint', '/bin/sh'] + if spec['readonly']: + line += ['--read-only'] + for cap in spec['cap_add']: + line += ['--cap-add', cap] + for key, value in self.labels(spec['role']).items(): + line += ['--label', key + '=' + value] + for destination, options in spec['tmpfs'].items(): + line += ['--tmpfs', destination + ':' + options] + for source, destination, writable in spec['binds']: + line += ['--mount', 'type=bind,src=%s,dst=%s%s' % (source, destination, '' if writable else ',readonly')] + if spec['alias']: + line += ['--network-alias', spec['alias']] + if spec['role'] in ('fetch', 'build'): + line += ['-w', '/work'] + for key, value in spec['explicit_env'].items(): + line += ['-e', key + '=' + value] + return line + [spec['image']] + spec['argv'] + + def validate(self, spec, data): + host, config = data['HostConfig'], data['Config'] + require(data['Name'] == '/' + spec['name'] and re.fullmatch('[0-9a-f]{64}', data['Id']), 'wrong effective name/ID') + require(config['Image'] == spec['image'] and self.image(data['Image'])['sha256'] == self.image(spec['image'])['sha256'], 'wrong effective image') + require(data['State']['Running'] is True and type(data['State']['Pid']) is int and data['State']['Pid'] > 0, 'role not held for readback') + require(config.get('WorkingDir', '') == ('/work' if spec['role'] in ('fetch', 'build') else self.image(spec['image'])['raw']['Config'].get('WorkingDir', '')), 'wrong effective working directory') + require(config['User'] == spec['user'] and config['Entrypoint'] == ['/bin/sh'] + and config['Cmd'] == spec['argv'], 'wrong user or deadline supervisor') + require(host['NanoCpus'] == spec['cpus'] and host['Memory'] == spec['memory'] + and host['MemorySwap'] == spec['memory'] and host['PidsLimit'] == spec['pids'], 'wrong effective resource cap') + require(host['ReadonlyRootfs'] is spec['readonly'] and host['Init'] is True + and host['Privileged'] is False, 'wrong root/init/privilege mode') + require(caps(host['CapDrop']) == ['ALL'] and caps(host['CapAdd']) == caps(spec['cap_add']), 'wrong capability set') + require(sorted(v.removesuffix(':true') for v in host['SecurityOpt']) == ['no-new-privileges'], 'wrong security options') + require(host['NetworkMode'] == spec['network'] and not host.get('PortBindings') and not host.get('PublishAllPorts') + and not host.get('Devices') and not host.get('DeviceRequests') and not host.get('DeviceCgroupRules') + and not host.get('VolumesFrom') and host.get('PidMode', '') == '' + and host.get('IpcMode', 'private') == 'private' and host.get('UTSMode', '') == '', + 'wrong effective network/device sharing') + if spec['network'] == self.task: + network = json.loads(self.call(['docker', 'network', 'inspect', self.task]).stdout)[0] + network_profile = self.validate_network(network, data['Id']) + aliases = set(data['NetworkSettings']['Networks'].get(self.task, {}).get('Aliases') or []) + require(not spec['alias'] or spec['alias'] in aliases, 'missing effective network alias') + require(set(data['NetworkSettings']['Networks']) == {spec['network']}, 'unexpected attached network') + require(all(config['Labels'].get(k) == v for k, v in self.labels(spec['role']).items()), 'wrong effective ownership') + require(env_map(config['Env']) == spec['env'], 'wrong effective environment') + require(host.get('Tmpfs', {}) == spec['tmpfs'], 'wrong tmpfs destinations/options') + actual = [(m['Source'], m['Destination'], m['RW']) for m in data['Mounts'] if m['Type'] == 'bind'] + require(sorted(actual) == sorted(spec['binds']) and all(m['Type'] in ('bind', 'tmpfs') for m in data['Mounts']), 'wrong bind/volume mounts') + require({m['Destination'] for m in data['Mounts'] if m['Type'] == 'tmpfs'} == set(spec['tmpfs']) + and len(data['Mounts']) == len(spec['tmpfs']) + len(spec['binds']), 'missing, duplicate or unexpected mount') + return {'role': spec['role'], 'container_id': data['Id'], 'image': spec['image'], + 'network_profile': network_profile if spec['network'] == self.task else None, + 'image_content_sha256': self.image(spec['image'])['sha256'], 'user': spec['user'], + 'memory': spec['memory'], 'memory_swap': spec['memory'], 'nano_cpus': spec['cpus'], + 'pids': spec['pids'], 'readonly': spec['readonly'], 'cap_drop': ['ALL'], 'cap_add': spec['cap_add'], + 'security_opt': ['no-new-privileges'], 'network': spec['network'], 'deadline_unix': spec['deadline'], + 'supervisor_argv_sha256': hashlib.sha256(json.dumps(spec['argv']).encode()).hexdigest(), + 'mount_destinations': sorted(set(spec['tmpfs']) | {m[1] for m in spec['binds']}), + 'labels': self.labels(spec['role'])} + + def start(self, spec, token='', key=''): + self.subject.verify() + self.created.append(spec['name']) # before a possibly partial daemon create + self.call(self.create_arguments(spec)) + self.call(['docker', 'start', spec['name']]) + data = json.loads(self.call(['docker', 'inspect', spec['name']]).stdout)[0] + receipt = self.validate(spec, data) + self.receipts.append(receipt) + # No image command starts before the effective independent readback. + self.call(['docker', 'exec', '-i', spec['name'], '/bin/sh', '-c', + 'umask 077; IFS= read -r token && IFS= read -r key && ' + '{ [ -z "$token" ] || printf %s "$token" > /credentials/token; } && ' + '{ [ -z "$key" ] || printf %s "$key" > /credentials/server-key; } && ' + ': > /credentials/ready.tmp && mv /credentials/ready.tmp /credentials/ready'], + input=token + '\n' + key + '\n', timeout=20) + return spec['name'] diff --git a/scripts/acceptance/condition-history/role-supervisor.sh b/scripts/acceptance/condition-history/role-supervisor.sh new file mode 100644 index 0000000..f884117 --- /dev/null +++ b/scripts/acceptance/condition-history/role-supervisor.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu +source=$1 +shift +if [ "${1-}" = '--child' ]; then + shift + credentials=$1 + shift + unset DW_AUTH_TOKEN DW_SERVER_KEY APP_KEY + # All roles, including builds/databases, wait for verified effective caps. + while [ ! -f /credentials/ready ]; do sleep 0.1; done + case "$credentials" in + none) ;; + auth|server|both) ;; + *) exit 125;; + esac + if [ "$credentials" = auth ] || [ "$credentials" = both ]; then + DW_AUTH_TOKEN=$(cat /credentials/token) + case "$DW_AUTH_TOKEN" in fixture-*) hex=${DW_AUTH_TOKEN#fixture-};; *) exit 125;; esac + [ "${#hex}" -eq 48 ] || exit 125 + case "$hex" in *[!0-9a-f]*) exit 125;; esac + export DW_AUTH_TOKEN + fi + if [ "$credentials" = server ] || [ "$credentials" = both ]; then + DW_SERVER_KEY=$(cat /credentials/server-key) + case "$DW_SERVER_KEY" in base64:*) key=${DW_SERVER_KEY#base64:};; *) exit 125;; esac + [ "${#key}" -eq 44 ] || exit 125 + case "$key" in *[!a-zA-Z0-9+/=]*) exit 125;; esac + export DW_SERVER_KEY + fi + exec "$@" +fi +deadline=$1 +grace=$2 +shift 2 +case "$deadline:$grace" in *[!0-9:]*|:*|*:) exit 125;; esac +remaining=$((deadline - $(date +%s))) +[ "$remaining" -gt 0 ] || exit 124 +# Re-execute captured inline bytes, never a mutable bind-mounted script path. +exec timeout --signal=TERM --kill-after="${grace}s" "${remaining}s" \ + /bin/sh -c "$source" -- "$source" --child "$@" diff --git a/scripts/acceptance/condition-history/run.py b/scripts/acceptance/condition-history/run.py new file mode 100644 index 0000000..9fc31a4 --- /dev/null +++ b/scripts/acceptance/condition-history/run.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Captured entry point. Invoke these Git-object bytes with Python -I -S -c.""" +import argparse +import pathlib +import re +import subprocess +import sys +import types + +SUBJECT = 'scripts/acceptance/condition-history/subject.py' + + +def main(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--checkout', required=True) + parser.add_argument('--runner-commit', required=True) + parser.add_argument('--package-commit', required=True) + args, _ = parser.parse_known_args() + if args.package_commit != '4b2440a5618ccc1c98960ea2e1e62127042c3590': + raise SystemExit('wrong fixed package source') + if not re.fullmatch('[0-9a-f]{40}', args.runner_commit): + raise SystemExit('invalid runner commit') + checkout = pathlib.Path(args.checkout).resolve(strict=True) + result = subprocess.run(['git', '--no-replace-objects', '-C', str(checkout), 'show', + args.runner_commit + ':' + SUBJECT], check=True, capture_output=True, timeout=20) + module = types.ModuleType('sealed_subject') + module.__file__ = 'git:' + args.runner_commit + ':' + SUBJECT + sys.modules['sealed_subject'] = module + exec(compile(result.stdout, module.__file__, 'exec'), module.__dict__) + module.bootstrap() + + +if __name__ == '__main__': + main() diff --git a/scripts/acceptance/condition-history/subject.py b/scripts/acceptance/condition-history/subject.py new file mode 100644 index 0000000..754bb76 --- /dev/null +++ b/scripts/acceptance/condition-history/subject.py @@ -0,0 +1,244 @@ +"""Capture one closed Git-object subject; never import executable checkout files.""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import stat +import subprocess +import sys +import time +import types + +PREFIX = 'scripts/acceptance/condition-history/' +H = '4b2440a5618ccc1c98960ea2e1e62127042c3590' +RUNTIME_FILES = tuple(PREFIX + p for p in ( + 'run.py', 'subject.py', 'engine.py', 'recover.py', 'resources.py', + 'role-supervisor.sh', 'check_history.py', 'guest-host.py', 'guest-profile.json', + 'fixture/Cargo.toml', 'fixture/tests/conditions.rs')) + ( + 'scripts/acceptance/installed-native/run.py', + 'scripts/acceptance/installed-native/cell-supervisor.sh') +PACKET_FILES = {p for p in RUNTIME_FILES if p.startswith(PREFIX)} | { + PREFIX + p for p in ('README.md', 'guest-proposal.md', 'test_guest_host.py', + 'test_packet.py', 'test_subject.py', 'test_resources.py', 'test_audit.py')} +MODULES = {'engine': PREFIX + 'engine.py', 'resources': PREFIX + 'resources.py', + 'native': 'scripts/acceptance/installed-native/run.py', 'comparator': PREFIX + 'check_history.py', + 'recovery': PREFIX + 'recover.py'} + + +def require(ok, label): + if not ok: + raise SystemExit(label) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode() + + +def git(checkout, *args, binary=False): + result = subprocess.run(['git', '--no-replace-objects', '--literal-pathspecs', '-C', str(checkout), *args], + check=True, capture_output=True, timeout=20) + return result.stdout if binary else result.stdout.decode().strip() + + +def manifest(checkout, revision): + require(re.fullmatch('[0-9a-f]{40}', revision), 'invalid runner commit') + listing = git(checkout, 'ls-tree', '-r', '-z', revision, '--', PREFIX, binary=True) + found = {entry.split(b'\t', 1)[1].decode() for entry in listing.split(b'\0') if entry} + require(found == PACKET_FILES, 'extra or missing packet Git member') + files, captured = {}, {} + for path in RUNTIME_FILES: + entry = git(checkout, 'ls-tree', revision, '--', path).split() + require(len(entry) == 4 and entry[0] in ('100644', '100755') and entry[1] == 'blob' + and entry[3] == path, 'nonregular Git member') + data = git(checkout, 'cat-file', 'blob', revision + ':' + path, binary=True) + require(len(data) <= 2 * 1024 * 1024, 'oversize sealed member') + captured[path] = data + files[path] = {'git_blob': entry[2], 'sha256': digest(data), 'bytes': len(data), 'mode': entry[0]} + value = {'schema': 'durable-workflow.condition-subject/v1', 'runner_commit': revision, + 'package_commit': H, 'files': files} + return value, captured + + +def regular(path, limit=2 * 1024 * 1024, mode=None): + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_uid == os.getuid(), 'nonregular, linked or foreign source/record') + if mode is not None: + require(stat.S_IMODE(before.st_mode) == mode, 'wrong private file mode') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino) + and stat.S_ISREG(opened.st_mode) and opened.st_nlink == 1, 'source/record changed on open') + data = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(data) <= limit and (opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns) + and after.st_nlink == 1, 'source/record changed on read') + return data + + +def exclusive(path, data, mode=0o600): + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode) + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size == len(data), 'unsafe exclusive handoff') + require(regular(path, max(len(data), 1), mode) == data, 'exclusive handoff changed') + return digest(data) + + +class Subject: + def __init__(self, value, captured, root): + self.value, self.captured, self.root = value, dict(captured), pathlib.Path(root) + self.sha256 = digest(canonical(value)) + self.loaded = {} + + def materialize(self): + self.root.mkdir(mode=0o700) + for path, data in self.captured.items(): + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + exclusive(target, data, 0o400) + exclusive(self.root / 'manifest.json', canonical(self.value), 0o400) + for parent, directories, _ in os.walk(self.root, topdown=False): + for directory in directories: + (pathlib.Path(parent) / directory).chmod(0o500) + self.root.chmod(0o500) + self.verify() + + def verify(self): + require(not self.root.is_symlink() and stat.S_ISDIR(self.root.lstat().st_mode), 'unsafe snapshot root') + found = set() + allowed_dirs = {p.as_posix() for name in self.captured for p in pathlib.PurePosixPath(name).parents if str(p) != '.'} + def unreadable(error): + raise SystemExit('unreadable snapshot directory') + for parent, directories, files in os.walk(self.root, followlinks=False, onerror=unreadable): + for name in directories: + path = pathlib.Path(parent) / name + require(not path.is_symlink() and stat.S_ISDIR(path.lstat().st_mode) + and path.relative_to(self.root).as_posix() in allowed_dirs, 'unsafe or extra snapshot directory') + for name in files: + path = pathlib.Path(parent) / name + relative = path.relative_to(self.root).as_posix() + found.add(relative) + data = regular(path, mode=0o400) + expected = canonical(self.value) if relative == 'manifest.json' else self.captured.get(relative) + require(expected is not None and data == expected, 'snapshot member changed or unexpected') + require(found == set(self.captured) | {'manifest.json'}, 'missing snapshot member') + + def read(self, path): + self.verify() + require(path in self.captured, 'member outside sealed subject') + return self.captured[path] + + def module(self, name): + require(name in MODULES, 'module outside sealed import set') + self.verify() + if name not in self.loaded: + path = MODULES[name] + module = types.ModuleType('sealed_' + name) + module.__file__ = str(self.root / path) + sys.modules[module.__name__] = module + exec(compile(self.captured[path], module.__file__, 'exec'), module.__dict__) + self.loaded[name] = module + return self.loaded[name] + + def export(self, prefix, destination): + self.verify() + for name, data in self.captured.items(): + if name.startswith(prefix): + target = destination / name.removeprefix(prefix) + target.parent.mkdir(parents=True, exist_ok=True) + exclusive(target, data) + + +def arguments(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=('run', 'recover', 'audit', 'manifest')) + for key in ('checkout', 'runner-commit', 'package-commit'): + parser.add_argument('--' + key, required=True) + parser.add_argument('--manifest-sha256') + parser.add_argument('--work-dir') + parser.add_argument('--run-token') + parser.add_argument('--docker-endpoint') + parser.add_argument('--guest-boot-id') + for key in ('started-unix', 'active-unix', 'cleanup-unix'): + parser.add_argument('--' + key, type=int) + parser.add_argument('--record-sha256') + parser.add_argument('--summary-sha256') + return parser.parse_args(argv) + + +def validate(args): + require(args.package_commit == H, 'wrong fixed package source') + require(git(args.checkout, 'rev-parse', 'HEAD') == args.runner_commit, 'wrong runner head') + require(not git(args.checkout, 'status', '--porcelain', '--untracked-files=all'), 'runner checkout is not clean') + require(git(args.checkout, 'rev-parse', 'refs/tags/4.12.0^{commit}') == H, 'wrong immutable package tag') + if args.mode != 'run': + return + require(args.work_dir and not os.path.lexists(args.work_dir), 'work directory is not fresh') + require(args.run_token and re.fullmatch('[0-9a-f]{16}', args.run_token), 'invalid run token') + require(args.docker_endpoint and args.docker_endpoint.startswith('unix:///'), 'local daemon endpoint required') + require(args.guest_boot_id and re.fullmatch('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', args.guest_boot_id), + 'exact disposable guest boot identity required') + require(type(args.started_unix) is int and type(args.active_unix) is int and type(args.cleanup_unix) is int + and args.cleanup_unix == args.started_unix + 1800 + and args.active_unix == args.cleanup_unix - 60, 'wrong absolute deadline envelope') + require(args.started_unix <= time.time() < args.started_unix + 30, 'stale launch timestamp') + + +def bootstrap(): + global CURRENT, ARGS, RECORD, RECORD_SHA + args = arguments() + validate(args) + value, captured = manifest(args.checkout, args.runner_commit) + actual = digest(canonical(value)) + if args.mode == 'manifest': + print(json.dumps({'manifest_sha256': actual, 'manifest': value}, sort_keys=True)) + return + require(actual == args.manifest_sha256, 'wrong reviewed manifest') + original = sys.orig_argv + require('-I' in original and '-S' in original and '-c' in original, 'captured isolated entry point required') + entry = original[original.index('-c') + 1].encode() + require(digest(entry) == value['files'][PREFIX + 'run.py']['sha256'], 'mutable or changed entry point') + sys.dont_write_bytecode = True + if args.mode in ('recover', 'audit'): + program = captured[PREFIX + 'recover.py'].decode() + argv = [sys.executable, '-I', '-S', '-c', program, args.mode, '--work-dir', args.work_dir, + '--record-sha256', args.record_sha256 or '', '--manifest-sha256', actual, + '--runner-commit', args.runner_commit, '--package-commit', H, + '--run-token', args.run_token or ''] + if args.summary_sha256: + argv += ['--summary-sha256', args.summary_sha256] + os.execv(sys.executable, argv) + # Collision reads precede directory creation, registry work and every role. + temporary = Subject(value, captured, pathlib.Path(args.work_dir) / 'snapshot') + ARGS, CURRENT = args, temporary + module = types.ModuleType('sealed_resource_preflight') + module.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'resources.py' + exec(compile(captured[PREFIX + 'resources.py'], module.__file__, 'exec'), module.__dict__) + recovery = types.ModuleType('captured_recovery_preflight') + recovery.__file__ = 'git:' + args.runner_commit + ':' + PREFIX + 'recover.py' + exec(compile(captured[PREFIX + 'recover.py'], recovery.__file__, 'exec'), recovery.__dict__) + args.guest_storage = recovery.storage_preflight(args, lambda line: module.call(args, line)) + args.storage_guard = lambda: recovery.validate_storage(args.guest_storage, args.work_dir) + module.preflight_collisions(args) + RECORD = recovery.make_record(args, actual, value['files'][PREFIX + 'recover.py']['sha256']) + RECORD_SHA = digest(canonical(RECORD)) + print(json.dumps({'private_launch': RECORD, 'record_sha256': RECORD_SHA}), flush=True) + pathlib.Path(args.work_dir).mkdir(mode=0o700) + exclusive(pathlib.Path(args.work_dir) / 'run.json', canonical(RECORD)) + for directory in ('public-home', 'public-docker'): + (pathlib.Path(args.work_dir) / directory).mkdir(mode=0o700) + exclusive(pathlib.Path(args.work_dir) / 'public-docker/config.json', canonical(recovery.PUBLIC_CONFIG)) + temporary.materialize() + CURRENT.module('engine').execute(args) diff --git a/scripts/acceptance/condition-history/test_audit.py b/scripts/acceptance/condition-history/test_audit.py new file mode 100644 index 0000000..b7cfd7f --- /dev/null +++ b/scripts/acceptance/condition-history/test_audit.py @@ -0,0 +1,120 @@ +"""Final handoff/recovery controls with synthetic records and a fake daemon.""" +import contextlib +import copy +import io +import json +import os +import pathlib +import tempfile +import unittest +from unittest import mock +import test_subject as controls +from test_subject import fixture_subject, recovery, remove_private, subject + + +class Audit(unittest.TestCase): + record = controls.SubjectControls.record + validate = controls.SubjectControls.validate + + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) + self.sealed = fixture_subject(self.root) + self.authority = self.record() + self.validate(self.authority) + self.record_sha = subject.digest(subject.canonical(self.authority)) + self.program = self.sealed.captured[subject.PREFIX + 'recover.py'].decode() + self.watcher = {'source_sha256': self.authority['recovery_sha256'], + 'process': {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 234567}} + watcher_sha = subject.exclusive(self.root / 'recovery-process.json', subject.canonical(self.watcher)) + self.summary = {'checks_outcome': 'pass', 'outcome': 'pending_independent_audit', + 'record_sha256': self.record_sha, 'runner_commit': self.authority['runner_commit'], + 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, + 'recovery_process_sha256': watcher_sha, 'cleanup_errors': [], + 'cleanup': {'pass': True, 'errors': [], + 'remaining': {'container': [], 'network': [], 'volume': []}}} + self.summary_sha = subject.exclusive(self.root / 'summary.json', subject.canonical(self.summary)) + + def args(self, mode='audit'): + result = [mode, '--work-dir', str(self.root), '--record-sha256', self.record_sha, + '--manifest-sha256', self.sealed.sha256, '--runner-commit', self.authority['runner_commit'], + '--package-commit', subject.H, '--run-token', self.authority['token']] + if mode == 'audit':result += ['--summary-sha256', self.summary_sha] + return result + + def invoke(self, mode='audit', objects=None, groups=None): + with mock.patch.object(recovery.sys, 'orig_argv', ['python', '-I', '-S', '-c', self.program]), \ + mock.patch.object(recovery, 'objects', side_effect=objects or (lambda *a: [])), \ + mock.patch.object(recovery, 'group_members', side_effect=groups or (lambda *a: [])), \ + mock.patch.object(recovery, 'docker', side_effect=AssertionError('no actual daemon')), \ + contextlib.redirect_stdout(io.StringIO()): + recovery.main(self.args(mode)) + + def test_independent_audit_binds_exact_summary_and_both_processes(self): + self.invoke() + result = json.loads(recovery.read(self.root / 'independent-audit.json')) + self.assertEqual(result['summary_sha256'], self.summary_sha) + self.assertEqual(result['remaining'], {'container': [], 'network': [], 'volume': []}) + self.assertTrue(result['owner_process_absent'] and result['recovery_process_absent']) + self.assertEqual(result['outcome'], 'pending_external_guest_teardown') + self.assertEqual(result['storage'], self.authority['storage']) + with self.assertRaises(FileExistsError):self.invoke() + + def test_changed_summary_never_gains_authority(self): + (self.root / 'summary.json').write_bytes(b'{}\n') + with self.assertRaisesRegex(SystemExit, 'digest mismatch'):self.invoke() + self.assertFalse((self.root / 'independent-audit.json').exists()) + + def test_changed_watcher_receipt_refuses_even_if_it_claims_no_survivor(self): + (self.root / 'recovery-process.json').write_bytes(subject.canonical(dict(self.watcher, process={}))) + with self.assertRaisesRegex(SystemExit, 'handoff changed'):self.invoke() + + def test_each_object_and_process_survivor_refuses(self): + for kind in ('container', 'network', 'volume'): + with self.subTest(kind=kind), self.assertRaisesRegex(SystemExit, 'object survivor'): + self.invoke(objects=lambda record, k: ['synthetic'] if k == kind else []) + for pgid in (12345, 23456): + with self.subTest(pgid=pgid), self.assertRaisesRegex(SystemExit, 'process survivor'): + self.invoke(groups=lambda group: [group] if group == pgid else []) + self.assertFalse((self.root / 'independent-audit.json').exists()) + + def test_audit_does_not_follow_redirected_summary(self): + (self.root / 'summary.json').unlink() + foreign = self.root / 'foreign';foreign.write_bytes(b'preserved') + (self.root / 'summary.json').symlink_to(foreign) + with self.assertRaises(SystemExit):self.invoke() + self.assertEqual(foreign.read_bytes(), b'preserved') + + def test_failed_or_incomplete_owner_result_never_gains_audit_pass(self): + for field, value in [('outcome', 'fail'), ('checks_outcome', 'fail'), + ('cleanup', {}), ('cleanup_errors', ['uncertain'])]: + bad = dict(self.summary, **{field: value}) + raw = subject.canonical(bad) + (self.root / 'summary.json').write_bytes(raw) + self.summary_sha = subject.digest(raw) + with self.subTest(field=field), self.assertRaisesRegex(SystemExit, 'owner checks or cleanup'): + self.invoke() + self.assertFalse((self.root / 'independent-audit.json').exists()) + + def test_late_source_drift_cannot_cancel_captured_exact_cleanup(self): + with mock.patch.object(recovery, 'validate_record', side_effect=[self.authority, SystemExit('drift')]), \ + mock.patch.object(recovery, 'stop_owner') as stop, \ + mock.patch.object(recovery, 'cleanup', return_value={'pass': True, 'errors': [], 'remaining': {}}) as cleanup, \ + self.assertRaisesRegex(SystemExit, 'recovery incomplete'): + self.invoke('recover') + stop.assert_called_once_with(self.authority) + cleanup.assert_called_once_with(self.authority) + result = json.loads(recovery.read(self.root / 'recovery-audit.json')) + self.assertFalse(result['pass']) + self.assertIn('source-or-record-drift', result['errors']) + + def test_uncertain_process_authority_still_performs_owned_daemon_cleanup(self): + with mock.patch.object(recovery, 'stop_owner', side_effect=SystemExit('foreign epoch')), \ + mock.patch.object(recovery, 'cleanup', return_value={'pass': True, 'errors': [], 'remaining': {}}) as cleanup, \ + self.assertRaisesRegex(SystemExit, 'recovery incomplete'): + self.invoke('recover') + cleanup.assert_called_once_with(self.authority) + self.assertIn('owner-process-uncertain', json.loads(recovery.read(self.root / 'recovery-audit.json'))['errors']) + + +if __name__ == '__main__':unittest.main() diff --git a/scripts/acceptance/condition-history/test_guest_host.py b/scripts/acceptance/condition-history/test_guest_host.py new file mode 100644 index 0000000..ed2e3e7 --- /dev/null +++ b/scripts/acceptance/condition-history/test_guest_host.py @@ -0,0 +1,445 @@ +"""External guest controls use fake native commands. No VM or disk is created.""" +import copy +import ctypes +import json +import os +import pathlib +import plistlib +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock +from test_subject import HERE, load + +host = load('condition_guest_host_tests', HERE / 'guest-host.py') +TOKEN = '0123456789abcdef' +BOOT = '11111111-1111-1111-1111-111111111111' + + +class GuestHost(unittest.TestCase): + def setUp(self): + self.temp = pathlib.Path(tempfile.mkdtemp(dir='/tmp', prefix='sr-')).resolve() + self.addCleanup(shutil.rmtree, self.temp) + self.parent = self.temp / ('sr-' + TOKEN) + self.parent.mkdir(mode=0o700) + self.record = {'parent': str(self.parent), 'token': TOKEN, 'limactl': '/reviewed/bin/limactl', + 'runner_commit': 'a' * 40, 'manifest_sha256': 'b' * 64, 'guardian_sha256': 'c' * 64, + 'guest_agent_binary_sha256': 'e' * 64, + 'started_unix': int(time.time()), 'active_unix': int(time.time()) + 1740, + 'cleanup_unix': int(time.time()) + 1800, '_cleanup_mono': time.monotonic() + 1800, + '_active_mono': time.monotonic() + 1740} + self.scoped = host.paths(str(self.parent), TOKEN) + self.profile = json.loads((HERE / 'guest-profile.json').read_bytes()) + + def put(self, name, value): + path = self.parent / name + path.write_bytes(host.canonical(value)) + return host.digest(path.read_bytes()) + + def proposal(self, profile=None): + return host.plan(str(self.parent), TOKEN, self.record['limactl'], + str(self.scoped['mount'] / 'a/guest.raw.gz'), profile or self.profile) + + def test_plan_uses_fixed_store_and_fresh_lima_home(self): + with mock.patch.object(host.subprocess, 'Popen', side_effect=AssertionError('inert plan')): + value = self.proposal() + self.assertEqual(value['host_store_bytes'], 32 * 1024**3) + self.assertEqual(value['guest_disk_bytes'], 20 * 1024**3) + self.assertEqual(value['create_store'][2:4], ['-sectors', '67108864']) + self.assertEqual(value['environment']['LIMA_HOME'], str(self.scoped['home'])) + self.assertNotIn('SSH_AUTH_SOCK', value['environment']) + self.assertNotIn('DOCKER_HOST', value['environment']) + self.assertNotIn('HTTP_PROXY', value['environment']) + self.assertEqual(value['profile']['mounts'], []) + + def test_any_template_change_is_a_new_source_subject(self): + for key, value in {'mounts': [{'location': '~'}], 'base': ['template:default'], + 'disk': '21GiB', 'cpus': 8, 'networks': [], 'provision': [], + 'propagateProxyEnv': True, 'portForwards': [], + 'ssh': dict(self.profile['ssh'], forwardAgent=True)}.items(): + candidate = copy.deepcopy(self.profile) + candidate[key] = value + with self.subTest(key=key), self.assertRaises(SystemExit): + self.proposal(candidate) + + def test_foreign_scope_and_unbounded_image_paths_refuse(self): + for root, token, image in [('/tmp/foreign', TOKEN, '/tmp/image'), + (str(self.parent), 'foreign', '/tmp/image'), + (str(self.parent), TOKEN, '/tmp/image')]: + with self.subTest(root=root, token=token), self.assertRaises(SystemExit): + host.plan(root, token, '/reviewed/limactl', image, self.profile) + + def test_public_darwin_struct_and_consistent_kernel_epoch(self): + self.assertEqual(ctypes.sizeof(host.BsdInfo), 136) + process = {'pid': 1234, 'pgid': 1234, 'uid': os.getuid(), + 'start_seconds': 1000, 'start_microseconds': 123456} + with mock.patch.object(host, 'boot_identity', return_value=BOOT), \ + mock.patch.object(host, 'bsd_identity', return_value=process): + self.assertEqual(host.process_identity(1234), dict(process, boot_id=BOOT)) + for changed in (dict(process, start_microseconds=123457), None): + with mock.patch.object(host, 'boot_identity', return_value=BOOT), \ + mock.patch.object(host, 'bsd_identity', side_effect=[process, changed]), self.assertRaises(SystemExit): + host.process_identity(1234) + with mock.patch.object(host, 'boot_identity', side_effect=[BOOT, 'other']), \ + mock.patch.object(host, 'bsd_identity', return_value=process), self.assertRaises(SystemExit): + host.process_identity(1234) + + def exports(self): + storage = {'guest_boot_id': BOOT} + summary_sha = self.put('guest-summary.json', {'runner_commit': self.record['runner_commit'], + 'manifest_sha256': self.record['manifest_sha256'], 'checks_outcome': 'pass', 'storage': storage}) + audit_sha = self.put('guest-audit.json', {'outcome': 'pending_external_guest_teardown', + 'manifest_sha256': self.record['manifest_sha256'], 'summary_sha256': summary_sha, 'storage': storage}) + exports_sha = self.put('exports.json', {'record_sha256': 'd' * 64, 'guest_boot_id': BOOT, + 'guest-summary.json': summary_sha, 'guest-audit.json': audit_sha}) + self.put('finish.json', {'record_sha256': 'd' * 64, 'exports_sha256': exports_sha}) + + def test_finish_requires_exact_external_exports_and_digests(self): + self.assertFalse(host.finish_requested(self.record, 'd' * 64)) + self.exports() + self.assertTrue(host.finish_requested(self.record, 'd' * 64)) + self.put('finish.json', {'record_sha256': 'foreign', 'exports_sha256': 'e' * 64}) + with self.assertRaises(SystemExit):host.finish_requested(self.record, 'd' * 64) + self.exports() + self.put('guest-summary.json', {'checks_outcome': 'pass'}) + with self.assertRaises(SystemExit):host.finish_requested(self.record, 'd' * 64) + + def audit_files(self): + self.exports() + self.put('host-ready.json', {'record_sha256': 'd' * 64, 'source_sha256': 'c' * 64, + 'process': {'pid': 1234}}) + self.put('vm-process.json', {'pid': 2345}) + self.put('vm-started.json', {'record_sha256': 'd' * 64, + 'agent': {'guest_boot_id': BOOT, 'sha256': self.record['guest_agent_binary_sha256']}}) + self.put('host-cleanup.json', {'record_sha256': 'd' * 64, 'guardian_sha256': 'c' * 64, + 'outcome': 'pending_independent_host_audit', 'guest_absent': True, + 'store_unmounted': True, 'image_absent': True}) + + def test_independent_host_audit_requires_guest_proof_and_both_processes_absent(self): + self.audit_files() + with mock.patch.object(host, 'process_identity', return_value=None): + self.assertEqual(host.audit(self.record, 'd' * 64, 'c' * 64)['outcome'], 'pass') + with mock.patch.object(host, 'process_identity', return_value=None), self.assertRaises(FileExistsError): + host.audit(self.record, 'd' * 64, 'c' * 64) + (self.parent / 'host-audit.json').unlink() + for survivor in (1234, 2345): + def observe(pid):return {'pid': pid} if pid == survivor else None + with self.subTest(survivor=survivor), mock.patch.object(host, 'process_identity', side_effect=observe), \ + self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + self.scoped['image'].write_bytes(b'survivor') + with self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + + def test_host_audit_refuses_rebound_guest_and_forged_cleanup(self): + self.audit_files() + for filename, key, value in [('guest-summary.json', 'runner_commit', 'e' * 40), + ('guest-audit.json', 'outcome', 'pass'), ('host-cleanup.json', 'guardian_sha256', 'foreign'), + ('vm-started.json', 'agent', {'guest_boot_id': BOOT, 'sha256': 'foreign'})]: + self.audit_files() + path = self.parent / filename + data = json.loads(path.read_bytes());data[key] = value + self.put(filename, data) + with self.subTest(filename=filename), mock.patch.object(host, 'process_identity', return_value=None), \ + self.assertRaises(SystemExit):host.audit(self.record, 'd' * 64, 'c' * 64) + + def test_native_control_stream_has_hard_output_and_time_bounds(self): + # This launches only an inert Python child, with neither native VM tools nor network. + result = host.call(self.record, [sys.executable, '-I', '-S', '-c', 'print("bounded")']) + self.assertEqual(result, b'bounded\n') + with self.assertRaisesRegex(SystemExit, 'oversize'): + host.call(self.record, [sys.executable, '-I', '-S', '-c', 'import os;os.write(1,b"x"*2200000)']) + self.record['_cleanup_mono'] = time.monotonic() - 1 + with mock.patch.object(host.subprocess, 'Popen', side_effect=AssertionError('deadline before launch')), \ + self.assertRaisesRegex(SystemExit, 'deadline'): + host.call(self.record, ['/never/launch']) + + def test_foreign_store_refuses_all_cleanup_mutations(self): + with mock.patch.object(host, 'verify_store', side_effect=SystemExit('foreign volume')), \ + mock.patch.object(host, 'call', side_effect=AssertionError('must not mutate')), self.assertRaises(SystemExit): + host.cleanup(self.record) + + def test_changed_process_epoch_refuses_scoped_stop(self): + self.put('vm-process.json', {'pid': 2345, 'start_microseconds': 1}) + with mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 2345, 'start_microseconds': 2}), \ + mock.patch.object(host, 'call', side_effect=AssertionError('must not stop')), self.assertRaises(SystemExit): + host.cleanup(self.record) + + def test_bad_guard_observation_still_attempts_cleanup(self): + self.record['_active_mono'] = time.monotonic() + 20 + result = {'guest_absent': True, 'store_unmounted': True, 'image_absent': True} + with mock.patch.object(host.sys, 'orig_argv', ['python', '-I', '-S', '-c', 'captured']), \ + mock.patch.object(host, 'authority', return_value=self.record), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'capture_agent', side_effect=SystemExit('uncertain')), \ + mock.patch.object(host, 'cleanup', return_value=result) as cleanup, self.assertRaises(SystemExit): + host.main(['guard', '--record', '/unused', '--record-sha256', 'd' * 64]) + cleanup.assert_called_once_with(self.record) + receipt = json.loads((self.parent / 'host-cleanup.json').read_bytes()) + self.assertEqual(receipt['outcome'], 'fail') + self.assertEqual(receipt['control_error_type'], 'SystemExit') + + def fake_setup(self, failure=None): + self.parent.rmdir() + self.record['_stage_mono'] = time.monotonic() + 300 + self.record.update(schema='durable-workflow.condition-host-setup/v1', host_boot_id=BOOT, + guest_agent='/reviewed/share/lima/lima-guestagent.Linux-x86_64.gz', cached_image='/public/guest.raw.gz') + proposal = self.proposal() + calls = [] + def command(record, argv): + calls.append(argv) + if argv == proposal['create_store']: + if failure == 'create':raise SystemExit('failed native create') + self.scoped['image'].write_bytes(b'fixed-image') + if argv == proposal['attach_store'] and failure == 'attach':raise SystemExit('lost attach response') + return b'' + def copy_input(path, size, sha, destination=None, deadline=None): + if destination: + if failure == 'copy':raise OSError('fixed disk full') + destination.write_bytes(b'pinned-public-image') + return proposal, calls, command, copy_input + + def test_resident_watchdog_owns_setup_and_never_starts_guest(self): + proposal, calls, command, copy_input = self.fake_setup() + with mock.patch.object(host, 'capacity', return_value={'available_bytes': 12 * 1024**3}), \ + mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'bound_input', return_value={'device': 1, 'inode': 2, 'bytes': host.IMAGE_BYTES, 'sha256': host.IMAGE_SHA}), \ + mock.patch.object(host, 'agent_binary_sha', return_value='e' * 64), \ + mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}): + value, sha = host.prepare(self.record, self.profile) + self.assertEqual(set(value), host.AUTHORITY_KEYS) + self.assertEqual(sha, host.digest((self.parent / 'host-record.json').read_bytes())) + self.assertEqual(calls, [proposal['create_store'], proposal['attach_store']]) + self.assertTrue((self.parent / 'setup-ready.json').exists()) + self.assertEqual(value['profile_sha256'], proposal['profile_sha256']) + self.assertNotIn('_stage_mono', self.record) + + def test_setup_failures_have_owned_or_explicit_uncertain_custody(self): + for failure in ('create', 'attach', 'copy'): + with self.subTest(stage=failure): + if self.parent.exists():shutil.rmtree(self.parent) + self.parent.mkdir(mode=0o700) + _, _, command, copy_input = self.fake_setup(failure) + with mock.patch.object(host, 'capacity', return_value={}), \ + mock.patch.object(host, 'input_bytes', side_effect=copy_input), \ + mock.patch.object(host, 'bound_input', return_value={'device': 1, 'inode': 2, 'bytes': host.IMAGE_BYTES, 'sha256': host.IMAGE_SHA}), \ + mock.patch.object(host, 'agent_binary_sha', return_value='e' * 64), \ + mock.patch.object(host, 'call', side_effect=command), mock.patch.object(host, 'verify_store'), \ + mock.patch.object(host, 'process_identity', return_value={'pid': 1234}), \ + mock.patch.object(host, 'observe_attachment', return_value={'whole_device': '/dev/disk9', 'volume_uuid': BOOT}), \ + mock.patch.object(host, 'remove_store') as remove, self.assertRaises((SystemExit, OSError)): + host.prepare(self.record, self.profile) + receipt = json.loads((self.parent / 'setup-failure.json').read_bytes()) + self.assertEqual(receipt['outcome'], 'fail') + if failure == 'create': + self.assertEqual(receipt['custody'], 'retained_uncertain_task_store') + remove.assert_not_called() + else: + self.assertEqual(receipt['custody'], 'owned_store_removed') + remove.assert_called_once() + self.assertIn('image_ino', receipt['store_authority']) + self.record.pop('image_ino', None) + self.record.pop('image_dev', None) + + def test_capacity_refuses_before_creating_a_disk_or_scope(self): + self.parent.rmdir() + with mock.patch.object(host, 'capacity', side_effect=SystemExit('insufficient memory')), \ + mock.patch.object(host, 'call', side_effect=AssertionError('no native mutation')), self.assertRaises(SystemExit): + host.prepare(self.record, self.profile) + self.assertFalse(self.parent.exists()) + + def test_native_memory_and_disk_readbacks_have_exact_refusal_gates(self): + def native(record, argv): + if '-productVersion' in argv:return b'15.8.2\n' + if '-buildVersion' in argv:return b'24H232\n' + return b'Mach Virtual Memory Statistics: (page size of 4096 bytes)\nPages free: 2000000.\nPages inactive: 1000000.\nPages speculative: 0.\n' + with mock.patch.object(host.sys, 'platform', 'darwin'), \ + mock.patch.object(host.os, 'uname', return_value=types.SimpleNamespace(machine='x86_64')), \ + mock.patch.object(host, 'call', side_effect=native), \ + mock.patch.object(host.os, 'statvfs', return_value=types.SimpleNamespace(f_bavail=60 * 1024**3, f_frsize=1)): + self.assertEqual(host.capacity(self.record)['available_bytes'], 3_000_000 * 4096) + with mock.patch.object(host.os, 'statvfs', return_value=types.SimpleNamespace(f_bavail=1024, f_frsize=1)), \ + self.assertRaises(SystemExit):host.capacity(self.record) + with mock.patch.object(host.sys, 'platform', 'darwin'), \ + mock.patch.object(host.os, 'uname', return_value=types.SimpleNamespace(machine='x86_64')), \ + mock.patch.object(host, 'call', return_value=b'unknown'), self.assertRaises(SystemExit): + host.capacity(self.record) + + def test_pinned_input_copy_refuses_changed_bytes_existing_destination_and_cutoff(self): + path = self.parent / 'public';path.write_bytes(b'public immutable input') + sha = host.digest(path.read_bytes());size = path.stat().st_size + target = self.parent / 'copied' + host.input_bytes(path, size, sha, target, time.monotonic() + 2) + self.assertEqual(path.read_bytes(), target.read_bytes()) + with self.assertRaises(FileExistsError):host.input_bytes(path, size, sha, target) + for length, digest, deadline in ((size + 1, sha, None), (size, 'f' * 64, None), (size, sha, time.monotonic() - 1)): + with self.subTest(length=length, digest=digest), self.assertRaises(SystemExit): + host.input_bytes(path, length, digest, deadline=deadline) + + def test_native_attachment_requires_unique_fixed_device_and_exact_mount(self): + attachment = {'images': [{'image-path': str(self.scoped['image']), 'system-entities': [ + {'dev-entry': '/dev/disk9'}, {'dev-entry': '/dev/disk10', 'mount-point': str(self.scoped['mount'])}]}]} + def native(record, argv): + if 'hdiutil' in argv[0]:return plistlib.dumps(attachment) + if argv[-1] == '/dev/disk9':return plistlib.dumps({'TotalSize': host.STORE_BYTES}) + if argv[-1] == '/dev/disk10':return plistlib.dumps({'TotalSize': host.STORE_BYTES - 4096}) + return plistlib.dumps({'FilesystemType': 'apfs', 'MountPoint': str(self.scoped['mount']), + 'TotalSize': host.STORE_BYTES - 4096, 'VolumeUUID': BOOT}) + with mock.patch.object(host, 'call', side_effect=native): + self.assertEqual(host.observe_attachment(self.record), {'whole_device': '/dev/disk9', 'volume_uuid': BOOT}) + attachment['images'][0]['system-entities'][1]['mount-point'] = '/foreign' + with self.assertRaises(SystemExit):host.observe_attachment(self.record) + + def held(self): + data = subprocess.check_output(['git', '--no-replace-objects', '-C', str(HERE.parents[2]), 'show', + 'b5cf1e9df9b3ffe88e3fa3290074ae10a8422171:scripts/acceptance/condition-history/guest-host.py']) + module = types.ModuleType('held_b5_guest_source') + exec(compile(data, 'git:held-b5-guardian', 'exec'), module.__dict__) + return module + + def store_fixture(self): + self.scoped['home'].mkdir(parents=True) + self.scoped['image'].write_bytes(b'fixed-image') + self.record['profile'] = self.proposal()['profile'] + self.record['profile_sha256'] = self.put('unused-profile.json', self.record['profile']) + (self.scoped['mount'] / 'guest.yaml').write_bytes(host.canonical(self.record['profile'])) + info = self.scoped['image'].stat() + self.record.update(image_dev=info.st_dev, image_ino=info.st_ino, whole_device='/dev/disk9', + volume_uuid=BOOT, host_boot_id=BOOT) + def native(record, argv): + if 'hdiutil' in argv[0]: + return plistlib.dumps({'images': [{'image-path': str(self.scoped['image']), + 'system-entities': [{'dev-entry': '/dev/disk9', 'mount-point': str(self.scoped['mount'])}]}]}) + if argv[-1] == '/dev/disk9':return plistlib.dumps({'TotalSize': host.STORE_BYTES}) + return plistlib.dumps({'VolumeUUID': BOOT, 'TotalSize': host.STORE_BYTES - 4096, + 'MountPoint': str(self.scoped['mount']), 'FilesystemType': 'apfs'}) + return native + + def test_actual_held_store_check_admits_destination_corruption(self): + old = self.held();native = self.store_fixture() + target = self.scoped['mount'] / 'a/guest.raw.gz';target.parent.mkdir();target.write_bytes(b'old') + self.record['guest_image'] = host.bound_input(target, 3, host.digest(b'old')) + target.write_bytes(b'bad') + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + with mock.patch.object(host, 'IMAGE_BYTES', 3), mock.patch.object(host, 'IMAGE_SHA', host.digest(b'old')), \ + self.assertRaisesRegex(SystemExit, 'digest'): + host.verify_guest_inputs(self.record) + replacement = target.with_name('replacement');replacement.write_bytes(b'old');replacement.replace(target) + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + with mock.patch.object(host, 'IMAGE_BYTES', 3), mock.patch.object(host, 'IMAGE_SHA', host.digest(b'old')), \ + self.assertRaisesRegex(SystemExit, 'identity'): + host.verify_guest_inputs(self.record) + + def test_actual_held_plan_admits_profile_edit_restore_but_new_create_consumes_captured_stdin(self): + old = self.held();native = self.store_fixture() + legacy = old.plan(str(self.parent), TOKEN, self.record['limactl'], + str(self.scoped['mount'] / 'a/guest.raw.gz'), self.profile) + path = self.scoped['mount'] / 'guest.yaml';reviewed = path.read_bytes() + foreign = json.loads(reviewed);foreign['mounts'] = [{'location': '/foreign/host'}] + path.write_bytes(host.canonical(foreign)) + self.assertEqual(json.loads(pathlib.Path(legacy['create_vm'][-1]).read_bytes())['mounts'], foreign['mounts']) + path.write_bytes(reviewed) + with mock.patch.object(old, 'call', side_effect=native):old.verify_store(self.record) + self.record['guest_image'] = {'device': 1, 'inode': 2, 'bytes': 3, 'sha256': 'f' * 64} + self.put('launch.json', {'record_sha256': 'd' * 64, 'manifest_sha256': self.record['manifest_sha256'], 'action': 'create-and-start'}) + consumed = [] + def command(record, argv, input_data=None): + if 'create' in argv: + path.write_bytes(host.canonical(foreign)) + consumed.append((argv, input_data)) + path.write_bytes(reviewed) + if 'shell' in argv: + return host.canonical({'pid': 100, 'start_ticks': 1000, 'guest_boot_id': BOOT, + 'sha256': self.record['guest_agent_binary_sha256']}) + return b'' + with mock.patch.object(host, 'verify_store'), mock.patch.object(host, 'verify_guest_inputs'), \ + mock.patch.object(host, 'call', side_effect=command), \ + mock.patch.object(host, 'effective_config', side_effect=[{'status': 'Stopped'}, {'status': 'Stopped'}, {'status': 'Running'}]), \ + mock.patch.object(host, 'capture_agent', return_value={'pid': 2345}): + host.launch_guest(self.record, 'd' * 64) + self.assertEqual(consumed[0][0][-1], '-') + self.assertEqual(consumed[0][1], reviewed) + self.assertEqual(json.loads(consumed[0][1])['mounts'], []) + self.assertTrue((self.parent / 'vm-started.json').exists()) + + def test_actual_held_authority_accepts_correct_hash_at_unselected_agent_path(self): + old = self.held();self.store_fixture() + binary = self.temp / 'tools/bin/limactl';binary.parent.mkdir(parents=True);binary.write_bytes(b'launcher') + unrelated = self.temp / 'unselected-agent.gz';unrelated.write_bytes(b'pinned agent bytes') + value = {key: self.record.get(key) for key in old.AUTHORITY_KEYS} + value.update(schema='durable-workflow.condition-host/v1', limactl=str(binary), guest_agent=str(unrelated)) + sha = self.put('legacy-record.json', value) + with mock.patch.object(old, 'LIMA_SHA', host.digest(binary.read_bytes())), \ + mock.patch.object(old, 'AGENT_SHA', host.digest(unrelated.read_bytes())), \ + mock.patch.object(old, 'boot_identity', return_value=BOOT): + accepted = old.authority(self.parent / 'legacy-record.json', sha, self.record['guardian_sha256']) + with self.assertRaisesRegex(SystemExit, 'not the executable-selected'): + host.selected_agent(accepted) + + def test_effective_config_refuses_resource_share_image_and_forwarding_changes(self): + self.record['profile'] = self.proposal()['profile'] + expected = {k: v for k, v in self.record['profile'].items() if k not in ('base', 'minimumLimaVersion')} + value = {'name': self.scoped['name'], 'dir': str(self.scoped['instance']), 'status': 'Stopped', + 'vmType': 'vz', 'arch': 'x86_64', 'cpus': 4, 'memory': host.MEMORY_BYTES, + 'disk': host.GUEST_BYTES, 'config': expected} + with mock.patch.object(host, 'call', return_value=host.canonical(value)): + self.assertEqual(host.effective_config(self.record, 'Stopped'), value) + for field, change in [('mounts', [{'location': '/foreign'}]), ('portForwards', []), + ('images', []), ('ssh', dict(expected['ssh'], forwardAgent=True))]: + changed = copy.deepcopy(value);changed['config'][field] = change + with self.subTest(field=field), mock.patch.object(host, 'call', return_value=host.canonical(changed)), \ + self.assertRaises(SystemExit):host.effective_config(self.record, 'Stopped') + for key, change in [('cpus', 8), ('memory', host.MEMORY_BYTES + 1), ('disk', host.GUEST_BYTES + 1)]: + changed = copy.deepcopy(value);changed[key] = change + with self.subTest(key=key), mock.patch.object(host, 'call', return_value=host.canonical(changed)), \ + self.assertRaises(SystemExit):host.effective_config(self.record, 'Stopped') + + def test_launch_refuses_unbound_request_before_native_create(self): + self.put('launch.json', {'action': 'create-and-start', 'record_sha256': 'foreign'}) + with mock.patch.object(host, 'call', side_effect=AssertionError('must not create')), self.assertRaises(SystemExit): + host.launch_guest(self.record, 'd' * 64) + + def test_actual_held_agent_capture_accepts_unselected_path_but_new_capture_refuses(self): + old = self.held() + self.scoped['instance'].mkdir(parents=True) + self.record['guest_agent'] = '/reviewed/share/lima/lima-guestagent.Linux-x86_64.gz' + (self.scoped['instance'] / 'ha.pid').write_text('2345\n') + process = {'pid': 2345, 'start_seconds': self.record['started_unix'], 'start_microseconds': 123} + command = [self.record['limactl'], 'hostagent', '--pidfile', str(self.scoped['instance'] / 'ha.pid'), + '--guestagent', '/unselected/agent.gz', self.scoped['name']] + import shlex + raw = shlex.join(command).encode() + with mock.patch.object(old, 'call', return_value=raw), mock.patch.object(old, 'process_identity', return_value=process): + self.assertEqual(old.capture_agent(self.record), process) + with mock.patch.object(host, 'call', return_value=raw), mock.patch.object(host, 'process_identity', return_value=process), \ + self.assertRaisesRegex(SystemExit, 'foreign'): + host.capture_agent(self.record) + command[command.index('--guestagent') + 1] = self.record['guest_agent'] + with mock.patch.object(host, 'call', return_value=shlex.join(command).encode()), \ + mock.patch.object(host, 'process_identity', return_value=process): + self.assertEqual(host.capture_agent(self.record), process) + + def test_running_guest_agent_kernel_hash_must_match_pinned_selected_binary(self): + self.store_fixture() + self.record['guest_image'] = {'device': 1, 'inode': 2, 'bytes': 3, 'sha256': 'f' * 64} + self.put('launch.json', {'record_sha256': 'd' * 64, 'manifest_sha256': self.record['manifest_sha256'], 'action': 'create-and-start'}) + def command(record, argv, input_data=None): + if 'shell' in argv:return host.canonical({'pid': 100, 'start_ticks': 1000, 'guest_boot_id': BOOT, 'sha256': 'wrong'}) + return b'' + with mock.patch.object(host, 'verify_store'), mock.patch.object(host, 'verify_guest_inputs'), \ + mock.patch.object(host, 'call', side_effect=command), \ + mock.patch.object(host, 'effective_config', side_effect=[{'status': 'Stopped'}, {'status': 'Stopped'}, {'status': 'Running'}]), \ + mock.patch.object(host, 'capture_agent', return_value={'pid': 2345}), \ + self.assertRaisesRegex(SystemExit, 'running guest agent'): + host.launch_guest(self.record, 'd' * 64) + self.assertTrue((self.parent / 'vm-created.json').exists()) + self.assertFalse((self.parent / 'vm-started.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/acceptance/condition-history/test_packet.py b/scripts/acceptance/condition-history/test_packet.py new file mode 100644 index 0000000..32f9a5d --- /dev/null +++ b/scripts/acceptance/condition-history/test_packet.py @@ -0,0 +1,236 @@ +"""Packet guard and unchanged history controls. No Docker/Cargo/network is executed.""" +import ast +import copy +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock +from test_subject import subject, fixture_subject, remove_private, REPO, HERE + +class Packet(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) + now = int(time.time()) + self.args = types.SimpleNamespace(mode='run', checkout=str(REPO), work_dir=str(self.root / 'fresh'), + package_commit=subject.H, runner_commit='a' * 40, run_token='0123456789abcdef', + docker_endpoint='unix:///synthetic.sock', guest_boot_id='11111111-1111-1111-1111-111111111111', started_unix=now, active_unix=now+1740, cleanup_unix=now+1800) + subject.ARGS = self.args + subject.CURRENT = fixture_subject(self.root) + self.packet = subject.CURRENT.module('engine') + global packet + packet = self.packet + + def git(self, checkout, *args): + return {'HEAD': 'a' * 40, 'refs/tags/4.12.0^{commit}': subject.H}.get(args[-1], '') + + def test_preflight_refusals_precede_any_side_effect(self): + for field, value in {'package_commit':'b'*40, 'run_token':'foreign', 'active_unix':0, + 'started_unix':0, 'docker_endpoint':'tcp://foreign'}.items(): + args = copy.copy(self.args);setattr(args, field, value) + with self.subTest(field=field), mock.patch.object(subject,'git',self.git), self.assertRaises(SystemExit): + subject.validate(args) + self.assertFalse(pathlib.Path(args.work_dir).exists()) + + def test_wrong_h_refuses_under_python_optimization(self): + result = subprocess.run([sys.executable,str(HERE/'run.py'),'run','--checkout',str(REPO), + '--runner-commit','a'*40,'--package-commit','b'*40], env=dict(os.environ,PYTHONOPTIMIZE='1'), + capture_output=True,text=True,timeout=5) + self.assertNotEqual(result.returncode,0) + self.assertIn('wrong fixed package source',result.stderr) + self.assertFalse(pathlib.Path(self.args.work_dir).exists()) + + def test_no_optimizable_runtime_assert(self): + for name in ('run.py','subject.py','engine.py','resources.py','recover.py','guest-host.py'): + with self.subTest(name=name): + self.assertFalse(any(isinstance(n,ast.Assert) for n in ast.walk(ast.parse((HERE/name).read_text())))) + + def recovery_start(self, changed=None, process_missing=False, exited=False): + pathlib.Path(self.args.work_dir).mkdir() + subject.RECORD_SHA = 'c' * 64 + process = {'pid': 23456, 'pgid': 23456, 'sid': 23456, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 234567} + recovery_sha = subject.CURRENT.value['files'][subject.PREFIX + 'recover.py']['sha256'] + ready = {'record_sha256': subject.RECORD_SHA, 'manifest_sha256': subject.CURRENT.sha256, + 'recovery_sha256': recovery_sha, 'process': process} + ready.update(changed or {}) + if not exited: + subject.exclusive(pathlib.Path(self.args.work_dir) / 'recovery-ready.json', subject.canonical(ready)) + watcher = mock.Mock(pid=process['pid']) + watcher.poll.return_value = 1 if exited else None + with mock.patch.object(packet.subprocess, 'Popen', return_value=watcher) as launch, \ + mock.patch.object(packet.recovery, 'process_identity', return_value=None if process_missing else process), \ + mock.patch.object(packet, 'registry_identity', side_effect=AssertionError('no registry before readiness')), \ + mock.patch.object(packet.resources, 'call', side_effect=AssertionError('no daemon')): + if changed or process_missing or exited: + with self.assertRaises(SystemExit): + packet.start_recovery(self.args, {'recovery_sha256': recovery_sha}) + else: + actual, receipt_sha = packet.start_recovery(self.args, {'recovery_sha256': recovery_sha}) + self.assertIs(actual, watcher) + receipt = (pathlib.Path(self.args.work_dir) / 'recovery-process.json').read_bytes() + self.assertEqual(subject.digest(receipt), receipt_sha) + self.assertEqual(json.loads(receipt), {'source_sha256': recovery_sha, 'process': process}) + argv = launch.call_args.args[0] + self.assertEqual(argv[:4], [sys.executable, '-I', '-S', '-c']) + self.assertEqual(subject.digest(argv[4].encode()), recovery_sha) + self.assertEqual(argv[-2:], ['--at', str(self.args.active_unix)]) + self.assertTrue(launch.call_args.kwargs['start_new_session']) + return watcher + + def test_captured_recovery_requires_exact_ready_receipt(self): + watcher = self.recovery_start() + watcher.terminate.assert_not_called() + + def test_wrong_recovery_ready_refuses_before_registry_or_daemon(self): + for key in ('record_sha256', 'manifest_sha256', 'recovery_sha256', 'process'): + with self.subTest(key=key): + watcher = self.recovery_start({key: 'foreign'}) + watcher.terminate.assert_called_once() + watcher.wait.assert_called_once_with(timeout=5) + import shutil + shutil.rmtree(self.args.work_dir) + + def test_missing_recovery_identity_stops_the_started_child(self): + watcher = self.recovery_start(process_missing=True) + watcher.terminate.assert_called_once() + watcher.wait.assert_called_once_with(timeout=5) + + def test_exited_recovery_refuses_without_launching_any_role(self): + watcher = self.recovery_start(exited=True) + watcher.terminate.assert_not_called() + + def test_dirty_head_tag_and_existing_work_refuse(self): + for response in ('dirty','wrong-head','wrong-tag','existing'): + def git(checkout,*args): + if response=='dirty' and args[0]=='status':return ' M source' + if response=='wrong-head' and args[-1]=='HEAD':return 'b'*40 + if response=='wrong-tag' and args[-1].startswith('refs/tags'):return 'b'*40 + return self.git(checkout,*args) + if response=='existing':pathlib.Path(self.args.work_dir).mkdir() + with self.subTest(response=response),mock.patch.object(subject,'git',git),self.assertRaises(SystemExit): + subject.validate(self.args) + + def test_exact_immutable_consumer_inputs_export_without_sdk_overlay(self): + fixture = self.root / 'fixture' + packet.export_consumer(fixture) + self.assertFalse((fixture / 'src').exists()) + self.assertEqual(packet.native.sha256(fixture / 'tests/official.rs'), + packet.INPUTS['tests/replay_regression_corpus.rs']) + for source, expected in packet.INPUTS.items(): + if source.endswith('.json'): + self.assertEqual(packet.native.sha256(fixture / source), expected) + manifest = packet.tomllib.loads((fixture / 'Cargo.toml').read_text()) + self.assertEqual(manifest['dependencies']['durable-workflow'], '=4.12.0') + self.assertNotIn('patch', manifest) + self.assertFalse((fixture / 'Cargo.lock').exists()) + self.assertFalse((fixture / '.cargo').exists()) + + def report(self): + fixture = packet.native.call(['git', '-C', str(packet.REPO), 'show', packet.H + + ':tests/fixtures/replay-regressions/two-member-condition-group-server-reopen.json']).stdout + events = json.loads(fixture)['history'] + events.append({'event_type': 'WorkflowCompleted', 'payload': {}}) + return {'outcome': 'pass', 'result': 'both', 'initial': events[:2], 'reopened': events[:6], 'history': events} + + def test_canonical_group_shape_is_accepted(self): + result = packet.comparator.check(self.report()) + self.assertEqual(result['reopened_sequences'], [3, 4]) + + def test_group_comparator_rejects_lineage_order_timeout_and_false_health(self): + cases = [lambda r: r.update(result='one'), + lambda r: r['history'][4]['payload'].update(parallel_group_index=1), + lambda r: r['history'][4]['payload'].update(condition_wait_occurrence_id='new-call'), + lambda r: r['history'][4]['payload'].update(timeout_seconds=301), + lambda r: r['history'][0]['payload'].update(sequence=True), + lambda r: r['history'][4]['payload'].update(parallel_group_index=False), + lambda r: r['history'].insert(0, {'event_type': 'ReplayFailure'}), + lambda r: r['history'][3]['payload'].update(sequence=1), + lambda r: r['history'][-2]['payload'].update(signal_name='wrong')] + for change in cases: + report = self.report() + change(report) + with self.subTest(change=change), self.assertRaises((ValueError, KeyError, TypeError)): + packet.comparator.check(report) + + def test_server_review_countercontrols(self): + cases = [] + report = self.report() + report['history'].insert(-1, {'event_type': 'TimerFired', 'payload': { + 'sequence': 3, 'condition_wait_id': 'foreign', 'timer_id': 'foreign', + 'timer_kind': 'condition_timeout', 'delay_seconds': 300}}) + cases.append(('foreign-timer', report)) + report = self.report() + report['history'].insert(-1, {'event_type': 'ConditionWaitTimedOut', 'payload': { + 'sequence': 999, 'condition_wait_id': 'foreign'}}) + cases.append(('foreign-timeout', report)) + report = self.report() + extra = copy.deepcopy(report['history'][4]) + extra['payload'].update(sequence=999, condition_wait_id='foreign', condition_key='foreign') + report['history'].insert(-1, extra) + cases.append(('foreign-fifth-open', report)) + report = self.report() + for row in report['history']: + if row['event_type'].startswith('ConditionWait'): + row['payload']['condition_definition_fingerprint'] = 'sha256:foreign' + cases.append(('consistently-wrong-fingerprint', report)) + report = self.report() + report['history'][0]['payload']['timeout_seconds'] = 300.0 + cases.append(('float-original-timeout', report)) + for label, report in cases: + with self.subTest(label=label), self.assertRaises(ValueError): + packet.comparator.check(report) + + def snapshots(self, events): + signals = [i for i, event in enumerate(events) if event['event_type'] == 'SignalReceived'] + return {'outcome': 'pass', 'result': 'both', 'initial': events[:signals[0]], + 'reopened': events[:signals[1]], 'history': events} + + def test_additional_reopens_keep_the_same_declared_members(self): + events = self.report()['history'] + extra = copy.deepcopy(events[4:6]) + for sequence, event in zip((5, 6), extra): + event['payload'].update(sequence=sequence, condition_wait_id='condition:' + str(sequence)) + events[6:6] = extra + report = self.snapshots(events) + self.assertEqual(packet.comparator.check(report)['all_open_sequences'], [1, 2, 3, 4, 5, 6]) + + def timer(self, kind, sequence): + return {'event_type': kind, 'payload': {'sequence': sequence, + 'condition_wait_id': 'condition:' + str(sequence), 'timer_id': 'timer:' + str(sequence), + 'timer_kind': 'condition_timeout', 'delay_seconds': 300}} + + def test_correlated_superseded_orphan_timer_is_distinct_from_the_current_tail(self): + events = self.report()['history'] + orphan = copy.deepcopy(events[0]) + orphan['event_type'] = 'ConditionWaitTimedOut' + events.insert(1, self.timer('TimerScheduled', 1)) + events[-3:-3] = [self.timer('TimerFired', 1), orphan] + self.assertEqual(packet.comparator.check(self.snapshots(events))['outcome'], 'pass') + + def test_timer_terminals_require_prior_schedule_and_row_ordered_supersession(self): + for label in ('missing-schedule', 'current-tail', 'before-reopen', 'duplicate-fire', 'wrong-delay'): + events = self.report()['history'] + if label == 'current-tail': + events.insert(5, self.timer('TimerScheduled', 3)) + events.insert(-1, self.timer('TimerFired', 3)) + else: + if label != 'missing-schedule': + events.insert(1, self.timer('TimerScheduled', 1)) + fired = self.timer('TimerFired', 1) + if label == 'wrong-delay': + fired['payload']['delay_seconds'] = 299 + events.insert(2 if label == 'before-reopen' else -1, fired) + if label == 'duplicate-fire': + events.insert(-1, copy.deepcopy(fired)) + report = self.snapshots(events) + with self.subTest(label=label), self.assertRaises(ValueError): + packet.comparator.check(report) + + +if __name__ == "__main__":unittest.main() diff --git a/scripts/acceptance/condition-history/test_resources.py b/scripts/acceptance/condition-history/test_resources.py new file mode 100644 index 0000000..66d88fd --- /dev/null +++ b/scripts/acceptance/condition-history/test_resources.py @@ -0,0 +1,230 @@ +"""Adversarial effective-cap/collision controls with a fake Docker CLI only.""" +import ast +import copy +import subprocess +import importlib.util +import json +import pathlib +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('resource_controls', HERE / 'resources.py') +resources = importlib.util.module_from_spec(spec) +spec.loader.exec_module(resources) +IMAGE = 'fixture@sha256:' + 'a' * 64 + +class Frozen: + sha256 = 'b' * 64 + def __init__(self):self.program = (HERE / 'role-supervisor.sh').read_bytes() + def verify(self):pass + def read(self, path):return self.program + +class Resources(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.root = pathlib.Path(self.tmp.name) + self.args = types.SimpleNamespace(work_dir=str(self.root / 'unused'), run_token='0123456789abcdef', + docker_endpoint='unix:///synthetic.sock', active_unix=int(time.time()) + 1740) + self.controller = resources.Docker(self.args, Frozen(), {}) + self.image = {'sha256': 'c' * 64, 'raw': {'Id': 'image-id', 'Os': 'linux', 'Architecture': 'amd64', + 'RootFS': {'Layers': []}, 'Config': {'Entrypoint': ['/entry'], 'Cmd': ['default'], + 'Env': ['PATH=/public'], 'WorkingDir': '', 'Volumes': {}}}} + self.controller.images[IMAGE] = self.image + self.controller.images['image-id'] = self.image + + def role(self, role='consumer'): + return self.controller.profile(role, IMAGE, ['explicit'], '65534:65534', 'none', self.root / 'fixture', + self.root / 'state', env={'CARGO_BUILD_JOBS': '2'} if role in ('fetch', 'build') else {}, + deadline=int(time.time()) + 500) + + def frame(self, role): + return {'Name': '/' + role['name'], 'Id': 'd' * 64, 'Image': 'image-id', 'State': {'Running': True, 'Pid': 44}, + 'Config': {'Image': IMAGE, 'User': role['user'], 'Entrypoint': ['/bin/sh'], 'Cmd': role['argv'], + 'Env': ['PATH=/public'] + [k + '=' + v for k, v in role['explicit_env'].items()], + 'Labels': self.controller.labels(role['role']), + 'WorkingDir': '/work' if role['role'] in ('fetch', 'build') else ''}, + 'HostConfig': {'NanoCpus': role['cpus'], 'Memory': role['memory'], 'MemorySwap': role['memory'], + 'PidsLimit': role['pids'], 'ReadonlyRootfs': role['readonly'], 'Init': True, 'Privileged': False, + 'CapDrop': ['ALL'], 'CapAdd': role['cap_add'], 'SecurityOpt': ['no-new-privileges'], + 'NetworkMode': role['network'], 'Tmpfs': role['tmpfs'], 'PortBindings': {}, + 'PublishAllPorts': False, 'Devices': [], 'VolumesFrom': []}, + 'Mounts': [{'Type': 'bind', 'Source': s, 'Destination': d, 'RW': rw} for s, d, rw in role['binds']] + [{'Type':'tmpfs', 'Destination':d} for d in role['tmpfs']], + 'NetworkSettings': {'Networks': {role['network']: {'Aliases': [role['alias']] if role['alias'] else []}}}} + + def network_frame(self): + return {'Id': 'e' * 64, 'Name': self.controller.task, 'Driver': 'bridge', 'Scope': 'local', + 'Internal': True, 'Attachable': False, 'Ingress': False, 'EnableIPv4': True, 'EnableIPv6': False, + 'ConfigOnly': False, 'ConfigFrom': {'Network': ''}, 'Labels': self.controller.labels('network'), + 'IPAM': copy.deepcopy(resources.NETWORK_IPAM), 'Options': dict(resources.NETWORK_OPTIONS), + 'Containers': {'d' * 64: {}}} + + def network(self): + return types.SimpleNamespace(stdout=json.dumps([self.network_frame()])) + + def test_actual_held_build_has_no_swap_cap(self): + repo=HERE.parents[2] + old=subprocess.check_output(['git','--no-replace-objects','-C',str(repo),'show', + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'],text=True) + node=next(n for n in ast.parse(old).body if isinstance(n,ast.FunctionDef) and n.name=='build') + ledger=[] + namespace={'remaining':lambda args:100,'os':__import__('os'),'RUST':IMAGE, + 'command':lambda args,line,**kw:ledger.append(line) or types.SimpleNamespace(stdout='',stderr=''), + 'native':types.SimpleNamespace(remove_owned_container=lambda *a:None)} + exec(compile(ast.Module(body=[node],type_ignores=[]),'held-f155-build','exec'),namespace) + owner=types.SimpleNamespace(base='old',labels=lambda name:[],best_effort=lambda *a:None) + namespace['build'](self.args,self.root,owner,'never-executed','build') + self.assertIn('--memory',ledger[0]);self.assertNotIn('--memory-swap',ledger[0]) + new=self.controller.create_arguments(self.role('build')) + self.assertEqual(new[new.index('--memory')+1],new[new.index('--memory-swap')+1]) + + def test_every_role_has_literal_equal_swap_and_full_caps(self): + expected = {'fetch': (3 * 1024**3, 1024), 'build': (3 * 1024**3, 1024), + 'mysql': (1280 * 1024**2, 512), 'redis': (128 * 1024**2, 256), + 'bootstrap': (768 * 1024**2, 256), 'runtime': (768 * 1024**2, 256), + 'queue': (512 * 1024**2, 256), 'consumer': (768 * 1024**2, 256)} + for name, (memory, pids) in expected.items(): + role = self.role(name);argv = self.controller.create_arguments(role) + self.assertEqual(argv[argv.index('--memory') + 1], str(memory)) + self.assertEqual(argv[argv.index('--memory-swap') + 1], str(memory)) + self.assertEqual(argv[argv.index('--pids-limit') + 1], str(pids)) + self.assertIn('--init', argv);self.assertIn('no-new-privileges', argv) + with self.subTest(role=name), mock.patch.object(self.controller, 'call', return_value=self.network()): + self.assertEqual(self.controller.validate(role, self.frame(role))['memory_swap'], memory) + + def test_each_bad_effective_field_refuses_including_offline_build(self): + for name in ('consumer', 'build'): + role = self.role(name) + changes = [('HostConfig', 'MemorySwap', role['memory'] * 2), ('HostConfig', 'Memory', 0), + ('HostConfig', 'NanoCpus', 0), ('HostConfig', 'PidsLimit', -1), + ('HostConfig', 'ReadonlyRootfs', not role['readonly']), ('HostConfig', 'Init', False), + ('HostConfig', 'Privileged', True), ('HostConfig', 'CapAdd', ['SYS_ADMIN']), + ('HostConfig', 'CapDrop', []), ('HostConfig', 'SecurityOpt', []), + ('HostConfig', 'NetworkMode', 'host'), ('HostConfig', 'PortBindings', {'80/tcp': [{}]}), + ('HostConfig', 'PidMode', 'container:foreign'), ('HostConfig', 'IpcMode', 'container:foreign'), + ('HostConfig', 'UTSMode', 'host'), ('HostConfig', 'DeviceRequests', [{'Count': -1}]), + ('HostConfig', 'DeviceCgroupRules', ['a *:* rwm']), + ('HostConfig', 'Tmpfs', {}), ('Config', 'User', '0:0'), ('Config', 'Cmd', ['foreign']), + ('Config', 'Entrypoint', ['foreign']), ('Config', 'Env', ['CARGO_BUILD_JOBS=99']), + ('Config', 'Image', 'foreign'), ('Config', 'WorkingDir', '/foreign'), ('Config', 'Labels', {}), + ('State', 'Running', False), ('State', 'Pid', 0)] + for section, key, value in changes: + frame = copy.deepcopy(self.frame(role));frame[section][key] = value + with self.subTest(role=name, field=key), mock.patch.object(self.controller, 'call', return_value=self.network()), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + + def test_internal_network_and_mount_authority_are_effective_checks(self): + role = self.role() + frame = self.frame(role) + network = self.network_frame();network['Internal'] = False + with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + for mutate in (lambda f: f['Mounts'].append({'Type': 'volume', 'Destination': '/extra'}), + lambda f: f['Mounts'].pop(), + lambda f: f.update(Name='/foreign'), + lambda f: f['Mounts'][0].update(RW=True), + lambda f: f['Mounts'][0].update(Source='/foreign'), + lambda f: f['NetworkSettings'].update(Networks={'foreign': {}})): + frame = copy.deepcopy(self.frame(role));mutate(frame) + with self.subTest(mutate=mutate), mock.patch.object(self.controller, 'call', return_value=self.network()), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + + def test_no_child_or_credential_gate_opens_before_readback(self): + role = self.role();frame = self.frame(role);frame['HostConfig']['MemorySwap'] *= 2 + ledger = [] + def cli(line, **kwargs): + ledger.append(line) + return types.SimpleNamespace(stdout=json.dumps([frame]) if line[:2] == ['docker', 'inspect'] else '') + with mock.patch.object(self.controller, 'call', cli), self.assertRaises(SystemExit): + self.controller.start(role, token='synthetic-only', key='synthetic-only') + self.assertFalse(any(line[:2] == ['docker', 'exec'] for line in ledger)) + self.assertEqual(self.controller.receipts, []) + + def test_valid_readback_precedes_private_credential_release(self): + role = self.role();frame = self.frame(role) + ledger = [] + def cli(line, **kwargs): + ledger.append((line, kwargs)) + if line[:3] == ['docker', 'network', 'inspect']: + return self.network() + if line[:2] == ['docker', 'inspect']: + return types.SimpleNamespace(stdout=json.dumps([frame])) + return types.SimpleNamespace(stdout='') + with mock.patch.object(self.controller, 'call', cli): + self.assertEqual(self.controller.start(role, token='synthetic-only'), role['name']) + self.assertEqual(len(self.controller.receipts), 1) + self.assertEqual(ledger[-1][0][:3], ['docker', 'exec', '-i']) + self.assertEqual(ledger[-1][1]['input'], 'synthetic-only\n\n') + self.assertLess(next(i for i, (line, _) in enumerate(ledger) if line[:2] == ['docker', 'inspect']), len(ledger) - 1) + + def test_compile_input_and_replay_input_mounts_are_readonly(self): + build = self.role('build');consumer = self.role('consumer') + self.assertIn((str(self.root / 'fixture'), '/work', False), build['binds']) + self.assertIn((str(self.root / 'fixture'), '/work', False), consumer['binds']) + self.assertEqual({m[1] for m in build['binds'] if m[2]}, {'/work/.cargo', '/work/target'}) + self.assertEqual(self.role('fetch')['network'], 'bridge') + self.assertEqual(build['network'], 'none') + + def test_wrong_platform_and_image_content_refuse(self): + controller = resources.Docker(self.args, Frozen(), {}) + raw = copy.deepcopy(self.image['raw']);raw['Architecture'] = 'arm64' + with mock.patch.object(controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([raw]))), self.assertRaises(SystemExit): + controller.image(IMAGE) + role = self.role();self.controller.images['image-id'] = dict(self.image, sha256='foreign') + frame = self.frame(role) + with self.assertRaises(SystemExit):self.controller.validate(role, frame) + + def test_any_owner_or_exact_name_collision_precedes_mutation(self): + for collide in ('container', 'network', 'volume', 'foreign-name', 'uncertain'): + ledger = [] + def cli(args, line, **kwargs): + ledger.append(line) + family = 'container' if line[1] == 'ps' else 'network' if line[1:3] == ['network', 'ls'] else 'volume' + if 'inspect' not in line: + return types.SimpleNamespace(stdout='existing' if family == collide else '', returncode=0, stderr='') + return types.SimpleNamespace(stdout='existing' if collide == 'foreign-name' else '', + returncode=0 if collide == 'foreign-name' else 1, + stderr='daemon unavailable' if collide == 'uncertain' else 'No such object') + with self.subTest(collide=collide), mock.patch.object(resources, 'call', cli), self.assertRaises(SystemExit): + resources.preflight_collisions(self.args) + self.assertFalse(pathlib.Path(self.args.work_dir).exists()) + self.assertFalse(any(x in line for line in ledger for x in ('create', 'pull', 'start', 'rm'))) + + def test_complete_network_profile_and_ipam_are_read_back(self): + role = self.role();frame = self.frame(role) + cases = [('Driver', 'overlay'), ('Scope', 'swarm'), ('Internal', False), ('Attachable', True), + ('Ingress', True), ('EnableIPv4', False), ('EnableIPv6', True), ('ConfigOnly', True), + ('ConfigFrom', {'Network': 'foreign'}), ('Options', {}), ('Labels', {}), + ('IPAM', dict(resources.NETWORK_IPAM, Driver='foreign')), + ('IPAM', dict(resources.NETWORK_IPAM, Config=[{'Subnet': '10.0.0.0/8'}])), + ('IPAM', dict(resources.NETWORK_IPAM, Options={'foreign': 'true'})), + ('Containers', {'d' * 64: {}, 'f' * 64: {}})] + for key, value in cases: + network = self.network_frame();network[key] = value + with self.subTest(key=key, value=value), mock.patch.object(self.controller, 'call', + return_value=types.SimpleNamespace(stdout=json.dumps([network]))), self.assertRaises(SystemExit): + self.controller.validate(role, frame) + args = self.controller.network_arguments() + self.assertIn('--ipv4=true', args);self.assertIn('--ipv6=false', args) + self.assertEqual(args[args.index('--subnet') + 1], '172.30.254.0/28') + + def test_held_network_check_accepts_foreign_driver_scope_and_ipam(self): + source = subprocess.check_output(['git', '--no-replace-objects', '-C', str(HERE.parents[2]), 'show', + 'e309116cb0dd79e415ae6a25b809515354cea026:scripts/acceptance/condition-history/resources.py']) + old = types.ModuleType('held_network_resource_controls') + exec(compile(source, 'held-e309-resources', 'exec'), old.__dict__) + controller = old.Docker(self.args, Frozen(), {}) + controller.images = dict(self.controller.images) + role = self.role();frame = self.frame(role) + network = self.network_frame();network.update(Driver='overlay', Scope='swarm', IPAM={'Driver': 'foreign'}) + with mock.patch.object(controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))): + self.assertEqual(controller.validate(role, frame)['container_id'], frame['Id']) + with mock.patch.object(self.controller, 'call', return_value=types.SimpleNamespace(stdout=json.dumps([network]))), \ + self.assertRaisesRegex(SystemExit, 'network profile'): + self.controller.validate(role, frame) + + +if __name__ == '__main__':unittest.main() diff --git a/scripts/acceptance/condition-history/test_subject.py b/scripts/acceptance/condition-history/test_subject.py new file mode 100644 index 0000000..55a6dab --- /dev/null +++ b/scripts/acceptance/condition-history/test_subject.py @@ -0,0 +1,404 @@ +"""Closed subject/record controls. No Docker, Cargo or network is executed.""" +import ast +import copy +import hashlib +import importlib.util +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +REPO = HERE.parents[2] + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + +subject = load('sealed_subject', HERE / 'subject.py') +recovery = load('subject_recovery_tests', HERE / 'recover.py') + + +def fixture_subject(root): + data = {p: (REPO / p).read_bytes() for p in subject.RUNTIME_FILES} + files = {p: {'git_blob': 'a' * 40, 'mode': '100644', 'bytes': len(v), 'sha256': subject.digest(v)} + for p, v in data.items()} + value = {'schema': 'durable-workflow.condition-subject/v1', 'runner_commit': 'a' * 40, + 'package_commit': subject.H, 'files': files} + sealed = subject.Subject(value, data, root / 'snapshot') + sealed.materialize() + return sealed + + +def remove_private(root): + for parent, dirs, files in os.walk(root, followlinks=False): + os.chmod(parent, 0o700) + for name in dirs: + p = pathlib.Path(parent) / name + if not p.is_symlink():p.chmod(0o700) + shutil.rmtree(root) + + +class SubjectControls(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(remove_private, self.root) + self.sealed = fixture_subject(self.root) + + def old_function(self, name, namespace): + # This ancestor preserves the held f155 run.py blob and remains reachable + # in ordinary clones after the original review branches are removed. + old = subprocess.check_output(['git', '--no-replace-objects', '-C', str(REPO), 'show', + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'], text=True) + node = next(n for n in ast.parse(old).body if isinstance(n, ast.FunctionDef) and n.name == name) + exec(compile(ast.Module(body=[node], type_ignores=[]), 'held-f155-function', 'exec'), namespace) + return namespace[name] + + def test_actual_held_export_function_admits_live_edit_restore(self): + live = self.root / 'live';(live / 'fixture').mkdir(parents=True) + source = live / 'fixture/marker';source.write_bytes(b'reviewed') + old_export = self.old_function('export_consumer', {'shutil':shutil,'HERE':live,'INPUTS':{}}) + source.write_bytes(b'unreviewed') + old_export(self.root / 'old-export') + source.write_bytes(b'reviewed') + self.assertEqual((self.root / 'old-export/marker').read_bytes(), b'unreviewed') + output = self.root / 'new-export' + self.sealed.export(subject.PREFIX+'fixture/', output) + self.assertEqual((output/'tests/conditions.rs').read_bytes(), + self.sealed.captured[subject.PREFIX+'fixture/tests/conditions.rs']) + + def test_actual_held_recovery_accepts_missing_source_and_deadline_authority(self): + path = self.root/'old-record.json' + path.write_text(json.dumps({'owner':recovery.PREFIX,'token':'0123456789abcdef','cells':[recovery.CELL]})) + calls=[] + native=types.SimpleNamespace(confined_child=lambda *a:path,Run=lambda *a,**k:None, + recover_run=lambda *a:calls.append(a) or {'pass':True}) + old_recover=self.old_function('recover',{'json':json,'re':__import__('re'),'native':native, + 'PREFIX':recovery.PREFIX,'CELL':recovery.CELL,'require':subject.require}) + self.assertEqual(old_recover(self.root,'0123456789abcdef'),{'pass':True}) + self.assertEqual(len(calls),1) + incomplete=json.loads(path.read_text()) + with self.assertRaises(SystemExit):self.validate(incomplete) + + def test_actual_held_summary_statement_overwrites_an_existing_handoff(self): + old = subprocess.check_output(['git','--no-replace-objects','-C',str(REPO),'show', + '67a37f7f0e42ba63ee4e68cae3090a9ba73e1cc2:scripts/acceptance/condition-history/run.py'],text=True) + node=next(n for n in ast.walk(ast.parse(old)) if isinstance(n,ast.Call) + and isinstance(n.func,ast.Attribute) and n.func.attr=='write_bytes' + and isinstance(n.func.value,ast.BinOp) and isinstance(n.func.value.right,ast.Constant) + and n.func.value.right.value=='summary.json') + path=self.root/'summary.json';path.write_bytes(b'prior-handoff') + eval(compile(ast.Expression(node),'held-f155-handoff','eval'), + {'args':types.SimpleNamespace(work_dir=self.root),'clean':b'overwritten'}) + self.assertEqual(path.read_bytes(),b'overwritten'+b'\n') + with self.assertRaises(FileExistsError):subject.exclusive(path,b'new') + self.assertEqual(path.read_bytes(),b'overwritten'+b'\n') + + def test_post_preflight_live_replacement_and_restore_never_changes_export(self): + live = self.root / 'live.rs' + path = subject.PREFIX + 'fixture/tests/conditions.rs' + reviewed = self.sealed.captured[path] + live.write_bytes(reviewed) + # Old source copy reads the changed live path and admits it. + live.write_bytes(b'unreviewed replacement') + old_copy = self.root / 'old-copy.rs' + shutil.copyfile(live, old_copy) + live.write_bytes(reviewed) + self.assertNotEqual(old_copy.read_bytes(), reviewed) + output = self.root / 'new-copy' + self.sealed.export(subject.PREFIX + 'fixture/', output) + self.assertEqual((output / 'tests/conditions.rs').read_bytes(), reviewed) + + def test_source_replacement_is_detected_before_module_use(self): + p = self.sealed.root / (subject.PREFIX + 'engine.py') + p.chmod(0o600) + p.write_bytes(b'raise SystemExit("changed")');p.chmod(0o400) + with self.assertRaisesRegex(SystemExit, 'changed'): + self.sealed.module('engine') + + def test_extra_importable_missing_link_and_nonregular_members_refuse(self): + for kind in ('extra', 'empty-dir', 'missing', 'symlink', 'hardlink', 'fifo'): + root = self.root / kind + root.mkdir() + sealed = fixture_subject(root) + target = sealed.root / (subject.PREFIX + 'engine.py') + target.parent.chmod(0o700) + if kind == 'extra': + (target.parent / 'sitecustomize.py').write_text('raise RuntimeError()') + elif kind == 'empty-dir':(target.parent / 'unreviewed').mkdir() + elif kind == 'missing':target.unlink() + elif kind == 'symlink': + target.unlink();target.symlink_to(self.root / 'foreign') + elif kind == 'hardlink':os.link(target, self.root / 'alias') + else: + target.unlink();os.mkfifo(target) + with self.subTest(kind=kind), self.assertRaises((SystemExit, OSError)): + sealed.verify() + + def test_captured_recovery_is_not_reopened_from_a_mutable_path(self): + path = subject.PREFIX + 'recover.py' + code = self.sealed.read(path) + command = [sys.executable, '-I', '-S', '-c', code.decode()] + target = self.sealed.root / path + target.chmod(0o600);target.write_bytes(b'print("foreign")') + self.assertEqual(subject.digest(command[-1].encode()), self.sealed.value['files'][path]['sha256']) + self.assertNotEqual(command[-1].encode(), target.read_bytes()) + with self.assertRaises(SystemExit):self.sealed.read(path) + + def test_closed_loader_refuses_any_other_module(self): + with self.assertRaises(SystemExit):self.sealed.module('sitecustomize') + + def test_exclusive_summary_handoff_rejects_existing_file_and_redirect(self): + data = b'{"checks_outcome":"pass"}\n' + path = self.root / 'summary.json' + sha = subject.exclusive(path, data) + self.assertEqual(sha, subject.digest(data)) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + with self.assertRaises(FileExistsError):subject.exclusive(path, b'changed') + foreign = self.root / 'foreign';foreign.write_bytes(b'preserved') + link = self.root / 'summary-link';link.symlink_to(foreign) + with self.assertRaises(FileExistsError):subject.exclusive(link, b'changed') + self.assertEqual(foreign.read_bytes(), b'preserved') + + def record(self): + now = int(time.time()) + return {'schema': 'durable-workflow.condition-run/v4', 'owner': recovery.PREFIX, + 'token': '0123456789abcdef', 'cells': [recovery.CELL], 'runner_commit': 'a' * 40, + 'package_commit': subject.H, 'manifest_sha256': self.sealed.sha256, + 'recovery_sha256': self.sealed.value['files'][subject.PREFIX + 'recover.py']['sha256'], + 'started_unix': now, 'active_unix': now + 1740, 'cleanup_unix': now + 1800, + 'work_dir': str(self.root), 'docker_endpoint': 'unix:///synthetic.sock', + 'uid': os.getuid(), 'gid': os.getgid(), + 'storage': {'schema': 'durable-workflow.condition-guest-storage/v1', 'guest_boot_id': '11111111-1111-1111-1111-111111111111', 'device': '8:1', 'block_bytes': 20 * 1024**3, 'filesystem_bytes': 19 * 1024**3, 'docker_root': '/synthetic/docker', 'daemon_id': 'synthetic-daemon-id'}, + 'owner_process': {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'boot_id': '11111111-1111-1111-1111-111111111111', 'start_ticks': 123456}} + + def validate(self, record, expected_sha=None): + path = self.root / 'run.json' + if path.exists():path.unlink() + raw = subject.canonical(record) + subject.exclusive(path, raw) + return recovery.validate_record(self.root, expected_sha or subject.digest(raw), 'a' * 40, + self.sealed.sha256, self.record()['recovery_sha256'], '0123456789abcdef') + + def test_complete_recovery_authority(self): + record = self.record() + self.assertEqual(self.validate(record), record) + for field, value in [('extra', 1), ('package_commit', 'b' * 40), ('runner_commit', 'b' * 40), + ('manifest_sha256', 'b' * 64), ('recovery_sha256', 'b' * 64), ('cleanup_unix', 0), + ('started_unix', True), ('owner', 'other'), ('work_dir', str(self.root / 'other'))]: + bad = copy.deepcopy(record);bad[field] = value + with self.subTest(field=field), self.assertRaises(SystemExit):self.validate(bad) + missing = dict(record);del missing['active_unix'] + with self.assertRaises(SystemExit):self.validate(missing) + + def test_edit_of_record_does_not_gain_authority_by_recomputing_other_fields(self): + record = self.record();expected = subject.digest(subject.canonical(record)) + record['token'] = 'fedcba9876543210' + with self.assertRaises(SystemExit):self.validate(record, expected) + + def test_record_links_and_wrong_modes_refuse(self): + record = self.record();self.validate(record) + record_sha = subject.digest(subject.canonical(record)) + path = self.root / 'run.json' + path.chmod(0o644) + with self.assertRaises(SystemExit): + recovery.validate_record(self.root, record_sha, 'a' * 40, + self.sealed.sha256, record['recovery_sha256'], record['token']) + path.chmod(0o600);os.link(path, self.root / 'record-alias') + with self.assertRaises(SystemExit): + recovery.validate_record(self.root, record_sha, 'a' * 40, + self.sealed.sha256, record['recovery_sha256'], record['token']) + + def test_foreign_owner_or_renamed_object_is_never_removed(self): + record = self.record() + base = recovery.PREFIX + '-' + record['token'] + '-' + recovery.CELL + good = {'labels': {recovery.RUN: record['token'], recovery.OWNER: recovery.PREFIX, + recovery.SUBJECT: record['manifest_sha256'], recovery.ROLE: 'consumer'}, + 'name': '/' + base + '-consumer', 'id': 'a' * 64} + for field, value in [('owner', 'foreign'), ('subject', 'b' * 64), ('name', '/foreign')]: + frame = copy.deepcopy(good) + if field == 'name':frame['name'] = value + else:frame['labels'][recovery.OWNER if field == 'owner' else recovery.SUBJECT] = value + with self.subTest(field=field), mock.patch.object(recovery, 'docker', + return_value=types.SimpleNamespace(returncode=0, stdout=json.dumps(frame))) as cli, self.assertRaises(SystemExit): + recovery.owned(record, 'container', 'candidate') + self.assertEqual(cli.call_count, 1) + + def test_changed_pid_epoch_never_kills_a_foreign_group(self): + record = self.record();changed = dict(record['owner_process'], start_ticks=654321) + with mock.patch.object(recovery, 'process_identity', return_value=changed), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaises(SystemExit): + recovery.stop_owner(record) + kill.assert_not_called() + + def test_held_whole_second_identity_accepts_a_recycled_session(self): + source = subprocess.check_output(['git', '--no-replace-objects', '-C', str(REPO), 'show', + 'e309116cb0dd79e415ae6a25b809515354cea026:scripts/acceptance/condition-history/recover.py']) + old = types.ModuleType('held_second_resolution_recovery') + exec(compile(source, 'held-e309-recovery', 'exec'), old.__dict__) + # Both incarnations have identical numeric IDs and ps whole-second text. + old.subprocess = types.SimpleNamespace(run=lambda *a, **k: types.SimpleNamespace( + returncode=0, stdout='Thu Nov 5 01:00:00 2026')) + old.os = types.SimpleNamespace(getpgid=lambda pid: pid, getsid=lambda pid: pid, killpg=mock.Mock()) + old.group_members = lambda pgid: [] + legacy = old.process_identity(12345) + old.stop_owner({'owner_process': legacy}) + old.os.killpg.assert_called_once_with(12345, old.signal.SIGTERM) + record = self.record() + for delta in ({'start_ticks': record['owner_process']['start_ticks'] + 1}, + {'boot_id': '22222222-2222-2222-2222-222222222222'}): + recycled = dict(record['owner_process'], **delta) + with self.subTest(delta=delta), mock.patch.object(recovery, 'process_identity', return_value=recycled), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'epoch changed'): + recovery.stop_owner(record) + kill.assert_not_called() + + def test_proc_stat_reads_kernel_fields_after_a_parenthesized_comm(self): + fields = ['R', '1', '12345', '12345'] + ['0'] * 15 + ['123456'] + path = self.root / 'proc-stat' + path.write_bytes(('12345 (comm with ) spaces (and)) ' + ' '.join(fields)).encode()) + original_open = os.open + with mock.patch.object(recovery.os, 'open', side_effect=lambda path_, flags: original_open(path, flags)): + result = recovery.process_stat(12345) + self.assertEqual(result, {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'start_ticks': 123456}) + + def test_inconsistent_or_unavailable_kernel_authority_refuses(self): + first = {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'start_ticks': 123456} + boot = self.record()['owner_process']['boot_id'] + with mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=[first, dict(first, start_ticks=123457)]), \ + self.assertRaisesRegex(SystemExit, 'during observation'): + recovery.process_identity(12345) + with mock.patch.object(recovery, 'boot_identity', side_effect=[boot, '22222222-2222-2222-2222-222222222222']), \ + mock.patch.object(recovery, 'process_stat', return_value=first), \ + self.assertRaisesRegex(SystemExit, 'during observation'): + recovery.process_identity(12345) + for error in (PermissionError('denied'), OSError('proc unavailable')): + with self.subTest(error=type(error).__name__), mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=error), self.assertRaises(OSError): + recovery.process_identity(12345) + with mock.patch.object(recovery, 'boot_identity', return_value=boot), \ + mock.patch.object(recovery, 'process_stat', side_effect=FileNotFoundError()): + self.assertIsNone(recovery.process_identity(12345)) + + def test_epoch_revalidated_immediately_before_term_and_kill(self): + record = self.record();expected = record['owner_process'] + recycled = dict(expected, start_ticks=expected['start_ticks'] + 1) + with mock.patch.object(recovery, 'process_identity', side_effect=[expected, recycled]), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'before signal'): + recovery.stop_owner(record) + kill.assert_not_called() + with mock.patch.object(recovery, 'process_identity', side_effect=[expected, expected, recycled]), \ + mock.patch.object(recovery, 'group_members', return_value=[expected['pid']]), \ + mock.patch.object(recovery.time, 'monotonic', side_effect=[0, 10]), \ + mock.patch.object(recovery.os, 'killpg') as kill, self.assertRaisesRegex(SystemExit, 'before signal'): + recovery.stop_owner(record) + kill.assert_called_once_with(expected['pgid'], recovery.signal.SIGTERM) + + def test_exact_epoch_can_receive_both_signals_with_fresh_authority(self): + record = self.record();expected = record['owner_process'] + with mock.patch.object(recovery, 'process_identity', return_value=expected) as observe, \ + mock.patch.object(recovery, 'group_members', side_effect=[[expected['pid']], [expected['pid']], []]), \ + mock.patch.object(recovery.time, 'monotonic', side_effect=[0, 10]), \ + mock.patch.object(recovery.os, 'killpg') as kill: + recovery.stop_owner(record) + self.assertEqual(observe.call_count, 3) + self.assertEqual(kill.call_args_list, [mock.call(expected['pgid'], recovery.signal.SIGTERM), + mock.call(expected['pgid'], recovery.signal.SIGKILL)]) + + def test_legacy_process_record_cannot_authorize_a_signal(self): + record = self.record() + record['owner_process'] = {'pid': 12345, 'pgid': 12345, 'sid': 12345, 'started': 'whole second'} + with self.assertRaisesRegex(SystemExit, 'captured owner process'): + self.validate(record) + + @unittest.skipUnless(sys.platform == 'linux', 'Linux proc authority is the execution contract') + def test_readonly_current_kernel_identity(self): + identity = recovery.process_identity(os.getpid()) + self.assertEqual(identity['pid'], os.getpid()) + self.assertEqual(identity['pgid'], os.getpgid(os.getpid())) + self.assertEqual(identity['sid'], os.getsid(os.getpid())) + self.assertGreater(identity['start_ticks'], 0) + self.assertEqual(identity['boot_id'], recovery.boot_identity()) + + def storage_frame(self): + return {k: v for k, v in self.record()['storage'].items() if k not in ('schema', 'daemon_id')} + + def observe_storage(self, *, block_bytes=None, foreign_device=False, mount_text=None, boot=None): + device = os.makedev(8, 1) + capacity = block_bytes or recovery.GUEST_DISK_BYTES + def read_text(path, *args, **kwargs): + if str(path).startswith('/sys/dev/block/'): + return str(capacity // 512) + return mount_text or '42 1 8:1 / / rw,nodev,nosuid - ext4 /dev/sda1 rw\n' + def file_stat(path, *args, **kwargs): + return types.SimpleNamespace(st_dev=os.makedev(8, 2) if foreign_device and str(path) == '/synthetic/docker' else device) + with mock.patch.object(recovery, 'boot_identity', return_value=boot or self.record()['storage']['guest_boot_id']), \ + mock.patch.object(recovery.pathlib.Path, 'stat', autospec=True, side_effect=file_stat), \ + mock.patch.object(recovery.pathlib.Path, 'resolve', autospec=True, side_effect=lambda path, **kw: path), \ + mock.patch.object(recovery.pathlib.Path, 'read_text', autospec=True, side_effect=read_text), \ + mock.patch.object(recovery.os, 'statvfs', return_value=types.SimpleNamespace(f_blocks=19 * 1024**3 // 4096, f_frsize=4096)): + return recovery.filesystem_storage('/synthetic/work', '/synthetic/docker', self.record()['storage']['guest_boot_id']) + + def test_fixed_guest_block_device_bounds_work_and_daemon_storage(self): + self.assertEqual(self.observe_storage(), self.storage_frame()) + + def test_large_shared_device_foreign_mount_or_boot_refuses(self): + for changes in [dict(block_bytes=1024**4), dict(foreign_device=True), + dict(boot='22222222-2222-2222-2222-222222222222'), + dict(mount_text='42 1 8:1 / / rw - overlay overlay rw\n'), + dict(mount_text='42 1 8:1 / / rw - ext4 /dev/sda1 rw\n43 1 8:1 / /share rw - virtiofs host rw\n')]: + with self.subTest(changes=changes), self.assertRaises(SystemExit): + self.observe_storage(**changes) + + def test_storage_authority_is_closed_and_reobserved(self): + storage = self.record()['storage'] + with mock.patch.object(recovery, 'filesystem_storage', return_value=self.storage_frame()): + recovery.validate_storage(storage, '/synthetic/work') + for field, value in [('extra', 1), ('block_bytes', recovery.GUEST_DISK_BYTES + 512), + ('filesystem_bytes', True), ('daemon_id', ''), ('guest_boot_id', 'foreign')]: + bad = dict(storage, **{field: value}) + with self.subTest(field=field), self.assertRaises(SystemExit): + recovery.validate_storage(bad, '/synthetic/work', observe=False) + changed = dict(self.storage_frame(), device='8:2') + with mock.patch.object(recovery, 'filesystem_storage', return_value=changed), \ + self.assertRaisesRegex(SystemExit, 'storage authority changed'): + recovery.validate_storage(storage, '/synthetic/work') + + def test_guest_daemon_images_and_build_cache_must_start_empty(self): + for existing in (None, 'image', 'cache'): + ledger = [] + rows = [{'Type': name, 'TotalCount': '1' if existing == 'cache' and name == 'Build Cache' else '0'} + for name in ('Images', 'Containers', 'Local Volumes', 'Build Cache')] + def command(line): + ledger.append(line) + if line[1] == 'info': + data = json.dumps({'DockerRootDir': '/synthetic/docker', 'ID': 'synthetic-daemon-id'}) + elif line[1] == 'image': + data = 'sha256:existing' if existing == 'image' else '' + elif line[1] == 'system': + data = '\n'.join(json.dumps(row) for row in rows) + else: + data = '' + return types.SimpleNamespace(stdout=data) + args = types.SimpleNamespace(work_dir='/synthetic/work', guest_boot_id=self.record()['storage']['guest_boot_id']) + with mock.patch.object(recovery, 'filesystem_storage', return_value=self.storage_frame()): + if existing: + with self.subTest(existing=existing), self.assertRaises(SystemExit): + recovery.storage_preflight(args, command) + else: + self.assertEqual(recovery.storage_preflight(args, command), self.record()['storage']) + self.assertFalse(any(word in line for line in ledger for word in ('pull', 'create', 'build', 'start', 'rm'))) + + +if __name__ == '__main__':unittest.main()