diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 56b6261..2ac8c03 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -73,7 +73,22 @@ The runner holds these exact bytes and compares the actual retained Avro result with this known fixture. It never executes an external codec program or accepts its self-reported decoded result. -Run source with isolated host Python and the private directory in +Obtain the outer bootstrap from the frozen reviewed Git commit, independently +verify its SHA256 from the owning review receipt, then pass those literal bytes +to isolated Python with `-c`. Do not execute any dispatcher or bootstrap by a +host pathname. The bootstrap obtains the dispatcher from the same frozen Git +object and checks its separately fixed SHA256 before compile/exec. Host-path +substitution of the outer dispatcher cannot replace those Git bytes. + +```bash +reviewed_source_commit=FULL_REVIEWED_COMMIT_FROM_THE_PR +bootstrap="$(git --no-replace-objects show "$reviewed_source_commit:scripts/acceptance/command-keys/git_bootstrap.py")"$'\n' +expected_bootstrap_sha256=4b34ceb7b8c1366bc46405310bc4779df6a3ee63eb274af41868265028a5bc12 +actual_bootstrap_sha256="$(printf '%s' "$bootstrap" | sha256sum | awk '{print $1}')" +test "$actual_bootstrap_sha256" = "$expected_bootstrap_sha256" +``` + +Run from that repository with the private directory in `CLI_216_WORKDIR`. Provide a clean environment without cloud, GitHub, Composer or registry credentials. Runtime authentication is synthetic and private. @@ -81,7 +96,7 @@ or registry credentials. Runtime authentication is synthetic and private. env -i PATH=/usr/bin:/bin \ CLI_216_WORKDIR="$experiment" \ CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ - timeout 180s python3 -I -B scripts/acceptance/command-keys/cli_216.py + timeout 180s python3 -I -B -c "$bootstrap" "$reviewed_source_commit" ``` For the restart phase, preserve the first phase's `admissions.json` as @@ -91,7 +106,7 @@ records separately too. `CLI_216_PHASE=fresh-selected` adds direct fresh explicit-run literal-key markers (the loss cells intentionally withhold those initial response bodies). `CLI_216_PHASE=typed` runs the separate typed-intent cases. Use a **new** private -directory for `legacy_cli.py`, with the same staged inputs and environment. +directory with `CLI_216_RUNNER=legacy`, with the same staged inputs and environment. The legacy adapter generates its own admissions and stopped-copy database. An outer experiment owner must also enforce the timeout and remove only the @@ -121,3 +136,110 @@ replacement, a forged codec, sealed-content writes, foreign-name survival and schema-only mutations that the predecessor's row-only comparison accepted. Modern and legacy zero-write checks use the same bounded typed logical snapshot, including catalog SQL, columns, indexes, triggers, views and all table rows. + +## Physical 3.8 upgrade + +`run_cli_216.py` verifies and holds literal-hash modern, legacy or physical implementation bytes +before executing any implementation code. Its reported runner digest comes +from those checked bytes, with a post-cleanup identity recheck. Direct execution +of the implementation refuses before effectful initialization. It consumes the +producer-owned real 3.8 API seed after the +[accepted handoff](https://github.com/durable-workflow/server/pull/1210#issuecomment-6860138558). +It does not call the synthetic transform or seed a replacement baseline. +The accepted fixture contract is Server commit +`a09f156399416216d2f097c257a2164f83c7eda7`. The original seed actually executed +commit `09a4ccda3da11853606af031a8970bbde4b2d6ec`, driver SHA256 +`01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b`. +Keep these two provenance meanings distinct. + +Use another fresh private directory with the common staged inputs above and: + +| Input | Source/identity | +| --- | --- | +| `CLI_216_PHYSICAL_BASELINE` | Absolute original stopped-backup path supplied by its owner; SHA256 `7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c` | +| `CLI_216_PHYSICAL_PACKET` | Absolute original seed JSON path supplied by its owner; SHA256 `8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5` | +| `physical-contract.json` | The legacy manifest at accepted Server `a09f1563`; SHA256 `ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f` | +| `0013_command_intents.sql` | `rust/migrations/sqlite/0013_command_intents.sql` at published Server source `5614274d2629d03f92932a07dfee0fb10349ddd9`; SHA256 `fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd` | + +The original backup and packet are held read-only and rechecked after use. +Only a newly created owned copy is mounted into the published 3.9.2 image. +The image runs its actual `schema-upgrade` command, requiring SQLite 12 → 13. +The comparison preserves all old business rows and schema objects, allowing +only tagged DDL plus explicitly validated schema-version/journal metadata. +The intent table must be empty. The new journal checksum, description, +version, success, timestamp interval and exact integer-zero execution time are checked. + +```bash +env -i PATH=/usr/bin:/bin \ + CLI_216_WORKDIR="$experiment" \ + CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ + CLI_216_RUNNER=physical \ + CLI_216_PHYSICAL_BASELINE="$original_backup" \ + CLI_216_PHYSICAL_PACKET="$original_packet" \ + timeout 180s python3 -I -B -c "$bootstrap" "$reviewed_source_commit" +``` + +Created names are `cli-216-392-upgrade` and `cli-216-392-physical`; cleanup +authority remains their captured IDs. The upgrade container is retained until +its actual exit/log result is read, then removed. The runtime exercises the +installed CLI's original keyed-update readbacks and name/run fences for all +three old-run states. Original NULL-key signals are checked only for preserved +identity/history, never manufactured retained-key or intent equality. +Full request-window snapshots must agree. A later post-stop maintenance +snapshot is a different observation and must not be relabeled as that window. + +The comparison's offline controls require the staged helper and tagged SQL: + +```bash +CLI_216_TEST_HELPER="$experiment/sqlite_legacy.py" \ + CLI_216_TEST_DELTA="$experiment/0013_command_intents.sql" \ + python3 -I -B scripts/acceptance/command-keys/test_physical_cli.py -v +``` + +These controls are not a physical upgrade result. Source review, current +normal gates, fresh actual execution and cleanup still qualify that cell. + +The manual experiment holds owned mode-0700 root/data directories and a +single-link mode-0600 database descriptor throughout each phase. SQLite opens +through the held real disk-file descriptor, preserving normal WAL/SHM behavior +without reopening the mutable database name. +Docker acquires the guarded canonical data bind. A literal in-container shell +preflight checks the actual mounted directory and database device/inode, +UID/GID, mode and link count before executing the native binary. Host checks +immediately before/after acquisition and inspection retain their held identities. +The env-file and Docker config client use inherited `/proc/self/fd` descriptors; +HOME/config/tmp/bin consumer directories are separately held and inherited too. +Exact-ID cleanup uses that held Docker configuration even after a workspace +pathname failure. This requires the local Linux Docker daemon. +Identity checks immediately surround snapshots, container creation/inspection, +API and installed CLI calls, cleanup, and report emission. Reports are written +through held no-follow descriptors, then their path, parent, and contents are +rechecked before printing a terminal result. A swap fails the run; exact-ID +container cleanup remains available even after a workspace identity failure. + +The synthetic transform uses a separate stopped sibling copy, verifies its +complete result against the accepted helper's fingerprints, and copies those +bytes into the existing held runtime database inode. It does not rename a new +file over the database that later containers exercise. The physical upgrade +also mutates only that held copied database. Reports include root/data/database +identities so the parent can compare them across modern process restarts. +Offline workspace controls demonstrate predecessor pathname substitution and +require refusal for root/data/database swaps before upgrade or runtime start, +report path/parent swaps during emission, in-place report edits, symlinks and +extra hard links. A positive WAL control preserves actual mutable SQLite use. + +Opt-in actual-container boundary controls (separate from installed qualification): + +```bash +CLI_216_CONTAINER_CONTROLS=1 \ + python3 -I -B scripts/acceptance/command-keys/test_workspace_container.py -v +``` + +These use five sequential `cli-152-boundary-*` containers with the same pinned +native image, one CPU, 512 MiB, 128 PIDs, no network or ports. Directory/file +substitution must exit 78 before native execution. The positive mount control +executes the native `schema-check` command and expects its SQLite-unsupported +diagnostic, proving the preflight reached the native binary without starting a +runtime or upgrading a database. Env/config controls use the real Docker reader +and retain exact-ID cleanup after substitution. Check host capacity first and +set TMPDIR to owned disk-backed scratch. No image build or credentials are used. diff --git a/scripts/acceptance/command-keys/boundaries.py b/scripts/acceptance/command-keys/boundaries.py index 2680401..33edff1 100644 --- a/scripts/acceptance/command-keys/boundaries.py +++ b/scripts/acceptance/command-keys/boundaries.py @@ -87,12 +87,247 @@ def private_directory(path): return path +# Adapted from sdk-python 377a3eee workspace_guards.py. The qualified +# image checks the actual acquired bind before executing its native binary. +NATIVE_PREFLIGHT = """set -eu +test -d "$1" && test ! -L "$1" || exit 78 +test -f "$1/$2" && test ! -L "$1/$2" || exit 78 +test "$(stat -c '%d:%i:%u:%g:%a:%h' -- "$1")" = "$3" || exit 78 +test "$(stat -c '%d:%i:%u:%g:%a:%h' -- "$1/$2")" = "$4" || exit 78 +shift 4 +exec /usr/local/bin/durable-workflow-server "$@" +""" + + +class HeldDirectory: + """Keep an owned directory inode, including its no-follow parent binding.""" + def __init__(self, path, parent=None): + self.path, self.parent = Path(path), parent + if not self.path.is_absolute() or self.path.resolve() != self.path: + raise ValueError("directory must be canonical without symlinks") + if parent is not None: + parent.recheck() + if self.path.parent != parent.path: + raise ValueError("held directory must be a direct child") + self.fd = os.open(self.path if parent is None else self.path.name, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, + dir_fd=None if parent is None else parent.fd) + self.identity = self.key(os.fstat(self.fd)) + try: + self.recheck() + except BaseException: + self.close() + raise + + @staticmethod + def key(value): + return (value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, value.st_nlink) + + def recheck(self): + if self.parent is not None: + self.parent.recheck() + value = os.fstat(self.fd) + named = os.stat(self.path if self.parent is None else self.path.name, + dir_fd=None if self.parent is None else self.parent.fd, follow_symlinks=False) + if (not stat.S_ISDIR(value.st_mode) or value.st_uid != os.getuid() + or stat.S_IMODE(value.st_mode) != 0o700 or value.st_nlink < 2 + or self.key(value) != self.identity or self.key(named) != self.identity + or self.path.resolve() != self.path or self.key(self.path.lstat()) != self.identity): + raise ValueError("held directory identity changed") + + @property + def anchor(self): + return "/proc/" + str(os.getpid()) + "/fd/" + str(self.fd) + + def close(self): + os.close(self.fd) + + +class HeldWritableFile: + """Mutable SQLite data or a frozen report, always tied to one parent/inode.""" + def __init__(self, parent, name, *, create=False, maximum=64 * 1024 * 1024): + if Path(name).name != name or name in (".", ".."): + raise ValueError("held file requires one filename") + self.parent, self.name, self.maximum, self.digest = parent, name, maximum, None + parent.recheck() + flags = os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + if create: + flags |= os.O_CREAT | os.O_EXCL + self.fd = os.open(name, flags, 0o600, dir_fd=parent.fd) + self.identity = HeldDirectory.key(os.fstat(self.fd)) + try: + self.recheck() + except BaseException: + self.close() + raise + + def recheck(self): + self.parent.recheck() + value = os.fstat(self.fd) + named = os.stat(self.name, dir_fd=self.parent.fd, follow_symlinks=False) + if (not stat.S_ISREG(value.st_mode) or value.st_uid != os.getuid() or value.st_nlink != 1 + or stat.S_IMODE(value.st_mode) != 0o600 or not 0 <= value.st_size <= self.maximum + or HeldDirectory.key(value) != self.identity or HeldDirectory.key(named) != self.identity): + raise ValueError("held writable file identity changed") + if self.digest is not None and hashlib.sha256(os.pread(self.fd, self.maximum + 1, 0)).hexdigest() != self.digest: + raise ValueError("held report content changed") + + @property + def anchor(self): + return "/proc/" + str(os.getpid()) + "/fd/" + str(self.fd) + + def write(self, data, *, freeze=False): + self.recheck() + if len(data) > self.maximum: + raise ValueError("held file byte bound exceeded") + self.digest = None + os.ftruncate(self.fd, 0) + offset = 0 + while offset < len(data): + count = os.pwrite(self.fd, data[offset:], offset) + if count <= 0: + raise OSError("held file write made no progress") + offset += count + os.fsync(self.fd) + self.digest = hashlib.sha256(data).hexdigest() if freeze else None + self.recheck() + + def close(self): + os.close(self.fd) + + +class Workspace: + """Bind comparison, Docker data, and terminal evidence to held objects.""" + def __init__(self, root): + for name in ("home", "config", "tmp", "docker-empty", "bin"): + private_directory(Path(root) / name) + self.root = HeldDirectory(root) + self.data = HeldDirectory(Path(root) / "data", self.root) + self.files = {} + self.children = {name: HeldDirectory(Path(root) / name, self.root) + for name in ("home", "config", "tmp", "docker-empty", "bin")} + try: + # The first phase creates the file; later phases hold the existing + # inode. Server mutates it across restarts and schema upgrades. + exists = os.path.lexists(Path(root) / "data/database.sqlite") + self.database = HeldWritableFile(self.data, "database.sqlite", create=not exists) + self.recheck() + except BaseException: + self.data.close() + self.root.close() + raise + + def recheck(self): + self.root.recheck() + self.data.recheck() + self.database.recheck() + for value in self.children.values(): + value.recheck() + for value in self.files.values(): + value.recheck() + + def evidence(self): + self.recheck() + return {"root": list(self.root.identity), "data": list(self.data.identity), + "database": list(self.database.identity)} + + def connection(self, *, readonly=True): + from contextlib import contextmanager + import sqlite3 + + @contextmanager + def opened(): + self.recheck() + # The real disk descriptor binds SQLite's opened main inode. + # PRAGMA database_list followed by stat would reopen a name and + # could accept swap-away/open-substitute/swap-back. SQLite retains + # normal disk-backed WAL/SHM behavior through this descriptor. + uri = "file:" + self.database.anchor + "?mode=" + ("ro" if readonly else "rw") + db = sqlite3.connect(uri, uri=True, timeout=2) + try: + self.recheck() + yield db + finally: + db.close() + self.recheck() + return opened() + + def mount_arguments(self): + self.recheck() + return ("--mount", "type=bind,source=" + str(self.data.path) + ",target=/app/database") + + def native_arguments(self, *arguments): + self.recheck() + def identity(descriptor): + value = os.fstat(descriptor) + return f"{value.st_dev}:{value.st_ino}:{value.st_uid}:{value.st_gid}:{stat.S_IMODE(value.st_mode):o}:{value.st_nlink}" + return ("-c", NATIVE_PREFLIGHT, "checked-native", "/app/database", "database.sqlite", + identity(self.data.fd), identity(self.database.fd), *arguments) + + def inspect_mounts(self, value): + self.recheck() + matches = [item for item in value["Mounts"] if item["Destination"] == "/app/database"] + if (len(matches) != 1 or matches[0]["Type"] != "bind" or matches[0]["Source"] != str(self.data.path) + or matches[0]["RW"] is not True): + raise ValueError("Docker did not acquire the guarded canonical data bind") + self.recheck() + + def child_anchor(self, name): + return self.children[name].anchor + + def inherited(self): + return tuple(item.fd for item in [*self.children.values(), *self.files.values()]) + + def read(self, name): + self.recheck() + if name not in self.files: + self.files[name] = HeldWritableFile(self.root, name, maximum=32 * 1024 * 1024) + item = self.files[name] + item.recheck() + data = os.pread(item.fd, item.maximum + 1, 0) + item.digest = hashlib.sha256(data).hexdigest() + self.recheck() + return data + + def write(self, name, data): + self.recheck() + if name not in self.files: + exists = os.path.lexists(self.root.path / name) + self.files[name] = HeldWritableFile(self.root, name, create=not exists, maximum=32 * 1024 * 1024) + self.files[name].write(data, freeze=True) + self.recheck() + + def unlink(self, name): + # Unlink only the captured entry through its held parent; root-path + # refusal must not prevent credential cleanup or touch a replacement. + item = self.files.get(name) + if item is not None: + value = os.stat(item.name, dir_fd=item.parent.fd, follow_symlinks=False) + if HeldDirectory.key(value) != item.identity or HeldDirectory.key(os.fstat(item.fd)) != item.identity: + raise ValueError("cleanup entry was replaced; preserving it") + os.unlink(name, dir_fd=self.root.fd) + item.close() + del self.files[name] + + def close(self): + for value in self.files.values(): + value.close() + self.database.close() + for value in self.children.values(): + value.close() + self.data.close() + self.root.close() + + class OwnedContainer: - def __init__(self, docker, name): + def __init__(self, docker, name, workspace=None): self.docker, self.name, self.id = docker, name, None + self.workspace = workspace self.created_ids = [] def run(self, *arguments): + if self.workspace is not None: + self.workspace.recheck() if self.exists() or self.docker("ps", "-aq", "--filter", "name=^" + self.name + "$"): raise ValueError("container name already exists; no cleanup authority acquired") created = self.docker("run", *arguments) @@ -105,19 +340,26 @@ def run(self, *arguments): def exists(self): return self.id is not None and self.docker("ps", "-aq", "--no-trunc", "--filter", "id=" + self.id) == self.id - def inspect(self): + def inspect(self, *, check_workspace=True): + if check_workspace and self.workspace is not None: + self.workspace.recheck() if not self.exists(): raise ValueError("created container no longer exists") value = json.loads(self.docker("inspect", self.id, "--format", "{{json .}}")) if value["Id"] != self.id or value["Name"] != "/" + self.name: raise ValueError("created container ID/name changed") + if check_workspace and self.workspace is not None: + self.workspace.inspect_mounts(value) return value def stop(self): self.inspect() self.docker("stop", "--time=10", self.id) + if self.workspace is not None: + self.workspace.recheck() def remove(self): if self.exists(): - self.inspect() + # A failed workspace check must not prevent exact-ID cleanup. + self.inspect(check_workspace=False) self.docker("rm", "-fv", self.id) diff --git a/scripts/acceptance/command-keys/cli_216.py b/scripts/acceptance/command-keys/cli_216.py index 9e1b052..bc2c1ba 100644 --- a/scripts/acceptance/command-keys/cli_216.py +++ b/scripts/acceptance/command-keys/cli_216.py @@ -14,8 +14,11 @@ import urllib.error import urllib.request +if __name__ == "__main__" and "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals(): + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") + SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = 'd377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525' +BOUNDARY_SHA = '49f44b870296b27955f116295f4e177a1fd285cc976809c13e5f57dc3cb1ebbf' # Bootstrap the local reviewed boundary module from the same bytes that are # checked. Subsequent inputs use its held descriptors and sealed memory files. _boundary_path = SOURCE / 'boundaries.py' @@ -48,7 +51,11 @@ def held(path, digest, maximum=16 * 1024 * 1024): HELD.append(value) return value -ENTRY_INPUT = held(Path(__file__).absolute(), None, 131072) +if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): + ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION + HELD.extend([BOOTSTRAP_HELD_BOUNDARY, ENTRY_INPUT]) +else: + ENTRY_INPUT = held(Path(__file__).absolute(), None, 131072) PROXY_INPUT = held(PROXY_SOURCE, PROXY_SHA256) shared_proxy = PROXY_INPUT.module('cli216_shared_proxy') CommittedResponseLoss = shared_proxy.CommittedResponseLoss @@ -72,6 +79,10 @@ def held(path, digest, maximum=16 * 1024 * 1024): 'DURABLE_WORKFLOW_AUTH_TOKEN': TOKEN} for directory in ('data', 'home', 'config', 'tmp', 'docker-empty'): boundary.private_directory(ROOT / directory) +WORKSPACE = boundary.Workspace(ROOT) +ENV.update({'PATH': WORKSPACE.child_anchor('bin') + ':/usr/bin:/bin', + 'HOME': WORKSPACE.child_anchor('home'), 'DW_CONFIG_HOME': WORKSPACE.child_anchor('config'), + 'TMPDIR': WORKSPACE.child_anchor('tmp')}) HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -84,13 +95,14 @@ def require(condition, label, detail=None): def docker(*args): - p = subprocess.run(['docker', '--config', str(ROOT / 'docker-empty'), *args], capture_output=True, text=True, timeout=50, env={'PATH': os.defpath, 'HOME': str(ROOT / 'home')}) + p = subprocess.run(['docker', '--config', '/proc/self/fd/' + str(WORKSPACE.children['docker-empty'].fd), *args], capture_output=True, text=True, timeout=50, env={'PATH': os.defpath, 'HOME': WORKSPACE.child_anchor('home')}, pass_fds=WORKSPACE.inherited()) if p.returncode: raise RuntimeError('docker operation failed: ' + p.stderr.replace(TOKEN, '[redacted]')[:2000]) return p.stdout.strip() def request(method, path, body=None, namespace='default'): + WORKSPACE.recheck() headers = {'Authorization': 'Bearer ' + TOKEN, 'X-Namespace': namespace, 'Content-Type': 'application/json'} worker = path.startswith('/api/worker/') headers['X-Durable-Workflow-Protocol-Version' if worker else 'X-Durable-Workflow-Control-Plane-Version'] = '1.20' if worker else '2' @@ -101,13 +113,17 @@ def request(method, path, body=None, namespace='default'): return response.status, json.load(response) except urllib.error.HTTPError as e: return e.code, json.load(e) + finally: + WORKSPACE.recheck() def cli(*args, namespace='default', success=True): + WORKSPACE.recheck() CLI_INPUT.recheck() p = subprocess.run(['dw', *args, '--server=' + PROXY.url, '--namespace=' + namespace, '--json'], - executable=CLI_INPUT.executable, env=ENV, capture_output=True, text=True, timeout=45, pass_fds=(CLI_INPUT.sealed_fd,)) + executable=CLI_INPUT.executable, env=ENV, capture_output=True, text=True, timeout=45, pass_fds=(CLI_INPUT.sealed_fd, *WORKSPACE.inherited())) CLI_INPUT.recheck() + WORKSPACE.recheck() try: body = json.loads(p.stdout) except ValueError: @@ -184,8 +200,7 @@ def history(f): def snapshot(): - from contextlib import closing - with closing(sqlite3.connect(DB.as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + with WORKSPACE.connection() as db: return legacy_helper.snapshot(db) @@ -258,7 +273,7 @@ def result_is_one(answer): def run_modern(): - require(shutil.which('dw', path=ENV['PATH']) == str(ROOT / 'bin/dw'), 'ordinary scoped PATH selects installed release') + require(shutil.which('dw', path=ENV['PATH']) == WORKSPACE.child_anchor('bin') + '/dw', 'ordinary scoped PATH selects installed release') require(hashlib.sha256((ROOT / 'bin/dw').read_bytes()).hexdigest() == 'da366c79c90978a3c0a92d723c66a96a2d0963f9dccbd41c8207f2c41848f814', 'published CLI binary SHA matches approved release') status, info = request('GET', '/api/cluster/info') require(status == 200 and info['version'] == '3.9.2' and info['capabilities']['command_request_keys'] is True, 'published Server 3.9.2 advertises command_request_keys', info.get('version')) @@ -332,7 +347,7 @@ def run_modern(): def run_selected_and_restart(): status, info = request('GET', '/api/cluster/info') require(status == 200 and info['version'] == '3.9.2', 'same published Server version after process restart') - previous = json.loads((ROOT / 'modern-admissions.json').read_text()) + previous = json.loads(WORKSPACE.read('modern-admissions.json')) recovered = 0 for record in previous: if not record['dropped']: @@ -399,19 +414,18 @@ def run_typed(): if __name__ == '__main__': outcome, error = 'runner-blocked', None - runtime = boundary.OwnedContainer(docker, NAME) + runtime = boundary.OwnedContainer(docker, NAME, WORKSPACE) try: observed = json.loads(docker('image', 'inspect', IMAGE, '--format', '{{json .}}')) require(observed['Architecture'] == 'amd64' and observed['Os'] == 'linux', 'observed native platform is linux/amd64') require(observed['Config']['Labels']['org.opencontainers.image.revision'] == '5614274d2629d03f92932a07dfee0fb10349ddd9', 'observed image source matches qualified producer') envfile = ROOT / 'runtime.env' - envfile.write_text('DW_AUTH_TOKEN=' + TOKEN + '\nDW_RUST_MAINTENANCE_LOOP=0\n') - envfile.chmod(0o600) + WORKSPACE.write('runtime.env', ('DW_AUTH_TOKEN=' + TOKEN + '\nDW_RUST_MAINTENANCE_LOOP=0\n').encode()) runtime.run('--rm', '-d', '--name', NAME, '--platform=linux/amd64', '--init', '--cpus=1', '--memory=1g', '--pids-limit=256', - '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--env-file', str(envfile), - '--mount', 'type=bind,source=' + str(ROOT / 'data') + ',target=/app/database', + '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--env-file', '/proc/self/fd/' + str(WORKSPACE.files['runtime.env'].fd), '--entrypoint=/bin/sh', + *WORKSPACE.mount_arguments(), '--tmpfs', '/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m', - '--tmpfs', '/tmp:rw,size=16m', '-p', '127.0.0.1::8080', IMAGE) + '--tmpfs', '/tmp:rw,size=16m', '-p', '127.0.0.1::8080', IMAGE, *WORKSPACE.native_arguments()) binding = runtime.inspect()['NetworkSettings']['Ports']['8080/tcp'][0] BASE = 'http://127.0.0.1:' + binding['HostPort'] for _ in range(100): @@ -448,6 +462,7 @@ def run_typed(): time.sleep(.1) if runtime.exists(): raise RuntimeError('created container removal did not complete') + WORKSPACE.recheck() for item in HELD: item.recheck() cleanup = 'created container and anonymous volumes removed; held inputs unchanged' @@ -455,15 +470,18 @@ def run_typed(): cleanup = 'cleanup or input identity requires investigation' outcome = 'runner-blocked' error = (type(failure).__name__ + ': ' + str(failure)).replace(TOKEN, '[redacted]') - (ROOT / 'runtime.env').unlink(missing_ok=True) + WORKSPACE.unlink('runtime.env') report = {'started_at': STARTED, 'finished_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), - 'outcome': outcome, 'created_container_ids': runtime.created_ids, 'checks': CHECKS, 'error': error, 'cleanup': cleanup, - 'runner_sha256': ENTRY_INPUT.digest, + 'outcome': outcome, 'workspace_identity': WORKSPACE.evidence(), 'created_container_ids': runtime.created_ids, 'checks': CHECKS, 'error': error, 'cleanup': cleanup, + 'runner_sha256': ENTRY_INPUT.digest, 'launcher_sha256': BOOTSTRAP_LAUNCHER_SHA256, + 'runner_source_commit': BOOTSTRAP_SOURCE_COMMIT, 'proxy_sha256': PROXY_INPUT.digest, 'server_index': IMAGE, 'server_source': '5614274d2629d03f92932a07dfee0fb10349ddd9', 'phase': os.environ.get('CLI_216_PHASE', 'modern')} - (ROOT / 'result.json').write_text(json.dumps(report, indent=2)) - (ROOT / 'invocations.json').write_text(json.dumps(INVOCATIONS, indent=2).replace(TOKEN, '[redacted]')) - (ROOT / 'admissions.json').write_text(json.dumps(ADMISSIONS, indent=2)) + WORKSPACE.write('result.json', json.dumps(report, indent=2).encode()) + WORKSPACE.write('invocations.json', json.dumps(INVOCATIONS, indent=2).replace(TOKEN, '[redacted]').encode()) + WORKSPACE.write('admissions.json', json.dumps(ADMISSIONS, indent=2).encode()) if PROXY: - (ROOT / 'wire.json').write_text(json.dumps(PROXY.records, indent=2).replace(TOKEN, '[redacted]')) + WORKSPACE.write('wire.json', json.dumps(PROXY.records, indent=2).replace(TOKEN, '[redacted]').encode()) + WORKSPACE.recheck() print(json.dumps(report), flush=True) + WORKSPACE.close() raise SystemExit(0 if outcome == 'pass' else 1) diff --git a/scripts/acceptance/command-keys/git_bootstrap.py b/scripts/acceptance/command-keys/git_bootstrap.py new file mode 100644 index 0000000..114977e --- /dev/null +++ b/scripts/acceptance/command-keys/git_bootstrap.py @@ -0,0 +1,22 @@ +"""Literal -c bootstrap: obtain reviewed dispatcher bytes from a frozen Git object.""" +import hashlib +import os +from pathlib import Path +import re +import subprocess +import sys + +if sys.argv[0] != "-c": + raise SystemExit("Use the reviewed literal -c Git bootstrap, not a host pathname") +commit = sys.argv.pop(1) +if not re.fullmatch(r"[0-9a-f]{40}", commit, re.ASCII): + raise ValueError("bootstrap requires one reviewed full Git commit") +relative = "scripts/acceptance/command-keys/run_cli_216.py" +expected = "b19a7a7be644f708f9855cbc5e9cc07e8bd67c60d17996ab69948f15d6db28ae" +data = subprocess.check_output(["/usr/bin/git", "--no-replace-objects", "show", commit + ":" + relative], + env={"PATH": os.defpath}, timeout=10) +if hashlib.sha256(data).hexdigest() != expected: + raise ValueError("reviewed dispatcher Git bytes differ") +context = {"__name__": "__main__", "__file__": str(Path.cwd() / relative), + "BOOTSTRAP_LAUNCHER_SHA256": expected, "BOOTSTRAP_SOURCE_COMMIT": commit} +exec(compile(data, context["__file__"], "exec"), context) diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index d568929..d467de3 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -13,6 +13,9 @@ import threading import time +if __name__ == "__main__" and "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals(): + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") + SOURCE = Path(__file__).resolve().parent ROOT = Path(os.environ["CLI_216_WORKDIR"]) @@ -39,32 +42,35 @@ def bound_driver(): os.close(descriptor) -BASE_SHA = "c6b7897a6b5c5208a236d296438fc56fa97d4b15ef28329ac719fade852dc562" +BASE_SHA = "507ff78a1a25bca43f4b5ce84abd395d59d24641bbac6db8c6ee86a665113417" d = bound_driver() -ENTRY_INPUT = d.held(Path(__file__).absolute(), None, 131072) +if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): + ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION + d.HELD.extend([BOOTSTRAP_HELD_BOUNDARY, ENTRY_INPUT]) +else: + ENTRY_INPUT = d.held(Path(__file__).absolute(), None, 131072) legacy = d.legacy_helper d.NAME = "cli-216-392-legacy" -runtime = d.boundary.OwnedContainer(d.docker, d.NAME) +runtime = d.boundary.OwnedContainer(d.docker, d.NAME, d.WORKSPACE) manifest = legacy.manifest() cases, selections, windows = [], [], [] started = datetime.datetime.now(datetime.timezone.utc).isoformat() def full_snapshot(): - with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True, timeout=2)) as db: + with d.WORKSPACE.connection() as db: return legacy.snapshot(db) def start(): envfile = ROOT / "runtime.env" - envfile.write_text("DW_AUTH_TOKEN=" + d.TOKEN + "\nDW_RUST_MAINTENANCE_LOOP=0\n") - envfile.chmod(0o600) + d.WORKSPACE.write("runtime.env", ("DW_AUTH_TOKEN=" + d.TOKEN + "\nDW_RUST_MAINTENANCE_LOOP=0\n").encode()) runtime.run("--rm", "-d", "--name", d.NAME, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", "--pids-limit=256", "--read-only", "--cap-drop=ALL", - "--security-opt=no-new-privileges", "--env-file", str(envfile), - "--mount", "type=bind,source=" + str(ROOT / "data") + ",target=/app/database", + "--security-opt=no-new-privileges", "--env-file", "/proc/self/fd/" + str(d.WORKSPACE.files["runtime.env"].fd), "--entrypoint=/bin/sh", + *d.WORKSPACE.mount_arguments(), "--tmpfs", "/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m", - "--tmpfs", "/tmp:rw,size=16m", "-p", "127.0.0.1::8080", d.IMAGE) + "--tmpfs", "/tmp:rw,size=16m", "-p", "127.0.0.1::8080", d.IMAGE, *d.WORKSPACE.native_arguments()) binding = runtime.inspect()["NetworkSettings"]["Ports"]["8080/tcp"][0] d.BASE = "http://127.0.0.1:" + binding["HostPort"] for _ in range(100): @@ -223,11 +229,41 @@ def exercise(): phase = "offline accepted transform" # This is a current-producer fixture. Checkpoint only after observed # process absence, then freeze the helper's untouched source inode. - with closing(sqlite3.connect(d.DB, timeout=2)) as db: + with d.WORKSPACE.connection(readonly=False) as db: db.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + # Preserve the held runtime DB inode. Transform a distinct stopped + # sibling copy, verify its complete result, then copy through the held + # DB descriptor before starting the next runtime on that same inode. + d.WORKSPACE.recheck() original = d.DB.with_name("seed.sqlite") - d.DB.rename(original) - transform = legacy.transform_copy(original, d.DB, selections, stopped=True) + descriptor = os.open(original.name, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=d.WORKSPACE.data.fd) + try: + with os.fdopen(os.dup(descriptor), "wb") as copied: + copied.write(os.pread(d.WORKSPACE.database.fd, 64 * 1024 * 1024 + 1, 0)) + finally: + os.close(descriptor) + seed_input = d.held(original, None, 64 * 1024 * 1024) + d.WORKSPACE.recheck() + transformed = d.DB.with_name("transformed.sqlite") + before = full_snapshot() + transform = legacy.transform_copy(original, transformed, selections, stopped=True) + copied_input = d.held(transformed, None, 64 * 1024 * 1024) + d.WORKSPACE.recheck() + seed_input.recheck() + # A real disk descriptor works with SQLite and binds the main inode; + # the sealed executable memfd does not. A post-open pathname stat + # cannot establish the connection's identity after a swap-back. + copied_input.recheck() + with closing(sqlite3.connect("file:/proc/self/fd/" + str(copied_input.fd) + "?mode=ro&immutable=1", uri=True)) as db: + transformed_snapshot = legacy.snapshot(db) + copied_input.recheck() + if (transform["before_sha256"] != legacy.fingerprint(before) + or transform["after_sha256"] != legacy.fingerprint(transformed_snapshot)): + raise ValueError("held transform source/result differs") + d.WORKSPACE.database.write(copied_input.data) + if full_snapshot() != transformed_snapshot: + raise ValueError("held runtime DB differs from accepted transformed copy") d.require(transform["transformed"] == 12, "exact twelve approved legacy transforms; all other schema/rows preserved") start() phase = "installed legacy readback" @@ -250,23 +286,27 @@ def exercise(): time.sleep(.1) if runtime.exists(): raise RuntimeError("created container removal did not complete") + d.WORKSPACE.recheck() for item in d.HELD: item.recheck() except Exception as failure: cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") outcome = "runner-blocked" - (ROOT / "runtime.env").unlink(missing_ok=True) + d.WORKSPACE.unlink("runtime.env") absent = not runtime.exists() - report = {"cell": "synthetic-legacy-shape", "outcome": outcome, "created_container_ids": runtime.created_ids, "error": error, + report = {"cell": "synthetic-legacy-shape", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "created_container_ids": runtime.created_ids, "error": error, "cleanup_error": cleanup_error, "phase": phase, "checks": len(d.CHECKS), "started_at": started, "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "driver_sha256": ENTRY_INPUT.digest, + "driver_sha256": ENTRY_INPUT.digest, "launcher_sha256": BOOTSTRAP_LAUNCHER_SHA256, + "runner_source_commit": BOOTSTRAP_SOURCE_COMMIT, "base_driver_sha256": d.ENTRY_INPUT.digest, "helper_sha256": d.HELPER_INPUT.digest, "manifest_sha256": legacy.MANIFEST_SHA256, "server_index": d.IMAGE, "transform": transform, "windows": windows, "cleanup": "created container/volumes/env removed; held inputs unchanged" if absent and cleanup_error is None else "cleanup/input identity failed"} - (ROOT / "legacy-result.json").write_text(json.dumps(report, indent=2)) - (ROOT / "legacy-wire.json").write_text(json.dumps({"seed": seed_wire, "readback": wire}, indent=2)) + d.WORKSPACE.write("legacy-result.json", json.dumps(report, indent=2).encode()) + d.WORKSPACE.write("legacy-wire.json", json.dumps({"seed": seed_wire, "readback": wire}, indent=2).encode()) + d.WORKSPACE.recheck() print(json.dumps(report), flush=True) + d.WORKSPACE.close() raise SystemExit(0 if outcome == "pass" and absent else 1) diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py new file mode 100644 index 0000000..6da4e6a --- /dev/null +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -0,0 +1,210 @@ +"""Installed CLI readback on a copy of the accepted real 3.8 API seed.""" +from contextlib import closing +from copy import deepcopy +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import stat +import time +import types + +SOURCE = Path(__file__).resolve().parent +LEGACY_SHA = "cc4018a807ff01e557676fa7ff4cb73f30872b004d04b42338fe96f65e18aec2" +BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" +PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" +CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" +DELTA_SHA = "fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd" +JOURNAL_CHECKSUM = "53b752529a1f7addaefe62ab61160dd42b3990f805cfdc4ecd1b48927a6c2f7ce42526fc04522ac887a37208f343fc73" + + +def load_legacy(): + path = SOURCE / "legacy_cli.py" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + observed = os.fstat(descriptor) + if (not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.getuid() + or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 131072): + raise ValueError("legacy adapter owner/type/mode differs") + data = os.pread(descriptor, 131073, 0) + if hashlib.sha256(data).hexdigest() != LEGACY_SHA: + raise ValueError("legacy adapter source differs") + module = types.ModuleType("cli216_physical_common") + module.__file__ = str(path) + exec(compile(data, str(path), "exec"), module.__dict__) + held = module.d.held(path, LEGACY_SHA) + if held.identity != held._identity(observed): + raise ValueError("legacy adapter changed during held import") + return module + finally: + os.close(descriptor) + + +def compare_upgrade(before, after, delta, started, ended, helper): + """Allow only the tagged DDL and narrowly validated migration metadata.""" + if before["tables"]["dw_server_schema"]["rows"] != [[["text", "rust-development"], ["integer", 12]]]: + raise ValueError("baseline is not native SQLite schema 12") + expected = deepcopy(before) + with closing(sqlite3.connect(":memory:")) as db: + db.execute("CREATE TABLE dw_server_schema(engine TEXT, version INTEGER)") + db.executescript(delta.decode()) + additions = helper.snapshot(db) + expected["schema"].extend(row for row in additions["schema"] if row[2] == "dw_command_intents") + expected["schema"].sort(key=lambda row: (row[0], row[1])) + expected["tables"]["dw_command_intents"] = additions["tables"]["dw_command_intents"] + expected["tables"]["dw_server_schema"]["rows"] = [[["text", "rust-development"], ["integer", 13]]] + journal = after["tables"]["_sqlx_migrations"] + columns = ["version", "description", "installed_on", "success", "checksum", "execution_time"] + if journal["columns"] != columns or len(journal["rows"]) != 1: + raise ValueError("upgraded migration journal shape differs") + row = journal["rows"][0] + old = before["tables"]["_sqlx_migrations"]["rows"] + if len(old) != 1 or old[0][0] != ["integer", 12]: + raise ValueError("baseline migration journal shape differs") + if (row[0] != ["integer", 13] or row[1] != ["text", old[0][1][1] + " and keyed command intents"] + or row[3] != ["integer", 1] or row[4] != ["blob", JOURNAL_CHECKSUM] + or row[5] != ["integer", 0] + or row[2][0] != "text"): + raise ValueError("migration journal content differs from tagged schema 13") + installed = datetime.datetime.fromisoformat(row[2][1]).replace(tzinfo=datetime.timezone.utc) + if not started - datetime.timedelta(seconds=1) <= installed <= ended + datetime.timedelta(seconds=1): + raise ValueError("migration journal time is outside the actual upgrade window") + expected["tables"]["_sqlx_migrations"]["rows"] = deepcopy(journal["rows"]) + if expected != after: + raise ValueError("upgrade changed an original business row/schema or unexpected metadata") + + +def physical_cases(packet, common): + d = common.d + for original in packet["cases"]: + kind, state = original["kind"], original["state"] + receipt = original["receipt"] + fixture = {"workflow": original["workflow_id"], "run": receipt["run_id"], "namespace": "default"} + d.require(d.history(fixture) == original["history"]["events"], kind + ": original " + state + " API history preserved") + if kind == "signal": + # Real 3.8 ordinary signals stored no caller key. Never issue a + # manufactured retained retry or infer intent equality for them. + with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True)) as db: + key = db.execute("SELECT request_id FROM workflow_commands WHERE id=?", (receipt["command_id"],)).fetchone() + d.require(key == (None,), "physical signal: original NULL key is compatibility only") + continue + record = {"request_id": original["original_body"]["request_id"], + "receipt": {key: receipt[key] for key in ("command_id", "run_id", "command_sequence", "update_id", "accepted_at")}} + case = {"kind": kind, "fixture": fixture, "record": record, + "state": {"open": "open_after_restart", "completed": "completed_after_restart", "continued": "continued_after_restart"}[state]} + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + replies = list(pool.map(lambda _: common.readback(case), range(4))) + d.require(len({reply["command_id"] for reply in replies}) == 1, "physical update: concurrent " + state + " retries retain original command") + common.readback(case, run=fixture["run"]) + for options in [{"name": "other"}, {"run": "00000000000000000000000000"}]: + answer = d.message("update", fixture["workflow"], record["request_id"], value=999, success=False, **options) + refusal = common.typed_error(answer, "request_id_conflict") + d.require(refusal.get("retained_command_id") == receipt["command_id"] and + refusal.get("conflict") == ["name" if "name" in options else "target"], + "physical update: correct original identity/name/run fence") + + +if __name__ == "__main__": + if "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals() or "BOOTSTRAP_HELD_BOUNDARY" not in globals(): + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") + common = load_legacy() + d = common.d + entry = BOOTSTRAP_HELD_IMPLEMENTATION + d.HELD.extend([BOOTSTRAP_HELD_BOUNDARY, entry]) + baseline = d.held(Path(os.environ["CLI_216_PHYSICAL_BASELINE"]), BASELINE_SHA, 64 * 1024 * 1024) + packet_input = d.held(Path(os.environ["CLI_216_PHYSICAL_PACKET"]), PACKET_SHA) + contract_input = d.held(d.ROOT / "physical-contract.json", CONTRACT_SHA) + delta_input = d.held(d.ROOT / "0013_command_intents.sql", DELTA_SHA) + packet = json.loads(packet_input.data) + if packet["schema"] != "durable-workflow.command-key-physical-seed.v1" or packet["manifest_sha256"] != CONTRACT_SHA: + raise ValueError("physical seed interface differs") + contract = json.loads(contract_input.data) + if packet["expected_external_baseline"] != contract["physical_upgrade"]["baseline"] or len(packet["cases"]) != 6: + raise ValueError("physical baseline/case identities differ") + d.NAME = "cli-216-392-physical" + common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME, d.WORKSPACE) + upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade", d.WORKSPACE) + outcome, error, phase = "runner-blocked", None, "preflight" + started = datetime.datetime.now(datetime.timezone.utc) + before, after = None, None + try: + observed = json.loads(d.docker("image", "inspect", d.IMAGE, "--format", "{{json .}}")) + d.require(observed["Architecture"] == "amd64" and observed["Os"] == "linux" and + observed["Config"]["Labels"]["org.opencontainers.image.revision"] == + "5614274d2629d03f92932a07dfee0fb10349ddd9", "physical candidate published platform/source identity") + d.WORKSPACE.recheck() + if os.fstat(d.WORKSPACE.database.fd).st_size != 0: + raise ValueError("physical upgrade needs a newly created empty DB inode") + d.WORKSPACE.database.write(baseline.data) + d.WORKSPACE.recheck() + before = common.full_snapshot() + phase = "explicit physical schema upgrade" + upgrade_started = datetime.datetime.now(datetime.timezone.utc) + upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", + "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--entrypoint=/bin/sh", "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", + *d.WORKSPACE.mount_arguments(), + "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, *d.WORKSPACE.native_arguments("schema-upgrade")) + upgrade.inspect() + d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") + state = upgrade.inspect()["State"] + d.require(state["ExitCode"] == 0 and not state["OOMKilled"], "upgrade process exits cleanly without OOM") + upgrade.inspect() + result = json.loads(d.docker("logs", upgrade.id)) + d.require(result == {"backend": "sqlite", "from_schema_version": 12, "schema_version": 13, "status": "upgraded"}, + "actual schema transition is 12 to 13") + upgrade_ended = datetime.datetime.now(datetime.timezone.utc) + after = common.full_snapshot() + compare_upgrade(before, after, delta_input.data, upgrade_started, upgrade_ended, common.legacy) + d.require(True, "every original business row/schema preserved; only tagged DDL and validated version/journal metadata changed") + upgrade.remove() + phase = "installed CLI physical readback" + common.start() + window = common.full_snapshot() + physical_cases(packet, common) + common.unchanged(window, "physical upgrade installed request window") + outcome = "pass" + except Exception as failure: + error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + finally: + if d.PROXY: + d.PROXY.shutdown() + d.PROXY.server_close() + cleanup_error = None + try: + for owned in (upgrade, common.runtime): + owned.remove() + for _ in range(50): + if not owned.exists(): + break + time.sleep(.1) + if owned.exists(): + raise RuntimeError("created physical container removal did not finish") + d.WORKSPACE.recheck() + for item in d.HELD: + item.recheck() + except Exception as failure: + outcome = "runner-blocked" + cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + d.WORKSPACE.unlink("runtime.env") + report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "error": error, "cleanup_error": cleanup_error, + "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "checks": len(d.CHECKS), "driver_sha256": entry.digest, "launcher_sha256": BOOTSTRAP_LAUNCHER_SHA256, + "runner_source_commit": BOOTSTRAP_SOURCE_COMMIT, + "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", + "actual_seed_source": "09a4ccda3da11853606af031a8970bbde4b2d6ec", + "actual_seed_driver_sha256": "01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b", + "baseline_sha256": baseline.digest, "packet_sha256": packet_input.digest, "contract_sha256": contract_input.digest, + "before_sha256": common.legacy.fingerprint(before) if before else None, + "after_upgrade_sha256": common.legacy.fingerprint(after) if after else None, + "request_windows": common.windows, "server_index": d.IMAGE, + "created_container_ids": upgrade.created_ids + common.runtime.created_ids, + "cleanup": "created IDs/volumes/env removed; original held inputs unchanged" if cleanup_error is None else "requires investigation"} + d.WORKSPACE.write("physical-result.json", json.dumps(report, indent=2).encode()) + d.WORKSPACE.recheck() + print(json.dumps(report), flush=True) + d.WORKSPACE.close() + raise SystemExit(0 if outcome == "pass" else 1) diff --git a/scripts/acceptance/command-keys/run_cli_216.py b/scripts/acceptance/command-keys/run_cli_216.py new file mode 100644 index 0000000..8c8bae3 --- /dev/null +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -0,0 +1,69 @@ +"""Verify the selected command-key implementation before executing its bytes.""" +import hashlib +import os +from pathlib import Path +import stat +import types + +if __name__ == "__main__" and "BOOTSTRAP_LAUNCHER_SHA256" not in globals(): + raise SystemExit("Use the reviewed literal -c Git bootstrap before the dispatcher") + +SOURCE = Path(__file__).resolve().parent +BOUNDARY_SHA = "49f44b870296b27955f116295f4e177a1fd285cc976809c13e5f57dc3cb1ebbf" +IMPLEMENTATIONS = { + 'modern': ('cli_216.py', '507ff78a1a25bca43f4b5ce84abd395d59d24641bbac6db8c6ee86a665113417'), + 'legacy': ('legacy_cli.py', 'cc4018a807ff01e557676fa7ff4cb73f30872b004d04b42338fe96f65e18aec2'), + 'physical': ('physical_impl.py', '640a3a8b9e3c802e965971ca1e8980200493513a81249a9265817b9a9bce3ba4'), +} + + +def load_boundary(): + path = SOURCE / "boundaries.py" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + observed = os.fstat(descriptor) + if (not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.getuid() + or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 65536): + raise ValueError("boundary source owner/type/mode differs") + data = os.pread(descriptor, 65537, 0) + if hashlib.sha256(data).hexdigest() != BOUNDARY_SHA: + raise ValueError("boundary source content differs") + module = types.ModuleType("physical_bootstrap_boundary") + exec(compile(data, str(path), "exec"), module.__dict__) + held = module.HeldInput(path, BOUNDARY_SHA) + if held.identity != held._identity(observed): + held.close() + raise ValueError("boundary source changed during held import") + return module, held + finally: + os.close(descriptor) + + +def run_implementation(path, digest, boundary, boundary_input): + implementation = boundary.HeldInput(path, digest, maximum=131072) + try: + scope = {"__name__": "__main__", "__file__": str(implementation.path), + "BOOTSTRAP_HELD_IMPLEMENTATION": implementation, + "BOOTSTRAP_HELD_BOUNDARY": boundary_input, + "BOOTSTRAP_LAUNCHER_SHA256": globals().get("BOOTSTRAP_LAUNCHER_SHA256"), + "BOOTSTRAP_SOURCE_COMMIT": globals().get("BOOTSTRAP_SOURCE_COMMIT")} + # The literal digest and held bytes precede every implementation + # effect, including imports and source-provenance capture. + exec(compile(implementation.data, str(implementation.path), "exec"), scope) + return scope + finally: + try: + implementation.recheck() + boundary_input.recheck() + finally: + implementation.close() + boundary_input.close() + + +if __name__ == "__main__": + choice = os.environ.get("CLI_216_RUNNER", "modern") + if choice not in IMPLEMENTATIONS: + raise SystemExit("CLI_216_RUNNER must be modern, legacy or physical") + filename, expected = IMPLEMENTATIONS[choice] + trusted_boundary, held_boundary = load_boundary() + run_implementation(SOURCE / filename, expected, trusted_boundary, held_boundary) diff --git a/scripts/acceptance/command-keys/test_boundaries.py b/scripts/acceptance/command-keys/test_boundaries.py index d84ae68..2f05a5e 100644 --- a/scripts/acceptance/command-keys/test_boundaries.py +++ b/scripts/acceptance/command-keys/test_boundaries.py @@ -223,7 +223,13 @@ def test_schema_only_changes_do_not_pass_zero_write_comparison(self): database = Path(temporary.name).resolve() / "snapshot.sqlite" tree = ast.parse((SOURCE / "cli_216.py").read_text()) function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "snapshot") - scope = {"DB": database, "sqlite3": sqlite3, "legacy_helper": helper} + data = database.parent / "data" + data.mkdir(mode=0o700) + database = data / "database.sqlite" + database.parent.parent.chmod(0o700) + workspace = boundary.Workspace(database.parent.parent) + self.addCleanup(workspace.close) + scope = {"WORKSPACE": workspace, "legacy_helper": helper} exec(compile(ast.Module(body=[function], type_ignores=[]), "snapshot-check", "exec"), scope) with closing(sqlite3.connect(database)) as db: db.execute("CREATE TABLE fixture(value INTEGER)") @@ -247,5 +253,230 @@ def predecessor(): self.assertNotEqual(current, scope["snapshot"]()) +class WorkspaceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.parent = Path(self.temporary.name).resolve() + self.root = self.parent / "work" + self.root.mkdir(mode=0o700) + (self.root / "data").mkdir(mode=0o700) + self.workspace = boundary.Workspace(self.root) + self.addCleanup(self.workspace.close) + self.database = self.root / "data/database.sqlite" + with self.workspace.connection(readonly=False) as db: + db.execute("CREATE TABLE fixture(value INTEGER)") + db.execute("INSERT INTO fixture VALUES (1)") + db.commit() + + def replace(self, target): + path = {"root": self.root, "data": self.root / "data", "database": self.database}[target] + renamed = path.with_name(path.name + "-original") + path.rename(renamed) + if target == "database": + path.write_bytes(renamed.read_bytes()) + path.chmod(0o600) + else: + import shutil + shutil.copytree(renamed, path) + # The predecessor could reopen the new pathname and observe the same + # schema/rows, even though Docker would exercise a different object. + with closing(sqlite3.connect(self.database)) as db: + self.assertEqual(db.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + + def test_parent_or_db_swap_after_copy_refuses_before_upgrade(self): + # Each subcase needs an independent held workspace. + for target in ("root", "data", "database"): + with self.subTest(target=target): + if target != "root": + self.setUp() + self.workspace.database.recheck() + self.replace(target) + docker = FakeDocker() + owned = boundary.OwnedContainer(docker, "cli-test", self.workspace) + with self.assertRaises(ValueError): + owned.run("--name", "cli-test") + self.assertFalse(docker.calls) + + def test_parent_or_db_swap_after_comparison_refuses_runtime_start(self): + for target in ("data", "database"): + with self.subTest(target=target): + if target == "database": + self.setUp() + with self.workspace.connection() as db: + self.assertEqual(db.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + self.replace(target) + with self.assertRaises(ValueError): + self.workspace.mount_arguments() + with self.assertRaises(ValueError): + with self.workspace.connection(): + self.fail("replaced database was opened") + + def test_root_swap_refuses_report_emission(self): + self.replace("root") + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + self.assertFalse((self.root / "physical-result.json").exists()) + + def test_report_replacement_is_refused_before_and_during_write(self): + self.workspace.write("physical-result.json", b"original") + path = self.root / "physical-result.json" + path.rename(self.root / "original-report") + path.write_bytes(b"replacement") + path.chmod(0o600) + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b"pass") + self.assertEqual(path.read_bytes(), b"replacement") + + def test_report_path_swap_during_emission_cannot_publish_pass(self): + original = os.pwrite + path = self.root / "physical-result.json" + def swapped(fd, data, offset): + result = original(fd, data, offset) + path.rename(self.root / "held-report") + path.write_bytes(b"forged") + path.chmod(0o600) + return result + with patch.object(boundary.os, "pwrite", swapped): + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + self.assertEqual(path.read_bytes(), b"forged") + + def test_report_parent_swap_during_emission_is_refused(self): + original = os.pwrite + def swapped(fd, data, offset): + result = original(fd, data, offset) + self.replace("root") + return result + with patch.object(boundary.os, "pwrite", swapped): + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + + def test_report_in_place_edit_is_refused(self): + self.workspace.write("physical-result.json", b"reviewed") + (self.root / "physical-result.json").write_bytes(b"tampered") + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_db_symlink_and_extra_link_are_refused(self): + original = self.database.with_name("original.sqlite") + self.database.rename(original) + self.database.symlink_to(original) + with self.assertRaises(ValueError): + self.workspace.recheck() + self.database.unlink() + original.rename(self.database) + os.link(self.database, self.database.with_name("second.sqlite")) + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_transformed_copy_sqlite_read_uses_held_disk_identity_not_memfd(self): + path = self.root / "data/transformed.sqlite" + with closing(sqlite3.connect(path)) as db: + db.execute("CREATE TABLE transformed(value INTEGER)") + db.execute("INSERT INTO transformed VALUES (7)") + db.commit() + path.chmod(0o600) + held = boundary.HeldInput(path, hashlib.sha256(path.read_bytes()).hexdigest()) + self.addCleanup(held.close) + with self.assertRaises(sqlite3.OperationalError): + with closing(sqlite3.connect(Path(held.executable).as_uri() + "?mode=ro&immutable=1", uri=True)) as db: + db.execute("SELECT * FROM transformed").fetchall() + held.recheck() + with closing(sqlite3.connect("file:/proc/self/fd/" + str(held.fd) + "?mode=ro&immutable=1", uri=True)) as db: + self.assertEqual(db.execute("SELECT * FROM transformed").fetchall(), [(7,)]) + held.recheck() + path.rename(path.with_name("original-transformed.sqlite")) + path.write_bytes(held.data) + path.chmod(0o600) + with self.assertRaises(ValueError): + held.recheck() + + def test_sqlite_swap_open_swap_back_fools_path_stat_but_not_held_fd(self): + connect = sqlite3.connect + def raced(*arguments, **keywords): + original = self.database.with_name("opened-original.sqlite") + self.database.rename(original) + self.database.write_bytes(original.read_bytes()) + self.database.chmod(0o600) + with closing(connect(self.database)) as replacement: + replacement.execute("UPDATE fixture SET value=99") + replacement.commit() + try: + opened = connect(*arguments, **keywords) + finally: + self.database.unlink() + original.rename(self.database) + # The predecessor's post-open name points back at the original, + # even if the returned connection already opened the substitute. + if "/database.sqlite?" in str(arguments[0]): + named = opened.execute("PRAGMA database_list").fetchone()[2] + self.assertEqual(boundary.HeldDirectory.key(os.stat(named)), self.workspace.database.identity) + self.workspace.recheck() + return opened + with patch.object(sqlite3, "connect", raced): + with closing(sqlite3.connect("file:" + self.workspace.data.anchor + "/database.sqlite?mode=ro", uri=True)) as old: + self.assertEqual(old.execute("SELECT * FROM fixture").fetchall(), [(99,)]) + with self.workspace.connection() as current: + self.assertEqual(current.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + self.workspace.recheck() + + def test_mutable_database_keeps_inode_and_wal_semantics(self): + identity = self.workspace.database.identity + with self.workspace.connection(readonly=False) as db: + db.execute("PRAGMA journal_mode=WAL") + db.execute("INSERT INTO fixture VALUES (2)") + db.commit() + with self.workspace.connection() as reader: + self.assertEqual(reader.execute("SELECT * FROM fixture").fetchall(), [(1,), (2,)]) + self.workspace.recheck() + self.assertEqual(self.workspace.database.identity, identity) + + def test_canonical_docker_mount_requires_literal_native_identity_preflight(self): + arguments = self.workspace.mount_arguments() + mounts = [{"Destination": "/app/database", "Source": str(self.root / "data"), "Type": "bind", "RW": True}] + self.assertEqual(arguments, ("--mount", "type=bind,source=" + str(self.root / "data") + ",target=/app/database")) + native = self.workspace.native_arguments("schema-check") + self.assertEqual(native[1], boundary.NATIVE_PREFLIGHT) + self.assertEqual(native[-1], "schema-check") + self.assertEqual(native[-2], ":".join(str(value) for value in [self.database.stat().st_dev, + self.database.stat().st_ino, os.getuid(), os.getgid(), 600, 1])) + self.workspace.inspect_mounts({"Mounts": mounts}) + mounts[0]["Source"] = str(self.root) + with self.assertRaises(ValueError): + self.workspace.inspect_mounts({"Mounts": mounts}) + + def test_consumer_state_directory_swaps_are_refused(self): + for name in ("home", "config", "tmp", "docker-empty", "bin"): + with self.subTest(name=name): + path = self.root / name + path.rename(self.root / (name + "-held")) + path.mkdir(mode=0o700) + with self.assertRaises(ValueError): + self.workspace.recheck() + path.rmdir() + (self.root / (name + "-held")).rename(path) + self.workspace.recheck() + + def test_env_cleanup_uses_held_parent_after_root_rename(self): + self.workspace.write("runtime.env", b"SYNTHETIC=value") + original = self.root.with_name("held-root") + self.root.rename(original) + self.root.mkdir(mode=0o700) + (self.root / "runtime.env").write_bytes(b"foreign") + self.workspace.unlink("runtime.env") + self.assertFalse((original / "runtime.env").exists()) + self.assertEqual((self.root / "runtime.env").read_bytes(), b"foreign") + + def test_failed_workspace_check_does_not_block_exact_id_cleanup(self): + docker = FakeDocker() + owned = boundary.OwnedContainer(docker, "cli-test") + owned.run("--name", "cli-test") + owned.workspace = self.workspace + self.replace("data") + owned.remove() + self.assertIn(("rm", "-fv", "a" * 64), docker.calls) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/acceptance/command-keys/test_physical_cli.py b/scripts/acceptance/command-keys/test_physical_cli.py new file mode 100644 index 0000000..156ef31 --- /dev/null +++ b/scripts/acceptance/command-keys/test_physical_cli.py @@ -0,0 +1,190 @@ +"""Offline checks for the physical-upgrade comparison's narrow exceptions.""" +import datetime +import hashlib +import importlib.util +import os +from pathlib import Path +import sqlite3 +import shutil +import subprocess +import sys +import tempfile +import unittest +import unittest.mock + +SOURCE = Path(__file__).resolve().parent + + +def local_module(name, filename): + spec = importlib.util.spec_from_file_location(name, SOURCE / filename) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + + +physical = local_module("physical_comparison", "physical_impl.py") +boundary = local_module("physical_boundaries", "boundaries.py") +bootstrap = local_module("physical_bootstrap", "run_cli_216.py") + + +class UpgradeComparisonTests(unittest.TestCase): + def setUp(self): + helper_input = boundary.HeldInput(Path(os.environ["CLI_216_TEST_HELPER"]), + "6fc6054ce087c136a86d7b6b222daeefaa87c58fa0b613b155a5df39e6a26fad") + self.addCleanup(helper_input.close) + self.helper = helper_input.module("physical_snapshot_helper") + delta_input = boundary.HeldInput(Path(os.environ["CLI_216_TEST_DELTA"]), physical.DELTA_SHA) + self.addCleanup(delta_input.close) + self.delta = delta_input.data + self.db = sqlite3.connect(":memory:") + self.addCleanup(self.db.close) + self.db.executescript(""" + CREATE TABLE dw_server_schema(engine TEXT, version INTEGER); + INSERT INTO dw_server_schema VALUES ('rust-development', 12); + CREATE TABLE _sqlx_migrations(version INTEGER, description TEXT, installed_on TEXT, + success INTEGER, checksum BLOB, execution_time INTEGER); + CREATE TABLE business(value INTEGER); + INSERT INTO business VALUES (1); + """) + self.db.execute("INSERT INTO _sqlx_migrations VALUES (12,'baseline','2026-01-01 00:00:00',1,?,1)", (bytes(48),)) + self.before = self.helper.snapshot(self.db) + self.started = datetime.datetime.now(datetime.timezone.utc) + self.db.executescript(self.delta.decode()) + self.db.execute("UPDATE _sqlx_migrations SET version=13,description='baseline and keyed command intents'," + "installed_on=?,checksum=?,execution_time=0", (self.started.strftime("%Y-%m-%d %H:%M:%S"), bytes.fromhex(physical.JOURNAL_CHECKSUM))) + self.ended = datetime.datetime.now(datetime.timezone.utc) + + def compare(self): + return physical.compare_upgrade(self.before, self.helper.snapshot(self.db), self.delta, + self.started, self.ended, self.helper) + + def refuses(self, statement): + self.db.execute(statement) + with self.assertRaises(ValueError): + self.compare() + + def test_exact_tagged_upgrade_and_bounded_metadata_pass(self): + self.compare() + + def test_business_row_change_is_not_a_metadata_exception(self): + self.refuses("UPDATE business SET value=2") + + def test_unexpected_schema_object_is_refused(self): + self.refuses("CREATE INDEX unrelated ON business(value)") + + def test_nonempty_new_intent_table_is_refused(self): + self.refuses("INSERT INTO dw_command_intents VALUES ('command','default','digest')") + + def test_wrong_journal_checksum_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET checksum=x'00'") + + def test_wrong_journal_description_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET description='other'") + + def test_wrong_journal_version_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET version=14") + + def test_extra_journal_row_is_refused(self): + self.refuses("INSERT INTO _sqlx_migrations SELECT * FROM _sqlx_migrations") + + def test_old_journal_timestamp_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET installed_on='2020-01-01 00:00:00'") + + def test_nonzero_execution_time_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET execution_time=1") + + def test_unbounded_execution_time_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET execution_time=999999999999999") + + +class PreCaptureTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.boundary_input = boundary.HeldInput(SOURCE / "boundaries.py", bootstrap.BOUNDARY_SHA) + self.addCleanup(self.boundary_input.close) + + def test_loaded_predecessor_can_relabel_while_candidate_refuses_before_execution(self): + source = self.root / "implementation.py" + marker = self.root / "effect" + original = ("captured = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()\n" + "Path(" + repr(str(marker)) + ").touch()\n").encode() + source.write_bytes(original) + loaded = compile(original, str(source), "exec") + source.rename(self.root / "original") + source.write_bytes(b"replacement = True\n") + scope = {"__file__": str(source), "hashlib": hashlib, "Path": Path} + exec(loaded, scope) + self.assertEqual(scope["captured"], hashlib.sha256(source.read_bytes()).hexdigest()) + self.assertNotEqual(scope["captured"], hashlib.sha256(original).hexdigest()) + self.assertTrue(marker.exists()) + marker.unlink() + with self.assertRaises(ValueError): + bootstrap.run_implementation(source, hashlib.sha256(original).hexdigest(), boundary, self.boundary_input) + self.assertFalse(marker.exists()) + + def test_direct_implementation_refuses_before_effectful_initialization(self): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "physical_impl.py")], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use run_cli_216.py", result.stderr) + + def test_direct_common_entrypoints_refuse_before_initialization(self): + for filename in ("cli_216.py", "legacy_cli.py"): + with self.subTest(filename=filename): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / filename)], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use run_cli_216.py", result.stderr) + + def test_unknown_loader_choice_refuses(self): + for choice in ("unknown", "../physical_impl.py"): + with self.subTest(choice=choice): + scope = {"__name__": "__main__", "__file__": str(SOURCE / "run_cli_216.py"), + "BOOTSTRAP_LAUNCHER_SHA256": "trusted-test"} + with unittest.mock.patch.dict(os.environ, {"CLI_216_RUNNER": choice}): + with self.assertRaisesRegex(SystemExit, "CLI_216_RUNNER must be"): + exec(compile((SOURCE / "run_cli_216.py").read_bytes(), scope["__file__"], "exec"), scope) + + def test_direct_outer_dispatcher_refuses_pathname_authority(self): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "run_cli_216.py")], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("literal -c Git bootstrap", result.stderr) + + def test_frozen_git_bootstrap_ignores_preopen_outer_path_substitution(self): + root = self.root / "repo" + root.mkdir() + folder = root / "scripts/acceptance/command-keys" + folder.mkdir(parents=True) + launcher = folder / "run_cli_216.py" + launcher.write_bytes((SOURCE / "run_cli_216.py").read_bytes()) + env = {"PATH": os.defpath, "HOME": str(self.root), "GIT_CONFIG_NOSYSTEM": "1"} + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], env=env, stderr=subprocess.DEVNULL) + git("init"); git("add", ".") + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-m", "Frozen dispatcher") + commit = git("rev-parse", "HEAD").decode().strip() + marker = self.root / "substitute-executed" + launcher.write_text("from pathlib import Path\nPath(" + repr(str(marker)) + ").touch()\n") + subprocess.run([sys.executable, "-I", "-B", str(launcher)], env=env, check=True) + self.assertTrue(marker.exists()) + marker.unlink() + code = (SOURCE / "git_bootstrap.py").read_text() + result = subprocess.run([sys.executable, "-I", "-B", "-c", code, commit], cwd=root, + env={**env, "CLI_216_RUNNER": "unknown"}, capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn("CLI_216_RUNNER must be", result.stderr) + self.assertFalse(marker.exists()) + + def test_literal_bound_implementation_executes_checked_bytes(self): + source = self.root / "implementation.py" + data = b"value = 'reviewed'\n" + source.write_bytes(data) + result = bootstrap.run_implementation(source, hashlib.sha256(data).hexdigest(), boundary, self.boundary_input) + self.assertEqual(result["value"], "reviewed") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/acceptance/command-keys/test_workspace_container.py b/scripts/acceptance/command-keys/test_workspace_container.py new file mode 100644 index 0000000..8121d8a --- /dev/null +++ b/scripts/acceptance/command-keys/test_workspace_container.py @@ -0,0 +1,128 @@ +"""Opt-in local Docker boundary controls, not installed CLI conformance.""" +import importlib.util +import json +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import tempfile +import unittest + +SOURCE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("container_boundaries", SOURCE / "boundaries.py") +boundary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(boundary) +IMAGE = "ghcr.io/durable-workflow/server@sha256:8ffcdb2a36af3129a1987849aafe8715fe0d3bf9a9648467f0b5cb8eeef43ba9" + + +@unittest.skipUnless(os.environ.get("CLI_216_CONTAINER_CONTROLS") == "1", "explicit local Docker control required") +class ContainerWorkspaceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() / "fixture" + self.root.mkdir(mode=0o700) + (self.root / "data").mkdir(mode=0o700) + self.workspace = boundary.Workspace(self.root) + self.addCleanup(self.workspace.close) + self.created = [] + self.addCleanup(self.cleanup) + + def docker(self, *arguments): + result = subprocess.run(["docker", "--config", "/proc/self/fd/" + str(self.workspace.children["docker-empty"].fd), *arguments], + env={"PATH": os.defpath, "HOME": self.workspace.child_anchor("home")}, + pass_fds=self.workspace.inherited(), capture_output=True, text=True, timeout=40) + if result.returncode: + raise RuntimeError("local Docker control failed: " + result.stderr[:1000]) + return (result.stdout + (result.stderr if arguments[0] == "logs" else "")).strip() + + def cleanup(self): + for owned in self.created: + owned.remove() + self.assertFalse(owned.exists(), "exact created control ID remains") + print(json.dumps({"control": self.id(), "created_id": owned.id, "absent": True}), flush=True) + + def run_container(self, *extra, command): + name = "cli-152-boundary-" + secrets.token_hex(5) + owned = boundary.OwnedContainer(self.docker, name) + self.created.append(owned) + owned.run("-d", "--name", name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=512m", + "--pids-limit=128", "--read-only", "--network=none", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--entrypoint=/bin/sh", "--env", "DB_CONNECTION=sqlite", *extra, IMAGE, *command) + code = self.docker("wait", owned.id) + state = owned.inspect()["State"] + self.assertFalse(state["OOMKilled"]) + return code, self.docker("logs", owned.id) + + def test_native_exec_on_actual_guarded_canonical_mount(self): + mounts = self.workspace.mount_arguments() + native = self.workspace.native_arguments("schema-check") + code, output = self.run_container(*mounts, "--tmpfs", "/tmp:rw,size=16m", command=native) + # This finite native command intentionally refuses SQLite. Its exact + # native diagnostic proves the mount preflight reached exec, without + # starting a runtime or performing an installed upgrade cohort. + self.assertEqual(code, "1", output) + self.assertIn("these development schema commands require DB_CONNECTION=pgsql, mysql or mariadb", output) + self.workspace.recheck() + self.workspace.inspect_mounts(self.created[-1].inspect()) + self.assertEqual(os.fstat(self.workspace.database.fd).st_size, 0) + + def replacement_refused(self, target): + mounts = self.workspace.mount_arguments() + native = self.workspace.native_arguments("schema-check") + path = self.root / "data" if target == "data" else self.root / "data/database.sqlite" + original = path.with_name(path.name + "-original") + path.rename(original) + if target == "data": + shutil.copytree(original, path) + else: + path.write_bytes(b"") + path.chmod(0o600) + code, output = self.run_container(*mounts, command=native) + self.assertEqual(code, "78") + self.assertEqual(output, "") + self.assertEqual((self.root / "data/database.sqlite").stat().st_size, 0, + "native bootstrap executed against replacement") + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_actual_preflight_refuses_directory_replacement_before_native_exec(self): + self.replacement_refused("data") + + def test_actual_preflight_refuses_db_replacement_before_native_exec(self): + self.replacement_refused("database") + + def test_actual_docker_env_reader_uses_held_descriptor(self): + self.workspace.write("runtime.env", b"CLI_FIXTURE_VALUE=held\n") + descriptor = self.workspace.files["runtime.env"].fd + path = self.root / "runtime.env" + path.rename(self.root / "original-env") + path.write_text("CLI_FIXTURE_VALUE=forged\n") + path.chmod(0o600) + # A pathname reader would consume the forged value. The actual Docker + # client inherits the original FD, even after that pathname changes. + self.assertIn("forged", path.read_text()) + code, output = self.run_container("--env-file", "/proc/self/fd/" + str(descriptor), + command=("-c", 'printf %s "$CLI_FIXTURE_VALUE"')) + self.assertEqual((code, output), ("0", "held")) + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_actual_docker_config_reader_and_cleanup_ignore_replaced_path(self): + path = self.root / "docker-empty" + path.rename(self.root / "original-docker") + path.mkdir(mode=0o700) + (path / "config.json").write_text('{"currentContext":"not-an-owned-context"}') + old = subprocess.run(["docker", "--config", str(path), "info", "--format", "{{.Name}}"], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=10) + self.assertNotEqual(old.returncode, 0) + code, output = self.run_container(command=("-c", "printf held-config")) + self.assertEqual((code, output), ("0", "held-config")) + with self.assertRaises(ValueError): + self.workspace.recheck() + # addCleanup still uses the held config FD and captured container ID. + + +if __name__ == "__main__": + unittest.main()