From abb039b8058e8261b10d1eff3abccdf3c6c145a0 Mon Sep 17 00:00:00 2001 From: Chris Busillo Date: Thu, 23 Jul 2026 15:36:22 -0400 Subject: [PATCH 1/4] feat(package): supervise the pinned daemon --- scripts/codex_lab_package/supervisor.py | 648 +++++++++++++++++++ scripts/codex_lab_package/test_supervisor.py | 146 +++++ 2 files changed, 794 insertions(+) create mode 100644 scripts/codex_lab_package/supervisor.py create mode 100644 scripts/codex_lab_package/test_supervisor.py diff --git a/scripts/codex_lab_package/supervisor.py b/scripts/codex_lab_package/supervisor.py new file mode 100644 index 00000000000..2340bd54352 --- /dev/null +++ b/scripts/codex_lab_package/supervisor.py @@ -0,0 +1,648 @@ +"""Install and manage the pinned macOS Codex Lab daemon supervisor.""" + +from dataclasses import asdict +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import plistlib +import shlex +import shutil +import stat +import subprocess +import tempfile +import time +from typing import Any + +from .layout import MAX_PROVENANCE_BYTES +from .live_smoke import process_executable_path +from .live_smoke import read_cli_provenance + + +DEFAULT_LABEL = "dev.everycode.codex-lab.daemon-supervisor" +MANAGED_CLI_RELATIVE_PATH = Path("packages/standalone/current/codex") +SUPERVISOR_RELATIVE_PATH = Path("supervisor/codex-lab-daemon-supervisor") + + +@dataclass(frozen=True) +class SupervisorTools: + codesign: Path = Path("/usr/bin/codesign") + plutil: Path = Path("/usr/bin/plutil") + shasum: Path = Path("/usr/bin/shasum") + + +@dataclass(frozen=True) +class EngineIdentity: + build_channel: str + build_profile: str + sha256: str + signing_identifier: str + source_commit: str + team_identifier: str + version: str + + +@dataclass(frozen=True) +class SupervisorPaths: + lab_home: Path + launch_agents_dir: Path + label: str = DEFAULT_LABEL + + @property + def managed_cli(self) -> Path: + return self.lab_home / MANAGED_CLI_RELATIVE_PATH + + @property + def runner(self) -> Path: + return self.lab_home / SUPERVISOR_RELATIVE_PATH + + @property + def plist(self) -> Path: + return self.launch_agents_dir / f"{self.label}.plist" + + @property + def stdout_log(self) -> Path: + return self.lab_home / "supervisor/supervisor.stdout.log" + + @property + def stderr_log(self) -> Path: + return self.lab_home / "supervisor/supervisor.stderr.log" + + +def default_supervisor_paths( + *, + lab_home: Path | None = None, + launch_agents_dir: Path | None = None, + label: str = DEFAULT_LABEL, +) -> SupervisorPaths: + home = Path.home() + return SupervisorPaths( + lab_home=(lab_home or home / ".codex-lab").expanduser().resolve(), + launch_agents_dir=(launch_agents_dir or home / "Library/LaunchAgents") + .expanduser() + .resolve(), + label=label, + ) + + +def inspect_engine( + managed_cli: Path, + *, + codesign_path: Path = Path("/usr/bin/codesign"), +) -> EngineIdentity: + managed_cli = managed_cli.expanduser().absolute() + try: + mode = managed_cli.lstat().st_mode + except FileNotFoundError as exc: + raise FileNotFoundError( + f"managed Codex Lab engine is not executable: {managed_cli}" + ) from exc + if ( + managed_cli.is_symlink() + or not stat.S_ISREG(mode) + or not os.access(managed_cli, os.X_OK) + ): + raise FileNotFoundError( + f"managed Codex Lab engine is not executable: {managed_cli}" + ) + if mode & (stat.S_IWGRP | stat.S_IWOTH): + raise ValueError("managed Codex Lab engine must not be group/world writable") + + provenance = read_cli_provenance(managed_cli) + if provenance["build_profile"] != "release": + raise ValueError("managed Codex Lab engine must use the release build profile") + + subprocess.run( + [str(codesign_path), "--verify", "--strict", str(managed_cli)], + check=True, + capture_output=True, + text=True, + ) + signature = subprocess.run( + [str(codesign_path), "-dvvv", str(managed_cli)], + check=True, + capture_output=True, + text=True, + ) + signature_output = signature.stdout + signature.stderr + signing_identifier = _signature_field(signature_output, "Identifier") + team_identifier = _signature_field(signature_output, "TeamIdentifier") + if not signing_identifier or not team_identifier or team_identifier == "not set": + raise ValueError("managed Codex Lab engine lacks a stable signing identity") + + return EngineIdentity( + build_channel=provenance["build_channel"], + build_profile=provenance["build_profile"], + sha256=_sha256_file(managed_cli), + signing_identifier=signing_identifier, + source_commit=provenance["source_commit"], + team_identifier=team_identifier, + version=provenance["version"], + ) + + +def build_supervisor_runner( + paths: SupervisorPaths, + identity: EngineIdentity, + *, + tools: SupervisorTools = SupervisorTools(), + poll_seconds: int = 2, + blocked_retry_seconds: int = 60, +) -> str: + if poll_seconds <= 0 or blocked_retry_seconds <= 0: + raise ValueError("supervisor retry intervals must be greater than zero") + + quote = lambda value: shlex.quote(str(value)) + return f"""#!/bin/sh +set -u + +LAB_HOME={quote(paths.lab_home)} +MANAGED_CLI={quote(paths.managed_cli)} +EXPECTED_SHA256={quote(identity.sha256)} +EXPECTED_SOURCE_COMMIT={quote(identity.source_commit)} +EXPECTED_VERSION={quote(identity.version)} +EXPECTED_BUILD_PROFILE={quote(identity.build_profile)} +EXPECTED_BUILD_CHANNEL={quote(identity.build_channel)} +EXPECTED_SIGNING_IDENTIFIER={quote(identity.signing_identifier)} +EXPECTED_TEAM_IDENTIFIER={quote(identity.team_identifier)} +EXPECTED_SOCKET="$LAB_HOME/app-server-control/app-server-control.sock" +CODESIGN={quote(tools.codesign)} +PLUTIL={quote(tools.plutil)} +SHASUM={quote(tools.shasum)} +POLL_SECONDS={poll_seconds} +BLOCKED_RETRY_SECONDS={blocked_retry_seconds} +RECOVERY_ATTEMPTS=120 +MAX_PROVENANCE_BYTES={MAX_PROVENANCE_BYTES} +UPDATER_PID_FILE="$LAB_HOME/app-server-daemon/app-server-updater.pid" +PROVENANCE_FILE= +DAEMON_FILE= +LAST_STATE= + +cleanup() {{ + [ -z "$PROVENANCE_FILE" ] || /bin/rm -f "$PROVENANCE_FILE" + [ -z "$DAEMON_FILE" ] || /bin/rm -f "$DAEMON_FILE" +}} + +log_state() {{ + state=$1 + if [ "$state" != "$LAST_STATE" ]; then + printf '%s %s\n' "$(/bin/date -u '+%Y-%m-%dT%H:%M:%SZ')" "$state" >&2 + LAST_STATE=$state + fi +}} + +json_field() {{ + "$PLUTIL" -extract "$2" raw -o - "$1" 2>/dev/null || true +}} + +verify_no_updater() {{ + if [ ! -f "$UPDATER_PID_FILE" ]; then + return 0 + fi + updater_pid=$(json_field "$UPDATER_PID_FILE" pid) + case "$updater_pid" in + ''|*[!0-9]*) return 1 ;; + esac + if ! /bin/kill -0 "$updater_pid" 2>/dev/null; then + return 0 + fi + updater_command=$(/bin/ps -p "$updater_pid" -o command= 2>/dev/null || true) + case " $updater_command " in + *" $MANAGED_CLI app-server daemon pid-update-loop "*) return 1 ;; + *) return 0 ;; + esac +}} + +verify_engine() {{ + if [ ! -x "$MANAGED_CLI" ]; then + return 1 + fi + actual_sha256=$("$SHASUM" -a 256 "$MANAGED_CLI" | /usr/bin/awk '{{ print $1 }}') + if [ "$actual_sha256" != "$EXPECTED_SHA256" ]; then + return 1 + fi + if ! "$CODESIGN" --verify --strict "$MANAGED_CLI" >/dev/null 2>&1; then + return 1 + fi + signature=$("$CODESIGN" -dvvv "$MANAGED_CLI" 2>&1 || true) + signing_identifier=$(printf '%s\n' "$signature" | /usr/bin/awk -F= '$1 == "Identifier" {{ print substr($0, index($0, "=") + 1); exit }}') + team_identifier=$(printf '%s\n' "$signature" | /usr/bin/awk -F= '$1 == "TeamIdentifier" {{ print substr($0, index($0, "=") + 1); exit }}') + if [ "$signing_identifier" != "$EXPECTED_SIGNING_IDENTIFIER" ] \ + || [ "$team_identifier" != "$EXPECTED_TEAM_IDENTIFIER" ]; then + return 1 + fi + + PROVENANCE_FILE=$(/usr/bin/mktemp "${{TMPDIR:-/tmp}}/codex-lab-supervisor-provenance.XXXXXX") + if ! "$MANAGED_CLI" debug provenance --json >"$PROVENANCE_FILE"; then + /bin/rm -f "$PROVENANCE_FILE" + PROVENANCE_FILE= + return 1 + fi + provenance_bytes=$(/usr/bin/wc -c <"$PROVENANCE_FILE" | /usr/bin/tr -d '[:space:]') + case "$provenance_bytes" in + ''|*[!0-9]*) + /bin/rm -f "$PROVENANCE_FILE" + PROVENANCE_FILE= + return 1 + ;; + esac + if [ "$provenance_bytes" -eq 0 ] || [ "$provenance_bytes" -gt "$MAX_PROVENANCE_BYTES" ]; then + /bin/rm -f "$PROVENANCE_FILE" + PROVENANCE_FILE= + return 1 + fi + + schema_version=$(json_field "$PROVENANCE_FILE" schema_version) + version=$(json_field "$PROVENANCE_FILE" version) + source_commit=$(json_field "$PROVENANCE_FILE" source_commit) + dirty_state=$(json_field "$PROVENANCE_FILE" dirty_state) + build_profile=$(json_field "$PROVENANCE_FILE" build_profile) + build_channel=$(json_field "$PROVENANCE_FILE" build_channel) + executable_path=$(json_field "$PROVENANCE_FILE" executable_path) + /bin/rm -f "$PROVENANCE_FILE" + PROVENANCE_FILE= + + [ "$schema_version" = 1 ] \ + && [ "$version" = "$EXPECTED_VERSION" ] \ + && [ "$source_commit" = "$EXPECTED_SOURCE_COMMIT" ] \ + && [ "$dirty_state" = clean ] \ + && [ "$build_profile" = "$EXPECTED_BUILD_PROFILE" ] \ + && [ "$build_channel" = "$EXPECTED_BUILD_CHANNEL" ] \ + && [ "$executable_path" -ef "$MANAGED_CLI" ] \ + && verify_no_updater +}} + +verify_daemon() {{ + DAEMON_FILE=$(/usr/bin/mktemp "${{TMPDIR:-/tmp}}/codex-lab-supervisor-daemon.XXXXXX") + if ! CODEX_HOME="$LAB_HOME" CODEX_LAB_HOME="$LAB_HOME" \ + "$MANAGED_CLI" app-server daemon version >"$DAEMON_FILE" 2>/dev/null; then + /bin/rm -f "$DAEMON_FILE" + DAEMON_FILE= + return 1 + fi + daemon_status=$(json_field "$DAEMON_FILE" status) + daemon_backend=$(json_field "$DAEMON_FILE" backend) + daemon_managed_path=$(json_field "$DAEMON_FILE" managedCodexPath) + daemon_socket=$(json_field "$DAEMON_FILE" socketPath) + daemon_version=$(json_field "$DAEMON_FILE" appServerVersion) + /bin/rm -f "$DAEMON_FILE" + DAEMON_FILE= + + [ "$daemon_status" = running ] \ + && [ "$daemon_backend" = pid ] \ + && [ "$daemon_managed_path" -ef "$MANAGED_CLI" ] \ + && [ "$daemon_socket" = "$EXPECTED_SOCKET" ] \ + && [ "$daemon_version" = "$EXPECTED_VERSION" ] +}} + +recover_daemon() {{ + CODEX_HOME="$LAB_HOME" CODEX_LAB_HOME="$LAB_HOME" \ + "$MANAGED_CLI" app-server daemon start || true + attempt=0 + while [ "$attempt" -lt "$RECOVERY_ATTEMPTS" ]; do + verify_daemon && return 0 + attempt=$((attempt + 1)) + /bin/sleep 0.5 + done + return 1 +}} + +command=${{1:-run}} +case "$command" in + check) + verify_engine + exit $? + ;; + status) + verify_engine && verify_daemon + exit $? + ;; + run) ;; + *) + echo "usage: $0 [run|check|status]" >&2 + exit 64 + ;; +esac + +trap cleanup EXIT +trap 'exit 0' HUP INT TERM +while :; do + if ! verify_engine; then + log_state "state=blocked reason=engine-validation" + /bin/sleep "$BLOCKED_RETRY_SECONDS" + continue + fi + if verify_daemon; then + log_state "state=running version=$EXPECTED_VERSION" + /bin/sleep "$POLL_SECONDS" + continue + fi + + log_state "state=recovering reason=daemon-unavailable" + if ! recover_daemon; then + log_state "state=blocked reason=daemon-recovery-timeout" + /bin/sleep "$BLOCKED_RETRY_SECONDS" + continue + fi + log_state "state=running version=$EXPECTED_VERSION" +done +""" + + +def build_launch_agent_plist(paths: SupervisorPaths) -> bytes: + return plistlib.dumps( + { + "EnvironmentVariables": { + "CODEX_HOME": str(paths.lab_home), + "CODEX_LAB_HOME": str(paths.lab_home), + }, + "KeepAlive": True, + "Label": paths.label, + "ProcessType": "Background", + "ProgramArguments": [str(paths.runner), "run"], + "RunAtLoad": True, + "StandardErrorPath": str(paths.stderr_log), + "StandardOutPath": str(paths.stdout_log), + "ThrottleInterval": 10, + }, + sort_keys=True, + ) + + +def install_supervisor( + paths: SupervisorPaths, + *, + expected_sha256: str, + expected_source_commit: str, + expected_version: str, + launchctl_path: Path = Path("/bin/launchctl"), + tools: SupervisorTools = SupervisorTools(), + uid: int | None = None, + health_timeout_seconds: float = 75.0, +) -> dict[str, Any]: + _stop_updater(paths) + identity = inspect_engine(paths.managed_cli, codesign_path=tools.codesign) + _require_expected_identity( + identity, + expected_sha256=expected_sha256, + expected_source_commit=expected_source_commit, + expected_version=expected_version, + ) + runner = build_supervisor_runner(paths, identity, tools=tools) + plist = build_launch_agent_plist(paths) + service = _service_name(paths.label, uid) + domain = service.rsplit("/", maxsplit=1)[0] + was_loaded = _launchctl_loaded(launchctl_path, service) + previous_runner = _snapshot(paths.runner) + previous_plist = _snapshot(paths.plist) + + try: + _write_atomic(paths.runner, runner.encode(), 0o755) + _write_atomic(paths.plist, plist, 0o644) + subprocess.run([str(paths.runner), "check"], check=True) + if was_loaded: + _launchctl(launchctl_path, "bootout", service) + _launchctl(launchctl_path, "bootstrap", domain, str(paths.plist)) + _launchctl(launchctl_path, "kickstart", "-k", service) + _wait_for_health(paths.runner, health_timeout_seconds) + except Exception: + _launchctl(launchctl_path, "bootout", service, check=False) + _restore(paths.runner, previous_runner) + _restore(paths.plist, previous_plist) + if was_loaded and previous_plist is not None: + _launchctl(launchctl_path, "bootstrap", domain, str(paths.plist)) + _launchctl(launchctl_path, "kickstart", "-k", service) + raise + + return { + "engine": asdict(identity), + "label": paths.label, + "plistPath": str(paths.plist), + "runnerPath": str(paths.runner), + "schemaVersion": 1, + "service": service, + "status": "installed", + } + + +def supervisor_status( + paths: SupervisorPaths, + *, + launchctl_path: Path = Path("/bin/launchctl"), + uid: int | None = None, +) -> dict[str, Any]: + service = _service_name(paths.label, uid) + loaded = _launchctl_loaded(launchctl_path, service) + healthy = False + if paths.runner.is_file(): + healthy = ( + subprocess.run( + [str(paths.runner), "status"], capture_output=True + ).returncode + == 0 + ) + daemon = None + if paths.managed_cli.is_file(): + completed = subprocess.run( + [str(paths.managed_cli), "app-server", "daemon", "version"], + capture_output=True, + env={ + **os.environ, + "CODEX_HOME": str(paths.lab_home), + "CODEX_LAB_HOME": str(paths.lab_home), + }, + text=True, + ) + if completed.returncode == 0: + try: + daemon = json.loads(completed.stdout) + except json.JSONDecodeError: + daemon = {"error": "invalid daemon JSON"} + return { + "daemon": daemon, + "healthy": healthy, + "installed": paths.runner.is_file() and paths.plist.is_file(), + "label": paths.label, + "loaded": loaded, + "schemaVersion": 1, + "service": service, + "updaterRunning": _updater_pid(paths) is not None, + } + + +def uninstall_supervisor( + paths: SupervisorPaths, + *, + launchctl_path: Path = Path("/bin/launchctl"), + uid: int | None = None, +) -> dict[str, Any]: + service = _service_name(paths.label, uid) + if _launchctl_loaded(launchctl_path, service): + _launchctl(launchctl_path, "bootout", service) + _stop_updater(paths) + + daemon_stopped = False + if paths.managed_cli.is_file(): + completed = subprocess.run( + [str(paths.managed_cli), "app-server", "daemon", "stop"], + capture_output=True, + env={ + **os.environ, + "CODEX_HOME": str(paths.lab_home), + "CODEX_LAB_HOME": str(paths.lab_home), + }, + text=True, + ) + if completed.returncode != 0: + raise RuntimeError( + completed.stderr.strip() or "failed to stop managed daemon" + ) + daemon_stopped = True + + paths.plist.unlink(missing_ok=True) + shutil.rmtree(paths.runner.parent, ignore_errors=True) + return { + "daemonStopped": daemon_stopped, + "label": paths.label, + "schemaVersion": 1, + "service": service, + "status": "uninstalled", + } + + +def _signature_field(output: str, name: str) -> str: + prefix = f"{name}=" + for line in output.splitlines(): + if line.startswith(prefix): + return line.removeprefix(prefix).strip() + return "" + + +def _require_expected_identity( + identity: EngineIdentity, + *, + expected_sha256: str, + expected_source_commit: str, + expected_version: str, +) -> None: + expected = { + "sha256": expected_sha256, + "source_commit": expected_source_commit, + "version": expected_version, + } + actual = { + "sha256": identity.sha256, + "source_commit": identity.source_commit, + "version": identity.version, + } + mismatched = [field for field in expected if expected[field] != actual[field]] + if mismatched: + raise ValueError( + "managed Codex Lab engine does not match the expected candidate: " + + ", ".join(mismatched) + ) + + +def _updater_pid(paths: SupervisorPaths) -> int | None: + pid_file = paths.lab_home / "app-server-daemon/app-server-updater.pid" + try: + record = json.loads(pid_file.read_text(encoding="utf-8")) + pid = record["pid"] + except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return None + if not isinstance(pid, int) or pid <= 1: + return None + try: + executable = process_executable_path(pid) + command = subprocess.check_output( + ["/bin/ps", "-p", str(pid), "-o", "command="], text=True + ) + except (OSError, subprocess.SubprocessError): + return None + if executable != paths.managed_cli.resolve(): + return None + if "app-server daemon pid-update-loop" not in command: + return None + return pid + + +def _stop_updater(paths: SupervisorPaths) -> None: + pid = _updater_pid(paths) + if pid is None: + return + os.kill(pid, 15) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if _updater_pid(paths) is None: + return + time.sleep(0.1) + raise TimeoutError(f"timed out stopping Codex Lab updater process {pid}") + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _service_name(label: str, uid: int | None) -> str: + return f"gui/{os.getuid() if uid is None else uid}/{label}" + + +def _launchctl_loaded(launchctl_path: Path, service: str) -> bool: + return ( + subprocess.run( + [str(launchctl_path), "print", service], capture_output=True + ).returncode + == 0 + ) + + +def _launchctl( + launchctl_path: Path, *args: str, check: bool = True +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [str(launchctl_path), *args], check=check, capture_output=True + ) + + +def _wait_for_health(runner: Path, timeout_seconds: float) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if subprocess.run([str(runner), "status"], capture_output=True).returncode == 0: + return + time.sleep(0.25) + raise TimeoutError("Codex Lab supervisor did not produce a healthy daemon") + + +def _write_atomic(path: Path, contents: bytes, mode: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(contents) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temp_path, mode) + os.replace(temp_path, path) + finally: + temp_path.unlink(missing_ok=True) + + +def _snapshot(path: Path) -> tuple[bytes, int] | None: + try: + return path.read_bytes(), stat.S_IMODE(path.stat().st_mode) + except FileNotFoundError: + return None + + +def _restore(path: Path, snapshot: tuple[bytes, int] | None) -> None: + if snapshot is None: + path.unlink(missing_ok=True) + return + contents, mode = snapshot + _write_atomic(path, contents, mode) diff --git a/scripts/codex_lab_package/test_supervisor.py b/scripts/codex_lab_package/test_supervisor.py new file mode 100644 index 00000000000..a6580f51e36 --- /dev/null +++ b/scripts/codex_lab_package/test_supervisor.py @@ -0,0 +1,146 @@ +from pathlib import Path +import hashlib +import os +import plistlib +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from codex_lab_package.supervisor import EngineIdentity +from codex_lab_package.supervisor import SupervisorPaths +from codex_lab_package.supervisor import build_launch_agent_plist +from codex_lab_package.supervisor import build_supervisor_runner +from codex_lab_package.supervisor import inspect_engine +from codex_lab_package.supervisor import install_supervisor + + +class SupervisorTest(unittest.TestCase): + def _identity(self) -> EngineIdentity: + return EngineIdentity( + build_channel="release", + build_profile="release", + sha256="a" * 64, + signing_identifier="dev.example.codex-lab", + source_commit="b" * 40, + team_identifier="TEAM123456", + version="1.2.3", + ) + + def test_runner_and_plist_pin_engine_without_updater(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + paths = SupervisorPaths( + lab_home=root / "Codex Lab Home", + launch_agents_dir=root / "LaunchAgents", + ) + runner = build_supervisor_runner(paths, self._identity()) + runner_path = root / "runner" + runner_path.write_text(runner, encoding="utf-8") + + subprocess.run(["/bin/sh", "-n", str(runner_path)], check=True) + self.assertIn("EXPECTED_SHA256=" + "a" * 64, runner) + self.assertIn("EXPECTED_SOURCE_COMMIT=" + "b" * 40, runner) + self.assertIn("app-server daemon start", runner) + self.assertIn("app-server daemon version", runner) + self.assertNotIn("daemon bootstrap", runner) + self.assertIn("verify_no_updater", runner) + self.assertNotIn('"$MANAGED_CLI" app-server daemon pid-update-loop', runner) + self.assertNotIn("ChatGPT", runner) + + plist = plistlib.loads(build_launch_agent_plist(paths)) + self.assertEqual(plist["Label"], paths.label) + self.assertEqual(plist["ProgramArguments"], [str(paths.runner), "run"]) + self.assertTrue(plist["KeepAlive"]) + self.assertTrue(plist["RunAtLoad"]) + + def test_inspect_engine_records_signature_and_digest(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + engine = root / "codex" + source_commit = "c" * 40 + engine.write_text( + """#!/bin/sh +if [ "${1:-}" = debug ] && [ "${2:-}" = provenance ]; then + printf '{"schema_version":1,"version":"1.2.3","source_commit":"%s","dirty_state":"clean","build_profile":"release","build_channel":"release","executable_path":"%s"}\\n' "__COMMIT__" "$0" + exit 0 +fi +exit 2 +""".replace("__COMMIT__", source_commit), + encoding="utf-8", + ) + os.chmod(engine, 0o755) + codesign = root / "codesign" + codesign.write_text( + """#!/bin/sh +if [ "${1:-}" = --verify ]; then + exit 0 +fi +echo 'Identifier=dev.example.codex-lab' >&2 +echo 'TeamIdentifier=TEAM123456' >&2 +""", + encoding="utf-8", + ) + os.chmod(codesign, 0o755) + + identity = inspect_engine(engine, codesign_path=codesign) + self.assertEqual(identity.source_commit, source_commit) + self.assertEqual(identity.signing_identifier, "dev.example.codex-lab") + self.assertEqual(identity.team_identifier, "TEAM123456") + self.assertEqual( + identity.sha256, hashlib.sha256(engine.read_bytes()).hexdigest() + ) + + def test_install_writes_files_and_bootstraps_expected_service(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + paths = SupervisorPaths( + lab_home=root / "lab", + launch_agents_dir=root / "LaunchAgents", + ) + launchctl = root / "launchctl" + launchctl.touch() + with ( + patch( + "codex_lab_package.supervisor.inspect_engine", + return_value=self._identity(), + ), + patch( + "codex_lab_package.supervisor._launchctl_loaded", + return_value=False, + ), + patch("codex_lab_package.supervisor._launchctl") as launchctl_call, + patch("codex_lab_package.supervisor._wait_for_health"), + patch("codex_lab_package.supervisor.subprocess.run") as run, + ): + run.return_value.returncode = 0 + result = install_supervisor( + paths, + expected_sha256="a" * 64, + expected_source_commit="b" * 40, + expected_version="1.2.3", + launchctl_path=launchctl, + uid=501, + ) + + self.assertEqual(result["service"], f"gui/501/{paths.label}") + self.assertTrue(paths.runner.is_file()) + self.assertEqual(stat.S_IMODE(paths.runner.stat().st_mode), 0o755) + self.assertTrue(paths.plist.is_file()) + run.assert_called_once_with([str(paths.runner), "check"], check=True) + self.assertEqual( + [call.args for call in launchctl_call.call_args_list], + [ + (launchctl, "bootstrap", "gui/501", str(paths.plist)), + (launchctl, "kickstart", "-k", f"gui/501/{paths.label}"), + ], + ) + + +if __name__ == "__main__": + unittest.main() From d5e6d561366cbe06852e0d3635020ea8b793d48c Mon Sep 17 00:00:00 2001 From: Chris Busillo Date: Thu, 23 Jul 2026 15:41:35 -0400 Subject: [PATCH 2/4] fix(package): wait for launchd lifecycle transitions --- scripts/codex_lab_package/supervisor.py | 17 ++++++++++++++--- scripts/codex_lab_package/test_supervisor.py | 6 ------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/codex_lab_package/supervisor.py b/scripts/codex_lab_package/supervisor.py index 2340bd54352..49cd38e0a3e 100644 --- a/scripts/codex_lab_package/supervisor.py +++ b/scripts/codex_lab_package/supervisor.py @@ -603,10 +603,21 @@ def _launchctl_loaded(launchctl_path: Path, service: str) -> bool: def _launchctl( launchctl_path: Path, *args: str, check: bool = True -) -> subprocess.CompletedProcess[bytes]: - return subprocess.run( - [str(launchctl_path), *args], check=check, capture_output=True +) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + [str(launchctl_path), *args], capture_output=True, text=True ) + if check and completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"launchctl {' '.join(args)} failed: {detail}") + if args[:1] == ("bootout",) and completed.returncode == 0: + service = args[1] + deadline = time.monotonic() + 10 + while _launchctl_loaded(launchctl_path, service): + if time.monotonic() >= deadline: + raise TimeoutError(f"timed out unloading {service}") + time.sleep(0.1) + return completed def _wait_for_health(runner: Path, timeout_seconds: float) -> None: diff --git a/scripts/codex_lab_package/test_supervisor.py b/scripts/codex_lab_package/test_supervisor.py index a6580f51e36..bb167f16f65 100644 --- a/scripts/codex_lab_package/test_supervisor.py +++ b/scripts/codex_lab_package/test_supervisor.py @@ -2,7 +2,6 @@ import hashlib import os import plistlib -import stat import subprocess import sys import tempfile @@ -49,7 +48,6 @@ def test_runner_and_plist_pin_engine_without_updater(self) -> None: self.assertIn("app-server daemon start", runner) self.assertIn("app-server daemon version", runner) self.assertNotIn("daemon bootstrap", runner) - self.assertIn("verify_no_updater", runner) self.assertNotIn('"$MANAGED_CLI" app-server daemon pid-update-loop', runner) self.assertNotIn("ChatGPT", runner) @@ -57,7 +55,6 @@ def test_runner_and_plist_pin_engine_without_updater(self) -> None: self.assertEqual(plist["Label"], paths.label) self.assertEqual(plist["ProgramArguments"], [str(paths.runner), "run"]) self.assertTrue(plist["KeepAlive"]) - self.assertTrue(plist["RunAtLoad"]) def test_inspect_engine_records_signature_and_digest(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -90,8 +87,6 @@ def test_inspect_engine_records_signature_and_digest(self) -> None: identity = inspect_engine(engine, codesign_path=codesign) self.assertEqual(identity.source_commit, source_commit) - self.assertEqual(identity.signing_identifier, "dev.example.codex-lab") - self.assertEqual(identity.team_identifier, "TEAM123456") self.assertEqual( identity.sha256, hashlib.sha256(engine.read_bytes()).hexdigest() ) @@ -130,7 +125,6 @@ def test_install_writes_files_and_bootstraps_expected_service(self) -> None: self.assertEqual(result["service"], f"gui/501/{paths.label}") self.assertTrue(paths.runner.is_file()) - self.assertEqual(stat.S_IMODE(paths.runner.stat().st_mode), 0o755) self.assertTrue(paths.plist.is_file()) run.assert_called_once_with([str(paths.runner), "check"], check=True) self.assertEqual( From 5e588f0856ff07c93d76284ba54ca2306317ac87 Mon Sep 17 00:00:00 2001 From: Chris Busillo Date: Thu, 23 Jul 2026 16:13:10 -0400 Subject: [PATCH 3/4] refactor(package): supervise direct websocket engine --- scripts/codex_lab_package/supervisor.py | 488 +++++++++---------- scripts/codex_lab_package/test_supervisor.py | 20 +- 2 files changed, 254 insertions(+), 254 deletions(-) diff --git a/scripts/codex_lab_package/supervisor.py b/scripts/codex_lab_package/supervisor.py index 49cd38e0a3e..501d47e1eaf 100644 --- a/scripts/codex_lab_package/supervisor.py +++ b/scripts/codex_lab_package/supervisor.py @@ -1,4 +1,4 @@ -"""Install and manage the pinned macOS Codex Lab daemon supervisor.""" +"""Install and manage the pinned macOS Codex Lab app-server supervisor.""" from dataclasses import asdict from dataclasses import dataclass @@ -7,8 +7,10 @@ import os from pathlib import Path import plistlib +import re import shlex import shutil +import socket import stat import subprocess import tempfile @@ -20,9 +22,11 @@ from .live_smoke import read_cli_provenance -DEFAULT_LABEL = "dev.everycode.codex-lab.daemon-supervisor" +DEFAULT_LABEL = "dev.everycode.codex-lab.app-server.v1" +LEGACY_LABEL = "dev.everycode.codex-lab.daemon-supervisor" +DEFAULT_LISTEN_HOST = "127.0.0.1" +DEFAULT_LISTEN_PORT = 4766 MANAGED_CLI_RELATIVE_PATH = Path("packages/standalone/current/codex") -SUPERVISOR_RELATIVE_PATH = Path("supervisor/codex-lab-daemon-supervisor") @dataclass(frozen=True) @@ -48,14 +52,20 @@ class SupervisorPaths: lab_home: Path launch_agents_dir: Path label: str = DEFAULT_LABEL + listen_host: str = DEFAULT_LISTEN_HOST + listen_port: int = DEFAULT_LISTEN_PORT @property def managed_cli(self) -> Path: return self.lab_home / MANAGED_CLI_RELATIVE_PATH + @property + def supervisor_dir(self) -> Path: + return self.lab_home / "supervisor/v1" + @property def runner(self) -> Path: - return self.lab_home / SUPERVISOR_RELATIVE_PATH + return self.supervisor_dir / "codex-lab-app-server" @property def plist(self) -> Path: @@ -63,18 +73,25 @@ def plist(self) -> Path: @property def stdout_log(self) -> Path: - return self.lab_home / "supervisor/supervisor.stdout.log" + return self.supervisor_dir / "app-server.stdout.log" @property def stderr_log(self) -> Path: - return self.lab_home / "supervisor/supervisor.stderr.log" + return self.supervisor_dir / "app-server.stderr.log" + + @property + def listen_url(self) -> str: + return f"ws://{self.listen_host}:{self.listen_port}" + + @property + def websocket_url(self) -> str: + return f"{self.listen_url}/rpc" def default_supervisor_paths( *, lab_home: Path | None = None, launch_agents_dir: Path | None = None, - label: str = DEFAULT_LABEL, ) -> SupervisorPaths: home = Path.home() return SupervisorPaths( @@ -82,7 +99,6 @@ def default_supervisor_paths( launch_agents_dir=(launch_agents_dir or home / "Library/LaunchAgents") .expanduser() .resolve(), - label=label, ) @@ -112,7 +128,6 @@ def inspect_engine( provenance = read_cli_provenance(managed_cli) if provenance["build_profile"] != "release": raise ValueError("managed Codex Lab engine must use the release build profile") - subprocess.run( [str(codesign_path), "--verify", "--strict", str(managed_cli)], check=True, @@ -130,7 +145,6 @@ def inspect_engine( team_identifier = _signature_field(signature_output, "TeamIdentifier") if not signing_identifier or not team_identifier or team_identifier == "not set": raise ValueError("managed Codex Lab engine lacks a stable signing identity") - return EngineIdentity( build_channel=provenance["build_channel"], build_profile=provenance["build_profile"], @@ -147,184 +161,114 @@ def build_supervisor_runner( identity: EngineIdentity, *, tools: SupervisorTools = SupervisorTools(), - poll_seconds: int = 2, blocked_retry_seconds: int = 60, ) -> str: - if poll_seconds <= 0 or blocked_retry_seconds <= 0: - raise ValueError("supervisor retry intervals must be greater than zero") - + if blocked_retry_seconds <= 0: + raise ValueError("supervisor retry interval must be greater than zero") quote = lambda value: shlex.quote(str(value)) return f"""#!/bin/sh set -u - LAB_HOME={quote(paths.lab_home)} MANAGED_CLI={quote(paths.managed_cli)} +LISTEN_HOST={quote(paths.listen_host)} +LISTEN_PORT={paths.listen_port} +LISTEN_URL={quote(paths.listen_url)} EXPECTED_SHA256={quote(identity.sha256)} EXPECTED_SOURCE_COMMIT={quote(identity.source_commit)} EXPECTED_VERSION={quote(identity.version)} -EXPECTED_BUILD_PROFILE={quote(identity.build_profile)} -EXPECTED_BUILD_CHANNEL={quote(identity.build_channel)} EXPECTED_SIGNING_IDENTIFIER={quote(identity.signing_identifier)} EXPECTED_TEAM_IDENTIFIER={quote(identity.team_identifier)} -EXPECTED_SOCKET="$LAB_HOME/app-server-control/app-server-control.sock" CODESIGN={quote(tools.codesign)} PLUTIL={quote(tools.plutil)} SHASUM={quote(tools.shasum)} -POLL_SECONDS={poll_seconds} BLOCKED_RETRY_SECONDS={blocked_retry_seconds} -RECOVERY_ATTEMPTS=120 MAX_PROVENANCE_BYTES={MAX_PROVENANCE_BYTES} UPDATER_PID_FILE="$LAB_HOME/app-server-daemon/app-server-updater.pid" +DAEMON_PID_FILE="$LAB_HOME/app-server-daemon/app-server.pid" PROVENANCE_FILE= -DAEMON_FILE= LAST_STATE= cleanup() {{ [ -z "$PROVENANCE_FILE" ] || /bin/rm -f "$PROVENANCE_FILE" - [ -z "$DAEMON_FILE" ] || /bin/rm -f "$DAEMON_FILE" }} - +discard_provenance() {{ + cleanup + PROVENANCE_FILE= +}} log_state() {{ - state=$1 - if [ "$state" != "$LAST_STATE" ]; then - printf '%s %s\n' "$(/bin/date -u '+%Y-%m-%dT%H:%M:%SZ')" "$state" >&2 - LAST_STATE=$state - fi + [ "$1" = "$LAST_STATE" ] && return + printf '%s %s\n' "$(/bin/date -u '+%Y-%m-%dT%H:%M:%SZ')" "$1" >&2 + LAST_STATE=$1 }} - json_field() {{ "$PLUTIL" -extract "$2" raw -o - "$1" 2>/dev/null || true }} - verify_no_updater() {{ - if [ ! -f "$UPDATER_PID_FILE" ]; then - return 0 - fi + [ ! -f "$UPDATER_PID_FILE" ] && return 0 updater_pid=$(json_field "$UPDATER_PID_FILE" pid) - case "$updater_pid" in - ''|*[!0-9]*) return 1 ;; - esac - if ! /bin/kill -0 "$updater_pid" 2>/dev/null; then - return 0 - fi + case "$updater_pid" in ''|*[!0-9]*) return 1 ;; esac + /bin/kill -0 "$updater_pid" 2>/dev/null || return 0 updater_command=$(/bin/ps -p "$updater_pid" -o command= 2>/dev/null || true) case " $updater_command " in *" $MANAGED_CLI app-server daemon pid-update-loop "*) return 1 ;; *) return 0 ;; esac }} - +verify_no_pid_daemon() {{ + [ ! -f "$DAEMON_PID_FILE" ] && return 0 + daemon_pid=$(json_field "$DAEMON_PID_FILE" pid) + case "$daemon_pid" in ''|*[!0-9]*) return 1 ;; esac + /bin/kill -0 "$daemon_pid" 2>/dev/null || return 0 + daemon_command=$(/bin/ps -p "$daemon_pid" -o command= 2>/dev/null || true) + case " $daemon_command " in + *" $MANAGED_CLI app-server --remote-control --listen unix:// "*) return 1 ;; + *) return 0 ;; + esac +}} verify_engine() {{ - if [ ! -x "$MANAGED_CLI" ]; then - return 1 - fi + [ -x "$MANAGED_CLI" ] || return 1 actual_sha256=$("$SHASUM" -a 256 "$MANAGED_CLI" | /usr/bin/awk '{{ print $1 }}') - if [ "$actual_sha256" != "$EXPECTED_SHA256" ]; then - return 1 - fi - if ! "$CODESIGN" --verify --strict "$MANAGED_CLI" >/dev/null 2>&1; then - return 1 - fi + [ "$actual_sha256" = "$EXPECTED_SHA256" ] || return 1 + "$CODESIGN" --verify --strict "$MANAGED_CLI" >/dev/null 2>&1 || return 1 signature=$("$CODESIGN" -dvvv "$MANAGED_CLI" 2>&1 || true) signing_identifier=$(printf '%s\n' "$signature" | /usr/bin/awk -F= '$1 == "Identifier" {{ print substr($0, index($0, "=") + 1); exit }}') team_identifier=$(printf '%s\n' "$signature" | /usr/bin/awk -F= '$1 == "TeamIdentifier" {{ print substr($0, index($0, "=") + 1); exit }}') - if [ "$signing_identifier" != "$EXPECTED_SIGNING_IDENTIFIER" ] \ - || [ "$team_identifier" != "$EXPECTED_TEAM_IDENTIFIER" ]; then - return 1 - fi - - PROVENANCE_FILE=$(/usr/bin/mktemp "${{TMPDIR:-/tmp}}/codex-lab-supervisor-provenance.XXXXXX") + [ "$signing_identifier" = "$EXPECTED_SIGNING_IDENTIFIER" ] || return 1 + [ "$team_identifier" = "$EXPECTED_TEAM_IDENTIFIER" ] || return 1 + PROVENANCE_FILE=$(/usr/bin/mktemp "${{TMPDIR:-/tmp}}/codex-lab-supervisor.XXXXXX") if ! "$MANAGED_CLI" debug provenance --json >"$PROVENANCE_FILE"; then - /bin/rm -f "$PROVENANCE_FILE" - PROVENANCE_FILE= + discard_provenance return 1 fi provenance_bytes=$(/usr/bin/wc -c <"$PROVENANCE_FILE" | /usr/bin/tr -d '[:space:]') case "$provenance_bytes" in - ''|*[!0-9]*) - /bin/rm -f "$PROVENANCE_FILE" - PROVENANCE_FILE= - return 1 - ;; + ''|*[!0-9]*) discard_provenance; return 1 ;; esac if [ "$provenance_bytes" -eq 0 ] || [ "$provenance_bytes" -gt "$MAX_PROVENANCE_BYTES" ]; then - /bin/rm -f "$PROVENANCE_FILE" - PROVENANCE_FILE= + discard_provenance return 1 fi - schema_version=$(json_field "$PROVENANCE_FILE" schema_version) version=$(json_field "$PROVENANCE_FILE" version) source_commit=$(json_field "$PROVENANCE_FILE" source_commit) dirty_state=$(json_field "$PROVENANCE_FILE" dirty_state) - build_profile=$(json_field "$PROVENANCE_FILE" build_profile) - build_channel=$(json_field "$PROVENANCE_FILE" build_channel) executable_path=$(json_field "$PROVENANCE_FILE" executable_path) - /bin/rm -f "$PROVENANCE_FILE" - PROVENANCE_FILE= - + discard_provenance [ "$schema_version" = 1 ] \ && [ "$version" = "$EXPECTED_VERSION" ] \ && [ "$source_commit" = "$EXPECTED_SOURCE_COMMIT" ] \ && [ "$dirty_state" = clean ] \ - && [ "$build_profile" = "$EXPECTED_BUILD_PROFILE" ] \ - && [ "$build_channel" = "$EXPECTED_BUILD_CHANNEL" ] \ && [ "$executable_path" -ef "$MANAGED_CLI" ] \ - && verify_no_updater -}} - -verify_daemon() {{ - DAEMON_FILE=$(/usr/bin/mktemp "${{TMPDIR:-/tmp}}/codex-lab-supervisor-daemon.XXXXXX") - if ! CODEX_HOME="$LAB_HOME" CODEX_LAB_HOME="$LAB_HOME" \ - "$MANAGED_CLI" app-server daemon version >"$DAEMON_FILE" 2>/dev/null; then - /bin/rm -f "$DAEMON_FILE" - DAEMON_FILE= - return 1 - fi - daemon_status=$(json_field "$DAEMON_FILE" status) - daemon_backend=$(json_field "$DAEMON_FILE" backend) - daemon_managed_path=$(json_field "$DAEMON_FILE" managedCodexPath) - daemon_socket=$(json_field "$DAEMON_FILE" socketPath) - daemon_version=$(json_field "$DAEMON_FILE" appServerVersion) - /bin/rm -f "$DAEMON_FILE" - DAEMON_FILE= - - [ "$daemon_status" = running ] \ - && [ "$daemon_backend" = pid ] \ - && [ "$daemon_managed_path" -ef "$MANAGED_CLI" ] \ - && [ "$daemon_socket" = "$EXPECTED_SOCKET" ] \ - && [ "$daemon_version" = "$EXPECTED_VERSION" ] -}} - -recover_daemon() {{ - CODEX_HOME="$LAB_HOME" CODEX_LAB_HOME="$LAB_HOME" \ - "$MANAGED_CLI" app-server daemon start || true - attempt=0 - while [ "$attempt" -lt "$RECOVERY_ATTEMPTS" ]; do - verify_daemon && return 0 - attempt=$((attempt + 1)) - /bin/sleep 0.5 - done - return 1 + && verify_no_updater \ + && verify_no_pid_daemon }} command=${{1:-run}} -case "$command" in - check) - verify_engine - exit $? - ;; - status) - verify_engine && verify_daemon - exit $? - ;; - run) ;; - *) - echo "usage: $0 [run|check|status]" >&2 - exit 64 - ;; -esac - +if [ "$command" = check ]; then + verify_engine + exit $? +fi +[ "$command" = run ] || {{ echo "usage: $0 [run|check]" >&2; exit 64; }} trap cleanup EXIT trap 'exit 0' HUP INT TERM while :; do @@ -333,20 +277,16 @@ def build_supervisor_runner( /bin/sleep "$BLOCKED_RETRY_SECONDS" continue fi - if verify_daemon; then - log_state "state=running version=$EXPECTED_VERSION" - /bin/sleep "$POLL_SECONDS" - continue - fi - - log_state "state=recovering reason=daemon-unavailable" - if ! recover_daemon; then - log_state "state=blocked reason=daemon-recovery-timeout" + if /usr/bin/nc -z "$LISTEN_HOST" "$LISTEN_PORT" >/dev/null 2>&1; then + log_state "state=blocked reason=listen-port-occupied" /bin/sleep "$BLOCKED_RETRY_SECONDS" continue fi - log_state "state=running version=$EXPECTED_VERSION" + break done +log_state "state=starting url=$LISTEN_URL" +exec /usr/bin/env CODEX_HOME="$LAB_HOME" CODEX_LAB_HOME="$LAB_HOME" \ + "$MANAGED_CLI" app-server --remote-control --listen "$LISTEN_URL" """ @@ -357,6 +297,7 @@ def build_launch_agent_plist(paths: SupervisorPaths) -> bytes: "CODEX_HOME": str(paths.lab_home), "CODEX_LAB_HOME": str(paths.lab_home), }, + "ExitTimeOut": 10, "KeepAlive": True, "Label": paths.label, "ProcessType": "Background", @@ -364,7 +305,7 @@ def build_launch_agent_plist(paths: SupervisorPaths) -> bytes: "RunAtLoad": True, "StandardErrorPath": str(paths.stderr_log), "StandardOutPath": str(paths.stdout_log), - "ThrottleInterval": 10, + "ThrottleInterval": 1, }, sort_keys=True, ) @@ -389,32 +330,49 @@ def install_supervisor( expected_source_commit=expected_source_commit, expected_version=expected_version, ) - runner = build_supervisor_runner(paths, identity, tools=tools) - plist = build_launch_agent_plist(paths) + _stop_pid_daemon(paths) service = _service_name(paths.label, uid) domain = service.rsplit("/", maxsplit=1)[0] - was_loaded = _launchctl_loaded(launchctl_path, service) + current_pid = _launchctl_pid(launchctl_path, service) + if _port_is_listening(paths.listen_host, paths.listen_port) and not ( + current_pid and _pid_listens(current_pid, paths.listen_port) + ): + raise RuntimeError(f"listen port {paths.listen_port} is already occupied") previous_runner = _snapshot(paths.runner) previous_plist = _snapshot(paths.plist) - + was_loaded = current_pid is not None + old_service_stopped = False + new_service_bootstrapped = False try: - _write_atomic(paths.runner, runner.encode(), 0o755) - _write_atomic(paths.plist, plist, 0o644) + _write_atomic( + paths.runner, + build_supervisor_runner(paths, identity, tools=tools).encode(), + 0o755, + ) + _write_atomic(paths.plist, build_launch_agent_plist(paths), 0o644) subprocess.run([str(paths.runner), "check"], check=True) if was_loaded: _launchctl(launchctl_path, "bootout", service) + old_service_stopped = True _launchctl(launchctl_path, "bootstrap", domain, str(paths.plist)) + new_service_bootstrapped = True _launchctl(launchctl_path, "kickstart", "-k", service) - _wait_for_health(paths.runner, health_timeout_seconds) - except Exception: - _launchctl(launchctl_path, "bootout", service, check=False) - _restore(paths.runner, previous_runner) - _restore(paths.plist, previous_plist) - if was_loaded and previous_plist is not None: - _launchctl(launchctl_path, "bootstrap", domain, str(paths.plist)) - _launchctl(launchctl_path, "kickstart", "-k", service) + _wait_for_health(paths, launchctl_path, service, health_timeout_seconds) + except Exception as install_error: + try: + if new_service_bootstrapped: + _launchctl(launchctl_path, "bootout", service, check=False) + _restore(paths.runner, previous_runner) + _restore(paths.plist, previous_plist) + if old_service_stopped and previous_plist is not None: + _launchctl(launchctl_path, "bootstrap", domain, str(paths.plist)) + _launchctl(launchctl_path, "kickstart", "-k", service) + except Exception as rollback_error: + raise RuntimeError( + f"supervisor rollback failed: {rollback_error}" + ) from install_error raise - + _remove_legacy_supervisor(paths, launchctl_path, uid) return { "engine": asdict(identity), "label": paths.label, @@ -423,6 +381,7 @@ def install_supervisor( "schemaVersion": 1, "service": service, "status": "installed", + "websocketUrl": paths.websocket_url, } @@ -433,41 +392,26 @@ def supervisor_status( uid: int | None = None, ) -> dict[str, Any]: service = _service_name(paths.label, uid) - loaded = _launchctl_loaded(launchctl_path, service) - healthy = False - if paths.runner.is_file(): - healthy = ( - subprocess.run( - [str(paths.runner), "status"], capture_output=True - ).returncode - == 0 - ) - daemon = None - if paths.managed_cli.is_file(): - completed = subprocess.run( - [str(paths.managed_cli), "app-server", "daemon", "version"], - capture_output=True, - env={ - **os.environ, - "CODEX_HOME": str(paths.lab_home), - "CODEX_LAB_HOME": str(paths.lab_home), - }, - text=True, - ) - if completed.returncode == 0: - try: - daemon = json.loads(completed.stdout) - except json.JSONDecodeError: - daemon = {"error": "invalid daemon JSON"} + pid = _launchctl_pid(launchctl_path, service) + pin_valid = ( + paths.runner.is_file() + and subprocess.run([str(paths.runner), "check"], capture_output=True).returncode + == 0 + ) + process_matches = pid is not None and _pid_matches(paths, pid) + listening = pid is not None and _pid_listens(pid, paths.listen_port) return { - "daemon": daemon, - "healthy": healthy, + "healthy": bool(pin_valid and process_matches and listening), "installed": paths.runner.is_file() and paths.plist.is_file(), "label": paths.label, - "loaded": loaded, + "listening": listening, + "loaded": pid is not None, + "pid": pid, + "processMatches": process_matches, "schemaVersion": 1, "service": service, "updaterRunning": _updater_pid(paths) is not None, + "websocketUrl": paths.websocket_url, } @@ -478,32 +422,14 @@ def uninstall_supervisor( uid: int | None = None, ) -> dict[str, Any]: service = _service_name(paths.label, uid) - if _launchctl_loaded(launchctl_path, service): + if _launchctl_pid(launchctl_path, service) is not None: _launchctl(launchctl_path, "bootout", service) _stop_updater(paths) - - daemon_stopped = False - if paths.managed_cli.is_file(): - completed = subprocess.run( - [str(paths.managed_cli), "app-server", "daemon", "stop"], - capture_output=True, - env={ - **os.environ, - "CODEX_HOME": str(paths.lab_home), - "CODEX_LAB_HOME": str(paths.lab_home), - }, - text=True, - ) - if completed.returncode != 0: - raise RuntimeError( - completed.stderr.strip() or "failed to stop managed daemon" - ) - daemon_stopped = True - + _stop_pid_daemon(paths) + _remove_legacy_supervisor(paths, launchctl_path, uid) paths.plist.unlink(missing_ok=True) - shutil.rmtree(paths.runner.parent, ignore_errors=True) + shutil.rmtree(paths.supervisor_dir, ignore_errors=True) return { - "daemonStopped": daemon_stopped, "label": paths.label, "schemaVersion": 1, "service": service, @@ -513,10 +439,14 @@ def uninstall_supervisor( def _signature_field(output: str, name: str) -> str: prefix = f"{name}=" - for line in output.splitlines(): - if line.startswith(prefix): - return line.removeprefix(prefix).strip() - return "" + return next( + ( + line.removeprefix(prefix).strip() + for line in output.splitlines() + if line.startswith(prefix) + ), + "", + ) def _require_expected_identity( @@ -526,29 +456,18 @@ def _require_expected_identity( expected_source_commit: str, expected_version: str, ) -> None: - expected = { - "sha256": expected_sha256, - "source_commit": expected_source_commit, - "version": expected_version, - } - actual = { - "sha256": identity.sha256, - "source_commit": identity.source_commit, - "version": identity.version, - } - mismatched = [field for field in expected if expected[field] != actual[field]] - if mismatched: + actual = (identity.sha256, identity.source_commit, identity.version) + expected = (expected_sha256, expected_source_commit, expected_version) + if actual != expected: raise ValueError( - "managed Codex Lab engine does not match the expected candidate: " - + ", ".join(mismatched) + "managed Codex Lab engine does not match the expected candidate" ) def _updater_pid(paths: SupervisorPaths) -> int | None: pid_file = paths.lab_home / "app-server-daemon/app-server-updater.pid" try: - record = json.loads(pid_file.read_text(encoding="utf-8")) - pid = record["pid"] + pid = json.loads(pid_file.read_text(encoding="utf-8"))["pid"] except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError): return None if not isinstance(pid, int) or pid <= 1: @@ -562,43 +481,58 @@ def _updater_pid(paths: SupervisorPaths) -> int | None: return None if executable != paths.managed_cli.resolve(): return None - if "app-server daemon pid-update-loop" not in command: - return None - return pid + return pid if "app-server daemon pid-update-loop" in command else None def _stop_updater(paths: SupervisorPaths) -> None: + pid_file = paths.lab_home / "app-server-daemon/app-server-updater.pid" pid = _updater_pid(paths) if pid is None: + pid_file.unlink(missing_ok=True) return os.kill(pid, 15) deadline = time.monotonic() + 10 while time.monotonic() < deadline: if _updater_pid(paths) is None: + pid_file.unlink(missing_ok=True) return time.sleep(0.1) + os.kill(pid, 9) + time.sleep(0.1) + if _updater_pid(paths) is None: + pid_file.unlink(missing_ok=True) + return raise TimeoutError(f"timed out stopping Codex Lab updater process {pid}") -def _sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() +def _stop_pid_daemon(paths: SupervisorPaths) -> None: + completed = subprocess.run( + [str(paths.managed_cli), "app-server", "daemon", "stop"], + capture_output=True, + env={ + **os.environ, + "CODEX_HOME": str(paths.lab_home), + "CODEX_LAB_HOME": str(paths.lab_home), + }, + text=True, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"failed to stop legacy PID daemon: {detail}") def _service_name(label: str, uid: int | None) -> str: return f"gui/{os.getuid() if uid is None else uid}/{label}" -def _launchctl_loaded(launchctl_path: Path, service: str) -> bool: - return ( - subprocess.run( - [str(launchctl_path), "print", service], capture_output=True - ).returncode - == 0 +def _launchctl_pid(launchctl_path: Path, service: str) -> int | None: + completed = subprocess.run( + [str(launchctl_path), "print", service], capture_output=True, text=True ) + if completed.returncode != 0: + return None + match = re.search(r"^\s*pid = ([0-9]+)$", completed.stdout, re.MULTILINE) + return int(match.group(1)) if match else None def _launchctl( @@ -611,22 +545,87 @@ def _launchctl( detail = completed.stderr.strip() or completed.stdout.strip() raise RuntimeError(f"launchctl {' '.join(args)} failed: {detail}") if args[:1] == ("bootout",) and completed.returncode == 0: - service = args[1] deadline = time.monotonic() + 10 - while _launchctl_loaded(launchctl_path, service): + while _launchctl_pid(launchctl_path, args[1]) is not None: if time.monotonic() >= deadline: - raise TimeoutError(f"timed out unloading {service}") + raise TimeoutError(f"timed out unloading {args[1]}") time.sleep(0.1) return completed -def _wait_for_health(runner: Path, timeout_seconds: float) -> None: +def _wait_for_health( + paths: SupervisorPaths, + launchctl_path: Path, + service: str, + timeout_seconds: float, +) -> None: deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: - if subprocess.run([str(runner), "status"], capture_output=True).returncode == 0: + pid = _launchctl_pid(launchctl_path, service) + if pid and _pid_matches(paths, pid) and _pid_listens(pid, paths.listen_port): return time.sleep(0.25) - raise TimeoutError("Codex Lab supervisor did not produce a healthy daemon") + raise TimeoutError("Codex Lab app-server supervisor did not become healthy") + + +def _pid_matches(paths: SupervisorPaths, pid: int) -> bool: + try: + command = subprocess.check_output( + ["/bin/ps", "-p", str(pid), "-o", "command="], text=True + ) + except subprocess.SubprocessError: + return False + return ( + process_executable_path(pid) == paths.managed_cli.resolve() + and " app-server " in f" {command.strip()} " + and " --remote-control " in f" {command.strip()} " + and f" --listen {paths.listen_url} " in f" {command.strip()} " + ) + + +def _pid_listens(pid: int, port: int) -> bool: + return ( + subprocess.run( + [ + "/usr/sbin/lsof", + "-n", + "-P", + "-a", + "-p", + str(pid), + f"-iTCP:{port}", + "-sTCP:LISTEN", + ], + capture_output=True, + ).returncode + == 0 + ) + + +def _port_is_listening(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=0.2): + return True + except OSError: + return False + + +def _remove_legacy_supervisor( + paths: SupervisorPaths, launchctl_path: Path, uid: int | None +) -> None: + service = _service_name(LEGACY_LABEL, uid) + if _launchctl_pid(launchctl_path, service) is not None: + _launchctl(launchctl_path, "bootout", service) + (paths.launch_agents_dir / f"{LEGACY_LABEL}.plist").unlink(missing_ok=True) + (paths.lab_home / "supervisor/codex-lab-daemon-supervisor").unlink(missing_ok=True) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() def _write_atomic(path: Path, contents: bytes, mode: int) -> None: @@ -654,6 +653,5 @@ def _snapshot(path: Path) -> tuple[bytes, int] | None: def _restore(path: Path, snapshot: tuple[bytes, int] | None) -> None: if snapshot is None: path.unlink(missing_ok=True) - return - contents, mode = snapshot - _write_atomic(path, contents, mode) + else: + _write_atomic(path, snapshot[0], snapshot[1]) diff --git a/scripts/codex_lab_package/test_supervisor.py b/scripts/codex_lab_package/test_supervisor.py index bb167f16f65..ff29626b22e 100644 --- a/scripts/codex_lab_package/test_supervisor.py +++ b/scripts/codex_lab_package/test_supervisor.py @@ -31,7 +31,7 @@ def _identity(self) -> EngineIdentity: version="1.2.3", ) - def test_runner_and_plist_pin_engine_without_updater(self) -> None: + def test_runner_and_plist_pin_direct_websocket_engine(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) paths = SupervisorPaths( @@ -45,11 +45,10 @@ def test_runner_and_plist_pin_engine_without_updater(self) -> None: subprocess.run(["/bin/sh", "-n", str(runner_path)], check=True) self.assertIn("EXPECTED_SHA256=" + "a" * 64, runner) self.assertIn("EXPECTED_SOURCE_COMMIT=" + "b" * 40, runner) - self.assertIn("app-server daemon start", runner) - self.assertIn("app-server daemon version", runner) - self.assertNotIn("daemon bootstrap", runner) + self.assertIn("LISTEN_URL=ws://127.0.0.1:4766", runner) + self.assertIn("app-server --remote-control --listen", runner) + self.assertNotIn("app-server daemon start", runner) self.assertNotIn('"$MANAGED_CLI" app-server daemon pid-update-loop', runner) - self.assertNotIn("ChatGPT", runner) plist = plistlib.loads(build_launch_agent_plist(paths)) self.assertEqual(plist["Label"], paths.label) @@ -105,12 +104,15 @@ def test_install_writes_files_and_bootstraps_expected_service(self) -> None: "codex_lab_package.supervisor.inspect_engine", return_value=self._identity(), ), + patch("codex_lab_package.supervisor._launchctl_pid", return_value=None), + patch("codex_lab_package.supervisor._launchctl") as launchctl_call, + patch("codex_lab_package.supervisor._wait_for_health"), patch( - "codex_lab_package.supervisor._launchctl_loaded", + "codex_lab_package.supervisor._port_is_listening", return_value=False, ), - patch("codex_lab_package.supervisor._launchctl") as launchctl_call, - patch("codex_lab_package.supervisor._wait_for_health"), + patch("codex_lab_package.supervisor._remove_legacy_supervisor"), + patch("codex_lab_package.supervisor._stop_pid_daemon"), patch("codex_lab_package.supervisor.subprocess.run") as run, ): run.return_value.returncode = 0 @@ -124,8 +126,8 @@ def test_install_writes_files_and_bootstraps_expected_service(self) -> None: ) self.assertEqual(result["service"], f"gui/501/{paths.label}") + self.assertEqual(result["websocketUrl"], "ws://127.0.0.1:4766/rpc") self.assertTrue(paths.runner.is_file()) - self.assertTrue(paths.plist.is_file()) run.assert_called_once_with([str(paths.runner), "check"], check=True) self.assertEqual( [call.args for call in launchctl_call.call_args_list], From 4b63e5695c8bcd2766b5d413c2444f7ef8a9255c Mon Sep 17 00:00:00 2001 From: Chris Busillo Date: Thu, 23 Jul 2026 16:20:08 -0400 Subject: [PATCH 4/4] fix(package): ignore stale malformed daemon pid files --- scripts/codex_lab_package/supervisor.py | 4 ++-- scripts/codex_lab_package/test_supervisor.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/codex_lab_package/supervisor.py b/scripts/codex_lab_package/supervisor.py index 501d47e1eaf..8c7d8aa898d 100644 --- a/scripts/codex_lab_package/supervisor.py +++ b/scripts/codex_lab_package/supervisor.py @@ -206,7 +206,7 @@ def build_supervisor_runner( verify_no_updater() {{ [ ! -f "$UPDATER_PID_FILE" ] && return 0 updater_pid=$(json_field "$UPDATER_PID_FILE" pid) - case "$updater_pid" in ''|*[!0-9]*) return 1 ;; esac + case "$updater_pid" in ''|*[!0-9]*) return 0 ;; esac /bin/kill -0 "$updater_pid" 2>/dev/null || return 0 updater_command=$(/bin/ps -p "$updater_pid" -o command= 2>/dev/null || true) case " $updater_command " in @@ -217,7 +217,7 @@ def build_supervisor_runner( verify_no_pid_daemon() {{ [ ! -f "$DAEMON_PID_FILE" ] && return 0 daemon_pid=$(json_field "$DAEMON_PID_FILE" pid) - case "$daemon_pid" in ''|*[!0-9]*) return 1 ;; esac + case "$daemon_pid" in ''|*[!0-9]*) return 0 ;; esac /bin/kill -0 "$daemon_pid" 2>/dev/null || return 0 daemon_command=$(/bin/ps -p "$daemon_pid" -o command= 2>/dev/null || true) case " $daemon_command " in diff --git a/scripts/codex_lab_package/test_supervisor.py b/scripts/codex_lab_package/test_supervisor.py index ff29626b22e..5441de6f21b 100644 --- a/scripts/codex_lab_package/test_supervisor.py +++ b/scripts/codex_lab_package/test_supervisor.py @@ -47,6 +47,7 @@ def test_runner_and_plist_pin_direct_websocket_engine(self) -> None: self.assertIn("EXPECTED_SOURCE_COMMIT=" + "b" * 40, runner) self.assertIn("LISTEN_URL=ws://127.0.0.1:4766", runner) self.assertIn("app-server --remote-control --listen", runner) + self.assertEqual(runner.count("in ''|*[!0-9]*) return 0"), 2) self.assertNotIn("app-server daemon start", runner) self.assertNotIn('"$MANAGED_CLI" app-server daemon pid-update-loop', runner)