From f5047e09e4e76df0ba32671561a3eaa82250b7e2 Mon Sep 17 00:00:00 2001 From: Raisal P Wardana Date: Mon, 16 Nov 2026 09:52:37 +0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(gateway):=20standby=20successor=20?= =?UTF-8?q?=E2=80=94=20takeover=20at=20predecessor=20exit,=20no=20guard=20?= =?UTF-8?q?bypass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding #2 resolution that sidesteps the custody question entirely: the successor boots to STANDBY (interpreter + config pre-warmed, watchdog fed, standby marker written), waits for the predecessor's exit, then runs the normal ownership bootstrap — attach decision, PID record, profile reservation and stores all see a dead owner, so no guard is bypassed and two processes never hold the stores. Blue promotes green at standby-confirm (sd_notify MAINPID from blue; green feeds the watchdog from standby so the early switch cannot starve it), then drains and exits; green boots fully into a freed home with no cold-start gap. The serve-during-drain protocol (finding #9/#10 custody, approval affinity, fixed ports) stays with the design session — this changes the takeover, not the drain semantics. Tests: standby marker round-trip/isolation, probe vs marker/dead-child, fallback termination, predecessor-exit arming, PID-reuse guard. The claim and identify-probe machinery was removed with the standby design. --- gateway/handover.py | 191 ++++++++++++++------------------- gateway/run_bootstrap.py | 16 ++- gateway/run_shutdown.py | 2 +- tests/gateway/test_handover.py | 121 +++++++-------------- 4 files changed, 131 insertions(+), 199 deletions(-) diff --git a/gateway/handover.py b/gateway/handover.py index 7943193b0aa56..2c222ca3022c6 100644 --- a/gateway/handover.py +++ b/gateway/handover.py @@ -30,15 +30,16 @@ from __future__ import annotations import asyncio +import contextlib import inspect import json import logging import os import shlex -import signal import subprocess import sys from dataclasses import dataclass +from pathlib import Path from typing import Any, Callable, Optional logger = logging.getLogger(__name__) @@ -50,13 +51,90 @@ READY_TIMEOUT_S = 20.0 FALLBACK_TERMINATE_S = 10.0 # Blue's drain budget already bounds its stop; this is only a safety net for arming the -# deferred PID-record claim, and arming late is safe (blue's own exit path also cleans up). +# standby wait, and an overdue wait is safe (blue's exit path also cleans up). PREDECESSOR_EXIT_TIMEOUT_S = 1800.0 _POLL_INTERVAL_S = 0.25 _parsed_context: Optional["HandoverContext"] = None +def standby_marker_path(parent_pid: int) -> str: + """Green writes this after interpreter/config pre-warm; blue polls for it before the + MAINPID switch. One file per predecessor pid, in HERMES_HOME (same trust domain as the + PID record), so a stale marker can never vouch for a recycled pid.""" + from hermes_constants import get_hermes_home + return str(Path(get_hermes_home()) / f"gateway.handover-standby.{os.getpid()}.{parent_pid}.tmp") + + +def write_standby_marker(parent_pid: int) -> bool: + """Green side: pre-warm done, predecessor-wait armed. Consumed by green after takeover.""" + try: + path = standby_marker_path(parent_pid) + with open(path, "w") as f: + f.write(json.dumps({"successor_pid": os.getpid(), "parent_pid": parent_pid})) + os.chmod(path, 0o600) + return True + except OSError: + return False + + +def clear_standby_marker(parent_pid: int) -> None: + from hermes_constants import get_hermes_home + with contextlib.suppress(OSError): + os.unlink(standby_marker_path(parent_pid)) + + +def poll_standby_marker(parent_pid: int) -> Optional[int]: + """Blue side: successor pid if its standby marker is present, else None.""" + from hermes_constants import get_hermes_home + path = Path(get_hermes_home()) / f"gateway.handover-standby.*.{parent_pid}.tmp" + matches = sorted(Path(get_hermes_home()).glob(path.name)) + for match in matches: + try: + payload = json.loads(match.read_text()) + pid = int(payload.get("successor_pid", 0)) + except (OSError, ValueError): + continue + if pid > 0: + return pid + return None + + +async def wait_successor_standby(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool: + """Blue side: poll for green's standby marker (pre-warm done, watching the predecessor). + + Green runs its ownership bootstrap only AFTER blue exits, so standby — not a bound + control socket — is the earliest trustworthy signal. Once green confirms standby it is + also feeding the watchdog, which is what makes an early MAINPID switch safe. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if proc.poll() is not None: + logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid) + return False + if poll_standby_marker(proc.pid) == proc.pid: + return True + await asyncio.sleep(_POLL_INTERVAL_S) + return False + + +async def feed_watchdog_until_predecessor_exit(ctx: HandoverContext) -> None: + """Green side: pet systemd's watchdog while standing by (green may already be MainPID + if blue promoted it at standby-confirm; a non-feeding main gets SIGABRTed).""" + from gateway.systemd_notify import notify, watchdog_interval_seconds + interval = watchdog_interval_seconds() + if not interval: + return + loop = asyncio.get_running_loop() + deadline = loop.time() + PREDECESSOR_EXIT_TIMEOUT_S + while loop.time() < deadline: + notify("WATCHDOG=1") + if not predecessor_alive(ctx): + return + await asyncio.sleep(max(1.0, interval / 2)) + + def enabled(flag: bool) -> bool: """Blue-green restart gate: pre-loaded config flag + notify socket (MainPID switch).""" if sys.platform == "win32" or not os.environ.get("NOTIFY_SOCKET", "").strip(): @@ -151,55 +229,6 @@ def spawn_successor() -> Optional[subprocess.Popen]: return proc -def _identify_pid_and_state(socket_path) -> Optional[tuple[int, str]]: - """One public ``identify`` round-trip; (answering pid, descriptor state) or None.""" - import socket as _socket - try: - with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock: - sock.settimeout(1.5) - sock.connect(str(socket_path)) - sock.sendall(json.dumps({"id": 1, "protocol": 1, "verb": "identify"}).encode() + b"\n") - with sock.makefile("rb") as reader: - line = reader.readline(4096) - payload = json.loads(line.decode("utf-8")) - except (OSError, ValueError, UnicodeDecodeError): - return None - if not isinstance(payload, dict) or payload.get("ok") is not True: - return None - result = payload.get("result") - if not isinstance(result, dict): - return None - pid = result.get("pid") - state = result.get("state") - if not isinstance(pid, int) or pid <= 0 or not isinstance(state, str): - return None - return pid, state - - -async def wait_successor_ready(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool: - """Blue side: poll until the successor ITSELF answers identify in the ready state. - - Blue may still answer until green binds the socket name, and a bound-but-starting - green answers ``state != 'ready'`` — neither counts. Blocking socket I/O runs in a - thread so the event loop (streaming, adapters, in-flight turns) never stalls. - """ - from gateway.control_socket import resolve_client_socket_path - from hermes_constants import get_hermes_home - loop = asyncio.get_running_loop() - deadline = loop.time() + timeout - while loop.time() < deadline: - if proc.poll() is not None: - logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid) - return False - path = resolve_client_socket_path(get_hermes_home()) - if path is not None: - answer = await asyncio.to_thread(_identify_pid_and_state, path) - if answer is not None and answer[0] == proc.pid and answer[1] == "ready": - return True - await asyncio.sleep(_POLL_INTERVAL_S) - return False - - def terminate_successor(proc: subprocess.Popen) -> None: """Blue side: a successor that was never promoted must not outlive the fallback.""" if proc.poll() is not None: @@ -231,66 +260,8 @@ async def arm_after_predecessor_exit( break await asyncio.sleep(_POLL_INTERVAL_S) else: - logger.warning("Predecessor (PID %d) still alive after %.0fs; claiming deferred records anyway", + logger.warning("Predecessor (PID %d) still alive after %.0fs; proceeding anyway", ctx.parent_pid, timeout_s) outcome = on_exit() if inspect.isawaitable(outcome): await outcome - - -def _self_stop() -> None: - """Stop this gateway through its own signal path (clean drain, supervisor revives).""" - os.kill(os.getpid(), signal.SIGTERM) - - -def _claim_pid_record(ctx: HandoverContext) -> None: - """Claim the PID record once the predecessor is gone (the body of the deferred claim). - - A lost claim leaves this process as an unrecorded gateway — fatal by self-stop so the - supervisor's next boot takes over cleanly instead of two gateways racing the record. - """ - import atexit - from gateway.status import get_process_start_time, get_running_pid, remove_pid_file, write_pid_file - recorded = get_running_pid() - if recorded is not None and recorded != os.getpid(): - try: - os.kill(recorded, 0) - alive = get_process_start_time(recorded) is not None - except OSError: - alive = False - if alive: - logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record; " - "this successor stops itself", recorded) - _self_stop() - return - logger.warning("Replacing stale predecessor PID record (PID %d)", recorded) - remove_pid_file() - try: - write_pid_file() - except FileExistsError as exc: - logger.error("Deferred PID claim lost the PID-file race (%s); this successor stops itself", exc) - _self_stop() - return - atexit.register(remove_pid_file) - from gateway.run import _claim_host_gateway_role - try: - # The predecessor held the host role; with its exit the role is free, but claim it - # explicitly (force = the handover contract) so host-attach resolution points at us. - _claim_host_gateway_role(force=True) - except SystemExit as exc: # the role claim refuses via SystemExit(75) — degrade, not die - logger.error("Deferred host-role claim refused after handover: %s", exc) - except Exception as exc: # noqa: BLE001 -- role claim failure must not undo the PID claim - logger.error("Deferred host-role claim failed after handover: %s", exc) - from hermes_cli.gateway_multiplex_mode import publish_pending_multiplex_decision - publish_pending_multiplex_decision() - logger.info("Blue-green handover complete: PID record claimed (predecessor PID %d exited)", - ctx.parent_pid) - - -def deferred_pid_claim(ctx: HandoverContext) -> None: - """Green side: schedule ``_claim_pid_record`` to run once blue exits.""" - - async def _run() -> None: - await arm_after_predecessor_exit(ctx, lambda: _claim_pid_record(ctx)) - - asyncio.get_running_loop().create_task(_run()) diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py index dec71c0254d1f..5089197d7273b 100644 --- a/gateway/run_bootstrap.py +++ b/gateway/run_bootstrap.py @@ -798,14 +798,20 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as # --force there disabled the one arbiter of the two-units-at-once race; a replace that took the # owner over already freed the lock with that process. - # A blue-green handover successor defers the claim instead: the predecessor owns the record - # until it exits, and the deferred task (armed after the control socket serves) takes it over. + # Blue-green handover: a standby successor pre-warms (interpreter + config are loaded by + # this point), feeds the watchdog, and waits for the predecessor's exit BEFORE the + # ownership bootstrap. Every guard below then sees a dead owner instead of a live + # sibling — attach, PID record, profile reservation and stores need no bypass. from gateway import handover as _handover _handover_ctx = _handover.handover_from_env() if _handover_ctx is not None: - logger.info("Blue-green handover: this boot succeeds gateway PID %d", _handover_ctx.parent_pid) - _handover.deferred_pid_claim(_handover_ctx) - elif not _start_gateway_claim_pid_file(force=force): + logger.info("Blue-green handover: this boot succeeds gateway PID %d; standing by", + _handover_ctx.parent_pid) + _handover.write_standby_marker(_handover_ctx.parent_pid) + asyncio.create_task(_handover.feed_watchdog_until_predecessor_exit(_handover_ctx)) + await _handover.arm_after_predecessor_exit(_handover_ctx, lambda: None) + _handover.clear_standby_marker(_handover_ctx.parent_pid) + if not _start_gateway_claim_pid_file(force=force): release_gateway_runtime_lock() return False diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index c29658e923d41..f395cd4fe6a46 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1705,7 +1705,7 @@ async def _run_restart() -> None: from gateway import handover as _handover if not self._stop_requested_by_signal and _handover.enabled(self._blue_green_restart): handover_child = _handover.spawn_successor() - ready = handover_child is not None and await _handover.wait_successor_ready(handover_child) + ready = handover_child is not None and await _handover.wait_successor_standby(handover_child) if ready: from gateway import systemd_notify if not systemd_notify.notify_mainpid(handover_child.pid): diff --git a/tests/gateway/test_handover.py b/tests/gateway/test_handover.py index b5cd821069f97..8adcbbe783fc4 100644 --- a/tests/gateway/test_handover.py +++ b/tests/gateway/test_handover.py @@ -8,7 +8,6 @@ import socket import subprocess import sys -import threading import pytest @@ -73,71 +72,71 @@ def test_notify_mainpid_payload(tmp_path): assert systemd_notify.notify_mainpid(1) is False -def _serve_identify(server, pid, state, replies=64): - server.setblocking(True) +def _write_standby_marker(tmp_path, successor_pid, parent_pid): + marker = tmp_path / f"gateway.handover-standby.{successor_pid}.{parent_pid}.tmp" + marker.write_text(json.dumps({"successor_pid": successor_pid, "parent_pid": parent_pid})) + return marker - def serve(): - for _ in range(replies): - try: - conn, _ = server.accept() - except OSError: - return - with conn: - conn.recv(4096) - conn.sendall(json.dumps({"ok": True, "result": {"pid": pid, "state": state}}).encode() + b"\n") - thread = threading.Thread(target=serve, daemon=True) - thread.start() - return thread - - -def test_wait_successor_ready_requires_ready_state(tmp_path, monkeypatch): - sock_path = tmp_path / "gateway.sock" - server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - server.bind(str(sock_path)) - server.listen(8) - successor_pid = os.getpid() + 777 - thread = _serve_identify(server, successor_pid, "ready") +def test_wait_successor_standby_marker(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.05) - monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + parent = proc.pid + 1 + _write_standby_marker(tmp_path, proc.pid, parent) try: loop = asyncio.new_event_loop() try: ready = loop.run_until_complete( - asyncio.wait_for(handover.wait_successor_ready(proc, timeout=5.0), 8)) + asyncio.wait_for(handover.wait_successor_standby(proc, timeout=3.0), 6)) assert ready is True finally: loop.close() finally: proc.kill() proc.wait() - server.close() - thread.join(timeout=2) -def test_wait_successor_ready_rejects_draining_and_wrong_pid(tmp_path, monkeypatch): - sock_path = tmp_path / "gateway.sock" - server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - server.bind(str(sock_path)) - server.listen(8) - # A draining blue answering on its own socket must never satisfy the probe. - thread = _serve_identify(server, os.getpid(), "draining") +def test_wait_successor_standby_rejects_draining_blue(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) - monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) try: loop = asyncio.new_event_loop() try: - ready = loop.run_until_complete(handover.wait_successor_ready(os.getpid() + 999, timeout=0.3)) + # No marker for THIS successor: blue's own identify/answer can never satisfy it. + ready = loop.run_until_complete(handover.wait_successor_standby(proc, timeout=0.3)) assert ready is False finally: loop.close() finally: proc.kill() proc.wait() - server.close() - thread.join(timeout=2) + + +def test_wait_successor_standby_dead_child(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.05) + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + loop = asyncio.new_event_loop() + try: + # A successor that already exited fails fast (no full-timeout wait). + ready = loop.run_until_complete( + asyncio.wait_for(handover.wait_successor_standby(proc, timeout=30.0), 5)) + assert ready is False + finally: + loop.close() + + +def test_standby_marker_roundtrip(monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + parent = os.getpid() + 321 + assert handover.poll_standby_marker(parent) is None # nothing written yet + assert handover.write_standby_marker(parent) is True + assert handover.poll_standby_marker(parent) == os.getpid() + handover.clear_standby_marker(parent) + assert handover.poll_standby_marker(parent) is None def test_terminate_successor_fallback(): @@ -174,47 +173,3 @@ def test_predecessor_alive_pid_reuse_guard(monkeypatch): finally: child.kill() child.wait() - - -def _patch_claim_status(monkeypatch, recorded_pid, recorded_alive, writes): - import gateway.status as status - monkeypatch.setattr(status, "get_running_pid", lambda *a, **k: recorded_pid) - if recorded_alive: - monkeypatch.setattr(status, "get_process_start_time", lambda pid: 12345) - - def _kill(pid, sig): - return None - else: - def _kill(pid, sig): - raise ProcessLookupError() - monkeypatch.setattr(os, "kill", _kill) - monkeypatch.setattr(status, "write_pid_file", lambda: writes.append(1)) - monkeypatch.setattr(status, "remove_pid_file", lambda: writes.append("removed")) - import hermes_cli.gateway_multiplex_mode as gmm - monkeypatch.setattr(gmm, "publish_pending_multiplex_decision", lambda: None) - import gateway.run as run_mod - monkeypatch.setattr(run_mod, "_claim_host_gateway_role", lambda force=False: None) - - -def test_claim_skips_live_owner(monkeypatch): - writes: list = [] - _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=True, writes=writes) - ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) - handover._claim_pid_record(ctx) - assert writes == [] # a live recorded gateway blocks the claim - - -def test_claim_replaces_stale_owner(monkeypatch): - writes: list = [] - _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=False, writes=writes) - ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) - handover._claim_pid_record(ctx) - assert writes == ["removed", 1] - - -def test_claim_when_record_free(monkeypatch): - writes: list = [] - _patch_claim_status(monkeypatch, recorded_pid=None, recorded_alive=False, writes=writes) - ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) - handover._claim_pid_record(ctx) - assert writes == [1] From 162102b5770fd58695875f0b03ab9bc8f8343838 Mon Sep 17 00:00:00 2001 From: Raisal P Wardana Date: Wed, 18 Nov 2026 19:26:37 +0700 Subject: [PATCH 2/2] fix(gateway): blue-green successor stands by before the single-owner guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The successor `gateway run` exited at once with "The host gateway already serves profile 'default'": run_gateway's host-attach guard saw blue alive and refused, before any handover code ran. The v2 standby (cherry-picked) waited inside start_gateway — still after that guard — so it could never have worked live. - handover.stand_by(): runs at the top of run_gateway, BEFORE every single-owner guard. Writes the standby marker, disarms the 300s startup watchdog (a long drain is not a wedged boot; run_gateway re-arms it), feeds systemd WATCHDOG=1 from a thread (green is MainPID from standby-confirm) through BOOT_FEED_S of post-takeover boot, and blocks until blue exits. All guards then see a dead owner; nothing is bypassed. - wait_successor_standby polled the marker by the CHILD's pid; green keys it by the PARENT's pid, so blue never saw a real standby. Poll by os.getpid(). - drain_control: allow config.get + session.stats during a restart's after-turn wait. ACP load_session calls both; refusing them made every editor reconnect fail with runtime_draining for the whole drain ("sessions failed"). - run_shutdown: getattr for the handover gate attrs (10 restart_drain tests on fork main crashed on runner doubles without them). Tests: stand_by blocks until predecessor exit + marker lifecycle; run_gateway stands by before the attach guard (the live regression); marker keyed by blue's pid; drain allowlist for ACP reattach reads. --- gateway/drain_control.py | 3 + gateway/handover.py | 116 +++++++++++++++------------------ gateway/run_bootstrap.py | 15 +---- gateway/run_shutdown.py | 3 +- hermes_cli/gateway.py | 5 ++ tests/gateway/test_handover.py | 67 ++++++++++++++----- 6 files changed, 117 insertions(+), 92 deletions(-) diff --git a/gateway/drain_control.py b/gateway/drain_control.py index 3ac3efbab9d96..66456e1129777 100644 --- a/gateway/drain_control.py +++ b/gateway/drain_control.py @@ -70,6 +70,9 @@ def restart_wait_read(runner, method, path) -> bool: "ping", "runtime.describe", "session.resume", "session.detach", "session.events.since", "session.info", "session.list", "prompt.submit", "prompt.receipt", "session.interrupt", "approval.pending", "approval.received", "approval.respond", "clarify.respond", "clarify.lock", + # Reads an ACP editor's reattach (load_session) needs before it answers; refusing them + # failed every editor reconnect for the whole after-turn wait. + "config.get", "session.stats", }) diff --git a/gateway/handover.py b/gateway/handover.py index 2c222ca3022c6..687fc34103f35 100644 --- a/gateway/handover.py +++ b/gateway/handover.py @@ -1,27 +1,17 @@ """Blue-green gateway restart handover. Successor-first restart: blue (the running gateway) spawns green (``gateway run``) as a -child in the same supervisor cgroup. Green serves sessions once its runtime reaches the -ready state; blue keeps serving its established connections until their turns finish, -then exits through the existing stop path. The MainPID switch is sent by BLUE (not green: -``NotifyAccess=main`` drops notify messages from any other pid) and only after green is -ready and feeding its watchdog, so ``Restart=always`` never fires behind green's back. - -Ownership rules that keep the overlap safe: - -- The PID record (``gateway.pid``, O_EXCL) stays with blue until blue exits; green claims - it from a deferred task that polls for the predecessor's exit. A lost claim is fatal for - green (SIGTERM self-stop): an unrecorded gateway must not keep serving. -- Platform locks: green carries NO takeover authority. Its adapters fail to lock while - blue holds them and pick up through the existing reconnect watcher once blue's stop - releases them; blue is never signalled by green. -- Blue sends ``MAINPID=`` only after green's control socket answers ``identify`` with the - successor pid AND descriptor state ``ready`` (adapters, cron, housekeeping running). - -Known protocol remainder (kept out of v1, flag stays default-off): the profile -reservation / session-store handoff, control-socket session affinity for turns parked on -approval/clarify, and fixed-port listeners (api_server) that cannot rebind until blue's -stop closes them. +child in the same supervisor cgroup. Green stands by (:func:`stand_by`) BEFORE any ownership +guard — host attach, duplicate-process, PID record, profile reservation — writes a standby +marker, feeds systemd's watchdog and blocks until blue exits. Blue sees the marker, sends +``MAINPID=`` (only the current main may: ``NotifyAccess=main``), drains and exits; +green then boots through the normal guards, which all see a dead owner. Nothing is +bypassed and two processes never hold the stores. + +Green carries NO platform-lock takeover authority (takeover SIGTERMs the holder). + +Not covered: serving NEW sessions while blue drains (store custody, approval affinity, +fixed-port listeners) — blue's drain still refuses them. Requires a ``Type=notify`` unit (``NOTIFY_SOCKET`` present) so the MainPID switch works; without it the handover is refused and restarts keep the classic drain-exit shape. @@ -31,16 +21,16 @@ import asyncio import contextlib -import inspect import json import logging import os import shlex import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable, Optional +from typing import Optional logger = logging.getLogger(__name__) @@ -50,12 +40,12 @@ READY_TIMEOUT_S = 20.0 FALLBACK_TERMINATE_S = 10.0 -# Blue's drain budget already bounds its stop; this is only a safety net for arming the -# standby wait, and an overdue wait is safe (blue's exit path also cleans up). -PREDECESSOR_EXIT_TIMEOUT_S = 1800.0 +# Green is MainPID from standby-confirm on, but its own SystemdWatchdog starts only at READY; +# keep feeding across the post-takeover boot (~30s observed vs WatchdogSec=30s). +BOOT_FEED_S = 120.0 _POLL_INTERVAL_S = 0.25 -_parsed_context: Optional["HandoverContext"] = None +_parsed_context: Optional[HandoverContext] = None def standby_marker_path(parent_pid: int) -> str: @@ -113,26 +103,47 @@ async def wait_successor_standby(proc: subprocess.Popen, timeout: float = READY_ if proc.poll() is not None: logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid) return False - if poll_standby_marker(proc.pid) == proc.pid: + if poll_standby_marker(os.getpid()) == proc.pid: # green keys its marker by OUR pid return True await asyncio.sleep(_POLL_INTERVAL_S) return False -async def feed_watchdog_until_predecessor_exit(ctx: HandoverContext) -> None: - """Green side: pet systemd's watchdog while standing by (green may already be MainPID - if blue promoted it at standby-confirm; a non-feeding main gets SIGABRTed).""" - from gateway.systemd_notify import notify, watchdog_interval_seconds - interval = watchdog_interval_seconds() - if not interval: - return - loop = asyncio.get_running_loop() - deadline = loop.time() + PREDECESSOR_EXIT_TIMEOUT_S - while loop.time() < deadline: +def stand_by(ctx: HandoverContext) -> None: + """Green side, before ANY ownership guard: mark standby, keep systemd's watchdog fed and + block until the predecessor exits. Sync on purpose — it runs before the event loop. + + No timeout: blue's own restart-handoff watchdog hard-exits it by its deadline, and giving + up early would boot green into a live owner's guards (exit -> unit restart -> SIGKILL of + blue mid-drain).""" + import threading + with contextlib.suppress(Exception): + # A long drain is not a wedged boot; run_gateway re-arms it after the guards. + from hermes_startup_watchdog import disarm_startup_watchdog + disarm_startup_watchdog() + gone = threading.Event() + threading.Thread(target=_feed_watchdog, args=(gone,), daemon=True, + name="gateway-handover-watchdog-feed").start() + write_standby_marker(ctx.parent_pid) + print(f"⏳ Blue-green handover: standing by until gateway PID {ctx.parent_pid} exits", flush=True) + try: + while predecessor_alive(ctx): + time.sleep(_POLL_INTERVAL_S) + finally: + clear_standby_marker(ctx.parent_pid) + gone.set() + + +def _feed_watchdog(predecessor_gone) -> None: + # ponytail: blind feed for BOOT_FEED_S after takeover — systemd cannot see a wedged boot in + # that window (the startup watchdog still can); stop on the runner's READY if it matters. + from gateway.systemd_notify import notify + while not predecessor_gone.wait(1.0): notify("WATCHDOG=1") - if not predecessor_alive(ctx): - return - await asyncio.sleep(max(1.0, interval / 2)) + end = time.monotonic() + BOOT_FEED_S + while time.monotonic() < end: + notify("WATCHDOG=1") + time.sleep(1.0) def enabled(flag: bool) -> bool: @@ -219,7 +230,7 @@ def spawn_successor() -> Optional[subprocess.Popen]: from hermes_cli.gateway import _profile_arg full_argv = [*argv, *shlex.split(_profile_arg()), "gateway", "run"] try: - proc = subprocess.Popen( # noqa: ASYNC220 -- spawn returns at once; the child outlives this task + proc = subprocess.Popen( full_argv, env={**os.environ, **successor_env()}, ) except OSError as exc: @@ -244,24 +255,3 @@ def terminate_successor(proc: subprocess.Popen) -> None: proc.wait(timeout=5) except subprocess.TimeoutExpired: pass - - -async def arm_after_predecessor_exit( - ctx: HandoverContext, - on_exit: Callable[[], Any], - timeout_s: float = PREDECESSOR_EXIT_TIMEOUT_S, -) -> None: - """Green side: run ``on_exit`` once the predecessor is gone (its stop path releases the - single-owner records). Arms anyway at the timeout — blue's exit path also cleans up.""" - loop = asyncio.get_running_loop() - deadline = loop.time() + timeout_s - while loop.time() < deadline: - if not predecessor_alive(ctx): - break - await asyncio.sleep(_POLL_INTERVAL_S) - else: - logger.warning("Predecessor (PID %d) still alive after %.0fs; proceeding anyway", - ctx.parent_pid, timeout_s) - outcome = on_exit() - if inspect.isawaitable(outcome): - await outcome diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py index 5089197d7273b..38bd895c58fae 100644 --- a/gateway/run_bootstrap.py +++ b/gateway/run_bootstrap.py @@ -798,19 +798,8 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as # --force there disabled the one arbiter of the two-units-at-once race; a replace that took the # owner over already freed the lock with that process. - # Blue-green handover: a standby successor pre-warms (interpreter + config are loaded by - # this point), feeds the watchdog, and waits for the predecessor's exit BEFORE the - # ownership bootstrap. Every guard below then sees a dead owner instead of a live - # sibling — attach, PID record, profile reservation and stores need no bypass. - from gateway import handover as _handover - _handover_ctx = _handover.handover_from_env() - if _handover_ctx is not None: - logger.info("Blue-green handover: this boot succeeds gateway PID %d; standing by", - _handover_ctx.parent_pid) - _handover.write_standby_marker(_handover_ctx.parent_pid) - asyncio.create_task(_handover.feed_watchdog_until_predecessor_exit(_handover_ctx)) - await _handover.arm_after_predecessor_exit(_handover_ctx, lambda: None) - _handover.clear_standby_marker(_handover_ctx.parent_pid) + # Blue-green handover successors stand by in ``run_gateway`` (before every guard), so by + # here the predecessor is gone and this is an ordinary claim. if not _start_gateway_claim_pid_file(force=force): release_gateway_runtime_lock() return False diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index f395cd4fe6a46..14c8fcd8cd043 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1703,7 +1703,8 @@ async def _run_restart() -> None: # own notify), and falls back to the classic drain-exit restart otherwise. handover_child: subprocess.Popen | None = None from gateway import handover as _handover - if not self._stop_requested_by_signal and _handover.enabled(self._blue_green_restart): + if (not getattr(self, "_stop_requested_by_signal", False) + and _handover.enabled(getattr(self, "_blue_green_restart", False))): handover_child = _handover.spawn_successor() ready = handover_child is not None and await _handover.wait_successor_standby(handover_child) if ready: diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index cbd3e49f79961..de77b1dc0e896 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -4562,6 +4562,11 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo idle_exit: a client auto-started this unsupervised gateway, so it exits once idle (``gateway.unmanaged_idle_exit_seconds``).""" _guard_official_docker_root_gateway() + # Blue-green successor: wait out the predecessor BEFORE the single-owner guards below, + # which would otherwise see it alive and refuse ("already serves profile"). + from gateway import handover as _handover + if (_handover_ctx := _handover.handover_from_env()) is not None: + _handover.stand_by(_handover_ctx) _attach_to_host_gateway_or_guard(force=force, replace=replace) _guard_supervised_gateway_conflict(force=force) _guard_existing_gateway_process_conflict(replace=replace) diff --git a/tests/gateway/test_handover.py b/tests/gateway/test_handover.py index 8adcbbe783fc4..147d3b69ed12a 100644 --- a/tests/gateway/test_handover.py +++ b/tests/gateway/test_handover.py @@ -1,4 +1,4 @@ -"""Blue-green handover primitives: env contract, gates, readiness probe, deferred claim.""" +"""Blue-green handover primitives: env contract, gates, standby, guard ordering.""" from __future__ import annotations @@ -82,8 +82,9 @@ def test_wait_successor_standby_marker(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.05) proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) - parent = proc.pid + 1 - _write_standby_marker(tmp_path, proc.pid, parent) + # Green keys the marker by BLUE's pid (this process); a marker keyed by the child's own + # pid is what the original poll looked for, so blue never saw a real standby. + _write_standby_marker(tmp_path, proc.pid, os.getpid()) try: loop = asyncio.new_event_loop() try: @@ -148,19 +149,55 @@ def test_terminate_successor_fallback(): handover.terminate_successor(already) # no-op on an exited child -def test_arm_after_predecessor_exit(monkeypatch): - child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.2)"]) - ctx = handover.HandoverContext(parent_pid=child.pid, parent_start_ticks=None) - fired = [] +def test_stand_by_blocks_until_predecessor_exit(monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("NOTIFY_SOCKET", raising=False) monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) - loop = asyncio.new_event_loop() - try: - loop.run_until_complete( - asyncio.wait_for(handover.arm_after_predecessor_exit(ctx, lambda: fired.append(1)), 10)) - finally: - loop.close() - child.wait() - assert fired == [1] + monkeypatch.setattr(handover, "BOOT_FEED_S", 0.0) + blue = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.5)"]) + ctx = handover.HandoverContext(parent_pid=blue.pid, parent_start_ticks=None) + seen = [] + real_alive = handover.predecessor_alive + + def alive(c): + seen.append(handover.poll_standby_marker(blue.pid)) # marker visible to blue while waiting + if blue.poll() is not None: + return False + return real_alive(c) + monkeypatch.setattr(handover, "predecessor_alive", alive) + handover.stand_by(ctx) + assert blue.poll() is not None # returned only after blue exited + assert os.getpid() in seen + assert handover.poll_standby_marker(blue.pid) is None # cleared on takeover + + +def test_run_gateway_stands_by_before_single_owner_guards(monkeypatch): + """Regression: the successor hit 'already serves profile' because the standby wait ran + inside start_gateway, after run_gateway's host-attach guard had already refused it.""" + from hermes_cli import gateway as cli_gateway + order = [] + ctx = handover.HandoverContext(parent_pid=os.getpid() + 1, parent_start_ticks=None) + monkeypatch.setattr(handover, "handover_from_env", lambda: ctx) + monkeypatch.setattr(handover, "stand_by", lambda c: order.append("stand_by")) + monkeypatch.setattr(cli_gateway, "_guard_official_docker_root_gateway", lambda: None) + + def attach(**_kw): + order.append("attach_guard") + raise SystemExit(0) + monkeypatch.setattr(cli_gateway, "_attach_to_host_gateway_or_guard", attach) + with pytest.raises(SystemExit): + cli_gateway.run_gateway() + assert order == ["stand_by", "attach_guard"] + + +def test_restart_wait_allows_acp_reattach_reads(): + from types import SimpleNamespace + + from gateway.drain_control import ws_method_refused_while_draining + draining = SimpleNamespace(_draining=True) + for method in ("config.get", "session.stats", "session.resume", "session.info"): + assert not ws_method_refused_while_draining(draining, method), method + assert ws_method_refused_while_draining(draining, "config.set") def test_predecessor_alive_pid_reuse_guard(monkeypatch):