Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions gateway/drain_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ def restart_wait_read(runner, method, path) -> bool:
"ping", "runtime.describe", "session.resume", "session.detach", "session.events.since",
"session.info", "session.list", "prompt.submit", "prompt.receipt", "session.interrupt",
"approval.pending", "approval.received", "approval.respond", "clarify.respond", "clarify.lock",
# Reads an ACP editor's reattach (load_session) needs before it answers; refusing them
# failed every editor reconnect for the whole after-turn wait.
"config.get", "session.stats",
})


Expand Down
275 changes: 118 additions & 157 deletions gateway/handover.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,17 @@
"""Blue-green gateway restart handover.

Successor-first restart: blue (the running gateway) spawns green (``gateway run``) as a
child in the same supervisor cgroup. Green serves sessions once its runtime reaches the
ready state; blue keeps serving its established connections until their turns finish,
then exits through the existing stop path. The MainPID switch is sent by BLUE (not green:
``NotifyAccess=main`` drops notify messages from any other pid) and only after green is
ready and feeding its watchdog, so ``Restart=always`` never fires behind green's back.

Ownership rules that keep the overlap safe:

- The PID record (``gateway.pid``, O_EXCL) stays with blue until blue exits; green claims
it from a deferred task that polls for the predecessor's exit. A lost claim is fatal for
green (SIGTERM self-stop): an unrecorded gateway must not keep serving.
- Platform locks: green carries NO takeover authority. Its adapters fail to lock while
blue holds them and pick up through the existing reconnect watcher once blue's stop
releases them; blue is never signalled by green.
- Blue sends ``MAINPID=`` only after green's control socket answers ``identify`` with the
successor pid AND descriptor state ``ready`` (adapters, cron, housekeeping running).

Known protocol remainder (kept out of v1, flag stays default-off): the profile
reservation / session-store handoff, control-socket session affinity for turns parked on
approval/clarify, and fixed-port listeners (api_server) that cannot rebind until blue's
stop closes them.
child in the same supervisor cgroup. Green stands by (:func:`stand_by`) BEFORE any ownership
guard — host attach, duplicate-process, PID record, profile reservation — writes a standby
marker, feeds systemd's watchdog and blocks until blue exits. Blue sees the marker, sends
``MAINPID=<green>`` (only the current main may: ``NotifyAccess=main``), drains and exits;
green then boots through the normal guards, which all see a dead owner. Nothing is
bypassed and two processes never hold the stores.

Green carries NO platform-lock takeover authority (takeover SIGTERMs the holder).

Not covered: serving NEW sessions while blue drains (store custody, approval affinity,
fixed-port listeners) — blue's drain still refuses them.

Requires a ``Type=notify`` unit (``NOTIFY_SOCKET`` present) so the MainPID switch works;
without it the handover is refused and restarts keep the classic drain-exit shape.
Expand All @@ -30,16 +20,17 @@
from __future__ import annotations

import asyncio
import inspect
import contextlib
import json
import logging
import os
import shlex
import signal
import subprocess
import sys
import time
from dataclasses import dataclass
from typing import Any, Callable, Optional
from pathlib import Path
from typing import Optional

logger = logging.getLogger(__name__)

Expand All @@ -49,12 +40,110 @@

READY_TIMEOUT_S = 20.0
FALLBACK_TERMINATE_S = 10.0
# Blue's drain budget already bounds its stop; this is only a safety net for arming the
# deferred PID-record claim, and arming late is safe (blue's own exit path also cleans up).
PREDECESSOR_EXIT_TIMEOUT_S = 1800.0
# Green is MainPID from standby-confirm on, but its own SystemdWatchdog starts only at READY;
# keep feeding across the post-takeover boot (~30s observed vs WatchdogSec=30s).
BOOT_FEED_S = 120.0
_POLL_INTERVAL_S = 0.25

_parsed_context: Optional["HandoverContext"] = None
_parsed_context: Optional[HandoverContext] = None


def standby_marker_path(parent_pid: int) -> str:
"""Green writes this after interpreter/config pre-warm; blue polls for it before the
MAINPID switch. One file per predecessor pid, in HERMES_HOME (same trust domain as the
PID record), so a stale marker can never vouch for a recycled pid."""
from hermes_constants import get_hermes_home
return str(Path(get_hermes_home()) / f"gateway.handover-standby.{os.getpid()}.{parent_pid}.tmp")


def write_standby_marker(parent_pid: int) -> bool:
"""Green side: pre-warm done, predecessor-wait armed. Consumed by green after takeover."""
try:
path = standby_marker_path(parent_pid)
with open(path, "w") as f:
f.write(json.dumps({"successor_pid": os.getpid(), "parent_pid": parent_pid}))
os.chmod(path, 0o600)
return True
except OSError:
return False


def clear_standby_marker(parent_pid: int) -> None:
from hermes_constants import get_hermes_home
with contextlib.suppress(OSError):
os.unlink(standby_marker_path(parent_pid))


def poll_standby_marker(parent_pid: int) -> Optional[int]:
"""Blue side: successor pid if its standby marker is present, else None."""
from hermes_constants import get_hermes_home
path = Path(get_hermes_home()) / f"gateway.handover-standby.*.{parent_pid}.tmp"
matches = sorted(Path(get_hermes_home()).glob(path.name))
for match in matches:
try:
payload = json.loads(match.read_text())
pid = int(payload.get("successor_pid", 0))
except (OSError, ValueError):
continue
if pid > 0:
return pid
return None


async def wait_successor_standby(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool:
"""Blue side: poll for green's standby marker (pre-warm done, watching the predecessor).

Green runs its ownership bootstrap only AFTER blue exits, so standby — not a bound
control socket — is the earliest trustworthy signal. Once green confirms standby it is
also feeding the watchdog, which is what makes an early MAINPID switch safe.
"""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
if proc.poll() is not None:
logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid)
return False
if poll_standby_marker(os.getpid()) == proc.pid: # green keys its marker by OUR pid
return True
await asyncio.sleep(_POLL_INTERVAL_S)
return False


def stand_by(ctx: HandoverContext) -> None:
"""Green side, before ANY ownership guard: mark standby, keep systemd's watchdog fed and
block until the predecessor exits. Sync on purpose — it runs before the event loop.

No timeout: blue's own restart-handoff watchdog hard-exits it by its deadline, and giving
up early would boot green into a live owner's guards (exit -> unit restart -> SIGKILL of
blue mid-drain)."""
import threading
with contextlib.suppress(Exception):
# A long drain is not a wedged boot; run_gateway re-arms it after the guards.
from hermes_startup_watchdog import disarm_startup_watchdog
disarm_startup_watchdog()
gone = threading.Event()
threading.Thread(target=_feed_watchdog, args=(gone,), daemon=True,
name="gateway-handover-watchdog-feed").start()
write_standby_marker(ctx.parent_pid)
print(f"⏳ Blue-green handover: standing by until gateway PID {ctx.parent_pid} exits", flush=True)
try:
while predecessor_alive(ctx):
time.sleep(_POLL_INTERVAL_S)
finally:
clear_standby_marker(ctx.parent_pid)
gone.set()


def _feed_watchdog(predecessor_gone) -> None:
# ponytail: blind feed for BOOT_FEED_S after takeover — systemd cannot see a wedged boot in
# that window (the startup watchdog still can); stop on the runner's READY if it matters.
from gateway.systemd_notify import notify
while not predecessor_gone.wait(1.0):
notify("WATCHDOG=1")
end = time.monotonic() + BOOT_FEED_S
while time.monotonic() < end:
notify("WATCHDOG=1")
time.sleep(1.0)


def enabled(flag: bool) -> bool:
Expand Down Expand Up @@ -141,7 +230,7 @@ def spawn_successor() -> Optional[subprocess.Popen]:
from hermes_cli.gateway import _profile_arg
full_argv = [*argv, *shlex.split(_profile_arg()), "gateway", "run"]
try:
proc = subprocess.Popen( # noqa: ASYNC220 -- spawn returns at once; the child outlives this task
proc = subprocess.Popen(
full_argv, env={**os.environ, **successor_env()},
)
except OSError as exc:
Expand All @@ -151,55 +240,6 @@ def spawn_successor() -> Optional[subprocess.Popen]:
return proc


def _identify_pid_and_state(socket_path) -> Optional[tuple[int, str]]:
"""One public ``identify`` round-trip; (answering pid, descriptor state) or None."""
import socket as _socket
try:
with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock:
sock.settimeout(1.5)
sock.connect(str(socket_path))
sock.sendall(json.dumps({"id": 1, "protocol": 1, "verb": "identify"}).encode() + b"\n")
with sock.makefile("rb") as reader:
line = reader.readline(4096)
payload = json.loads(line.decode("utf-8"))
except (OSError, ValueError, UnicodeDecodeError):
return None
if not isinstance(payload, dict) or payload.get("ok") is not True:
return None
result = payload.get("result")
if not isinstance(result, dict):
return None
pid = result.get("pid")
state = result.get("state")
if not isinstance(pid, int) or pid <= 0 or not isinstance(state, str):
return None
return pid, state


async def wait_successor_ready(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool:
"""Blue side: poll until the successor ITSELF answers identify in the ready state.

Blue may still answer until green binds the socket name, and a bound-but-starting
green answers ``state != 'ready'`` — neither counts. Blocking socket I/O runs in a
thread so the event loop (streaming, adapters, in-flight turns) never stalls.
"""
from gateway.control_socket import resolve_client_socket_path
from hermes_constants import get_hermes_home
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
if proc.poll() is not None:
logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid)
return False
path = resolve_client_socket_path(get_hermes_home())
if path is not None:
answer = await asyncio.to_thread(_identify_pid_and_state, path)
if answer is not None and answer[0] == proc.pid and answer[1] == "ready":
return True
await asyncio.sleep(_POLL_INTERVAL_S)
return False


def terminate_successor(proc: subprocess.Popen) -> None:
"""Blue side: a successor that was never promoted must not outlive the fallback."""
if proc.poll() is not None:
Expand All @@ -215,82 +255,3 @@ def terminate_successor(proc: subprocess.Popen) -> None:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass


async def arm_after_predecessor_exit(
ctx: HandoverContext,
on_exit: Callable[[], Any],
timeout_s: float = PREDECESSOR_EXIT_TIMEOUT_S,
) -> None:
"""Green side: run ``on_exit`` once the predecessor is gone (its stop path releases the
single-owner records). Arms anyway at the timeout — blue's exit path also cleans up."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout_s
while loop.time() < deadline:
if not predecessor_alive(ctx):
break
await asyncio.sleep(_POLL_INTERVAL_S)
else:
logger.warning("Predecessor (PID %d) still alive after %.0fs; claiming deferred records anyway",
ctx.parent_pid, timeout_s)
outcome = on_exit()
if inspect.isawaitable(outcome):
await outcome


def _self_stop() -> None:
"""Stop this gateway through its own signal path (clean drain, supervisor revives)."""
os.kill(os.getpid(), signal.SIGTERM)


def _claim_pid_record(ctx: HandoverContext) -> None:
"""Claim the PID record once the predecessor is gone (the body of the deferred claim).

A lost claim leaves this process as an unrecorded gateway — fatal by self-stop so the
supervisor's next boot takes over cleanly instead of two gateways racing the record.
"""
import atexit
from gateway.status import get_process_start_time, get_running_pid, remove_pid_file, write_pid_file
recorded = get_running_pid()
if recorded is not None and recorded != os.getpid():
try:
os.kill(recorded, 0)
alive = get_process_start_time(recorded) is not None
except OSError:
alive = False
if alive:
logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record; "
"this successor stops itself", recorded)
_self_stop()
return
logger.warning("Replacing stale predecessor PID record (PID %d)", recorded)
remove_pid_file()
try:
write_pid_file()
except FileExistsError as exc:
logger.error("Deferred PID claim lost the PID-file race (%s); this successor stops itself", exc)
_self_stop()
return
atexit.register(remove_pid_file)
from gateway.run import _claim_host_gateway_role
try:
# The predecessor held the host role; with its exit the role is free, but claim it
# explicitly (force = the handover contract) so host-attach resolution points at us.
_claim_host_gateway_role(force=True)
except SystemExit as exc: # the role claim refuses via SystemExit(75) — degrade, not die
logger.error("Deferred host-role claim refused after handover: %s", exc)
except Exception as exc: # noqa: BLE001 -- role claim failure must not undo the PID claim
logger.error("Deferred host-role claim failed after handover: %s", exc)
from hermes_cli.gateway_multiplex_mode import publish_pending_multiplex_decision
publish_pending_multiplex_decision()
logger.info("Blue-green handover complete: PID record claimed (predecessor PID %d exited)",
ctx.parent_pid)


def deferred_pid_claim(ctx: HandoverContext) -> None:
"""Green side: schedule ``_claim_pid_record`` to run once blue exits."""

async def _run() -> None:
await arm_after_predecessor_exit(ctx, lambda: _claim_pid_record(ctx))

asyncio.get_running_loop().create_task(_run())
11 changes: 3 additions & 8 deletions gateway/run_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -798,14 +798,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
# Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as
# --force there disabled the one arbiter of the two-units-at-once race; a replace that took the
# owner over already freed the lock with that process.
# A blue-green handover successor defers the claim instead: the predecessor owns the record
# until it exits, and the deferred task (armed after the control socket serves) takes it over.
from gateway import handover as _handover
_handover_ctx = _handover.handover_from_env()
if _handover_ctx is not None:
logger.info("Blue-green handover: this boot succeeds gateway PID %d", _handover_ctx.parent_pid)
_handover.deferred_pid_claim(_handover_ctx)
elif not _start_gateway_claim_pid_file(force=force):
# Blue-green handover successors stand by in ``run_gateway`` (before every guard), so by
# here the predecessor is gone and this is an ordinary claim.
if not _start_gateway_claim_pid_file(force=force):
release_gateway_runtime_lock()
return False

Expand Down
5 changes: 3 additions & 2 deletions gateway/run_shutdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -1703,9 +1703,10 @@ async def _run_restart() -> None:
# own notify), and falls back to the classic drain-exit restart otherwise.
handover_child: subprocess.Popen | None = None
from gateway import handover as _handover
if not self._stop_requested_by_signal and _handover.enabled(self._blue_green_restart):
if (not getattr(self, "_stop_requested_by_signal", False)
and _handover.enabled(getattr(self, "_blue_green_restart", False))):
handover_child = _handover.spawn_successor()
ready = handover_child is not None and await _handover.wait_successor_ready(handover_child)
ready = handover_child is not None and await _handover.wait_successor_standby(handover_child)
if ready:
from gateway import systemd_notify
if not systemd_notify.notify_mainpid(handover_child.pid):
Expand Down
5 changes: 5 additions & 0 deletions hermes_cli/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -4562,6 +4562,11 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo
idle_exit: a client auto-started this unsupervised gateway, so it exits once idle
(``gateway.unmanaged_idle_exit_seconds``)."""
_guard_official_docker_root_gateway()
# Blue-green successor: wait out the predecessor BEFORE the single-owner guards below,
# which would otherwise see it alive and refuse ("already serves profile").
from gateway import handover as _handover
if (_handover_ctx := _handover.handover_from_env()) is not None:
_handover.stand_by(_handover_ctx)
_attach_to_host_gateway_or_guard(force=force, replace=replace)
_guard_supervised_gateway_conflict(force=force)
_guard_existing_gateway_process_conflict(replace=replace)
Expand Down
Loading