feat(ops): recreate VM containers that are persistently unhealthy

Dockerfile.test

@@ -25,7 +25,7 @@ COPY tests ./tests
 # docker-compose.yml.j2 is here because the compose-generation test renders the
 # REAL template — a stub would assert nothing about what actually ships.
 COPY requirements-api.txt requirements-mcpunifier.txt docker-compose.yml.example docker-compose.yml.j2 run.sh ./
-COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py scripts/wickworks-healthcheck.py scripts/recreate-vm.sh ./scripts/
+COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py scripts/wickworks-healthcheck.py scripts/vm-watchdog.py scripts/recreate-vm.sh ./scripts/
 COPY assets/binaries.lock.json ./assets/
 
 ENV PYTHONPATH=/app

Dockerfile.watchdog

@@ -0,0 +1,30 @@
+# VM crash watchdog.
+#
+# Recovery is a coordinated compose recreate (VM + the sidecars sharing its
+# netns), not a Docker-API restart — see scripts/vm-watchdog.py for why a
+# restart strands the sidecar. That means this image needs the docker CLI and
+# the compose plugin, plus bash and PyYAML for scripts/recreate-vm.sh, which is
+# the helper it shells out to.
+#
+# Pinned by digest, not tag: this container mounts the Docker socket, which is
+# root-equivalent on the host, and a moving tag would hand that to whatever the
+# registry serves tomorrow. The digest is the multi-arch OCI index for
+# python:3.12-alpine, so it still resolves per-platform.
+# To update: docker buildx imagetools inspect python:3.12-alpine
+FROM python:3.12-alpine@sha256:d09d15e60962ca365d1cd544a48773bac9d33f2fb1b00f2aa0deec78ade7dc31
+
+# docker-cli-compose provides `docker compose`; recreate-vm.sh is bash and
+# parses the compose file with PyYAML. The Python dependency is pinned by
+# version and hash (see requirements-watchdog.txt for why); --require-hashes
+# makes pip fail closed on anything not in that file.
+COPY requirements-watchdog.txt /tmp/requirements-watchdog.txt
+RUN apk add --no-cache bash docker-cli docker-cli-compose \
+    && pip install --no-cache-dir --require-hashes -r /tmp/requirements-watchdog.txt \
+    && rm /tmp/requirements-watchdog.txt
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# The watchdog script and recreate-vm.sh stay bind-mounted by compose rather
+# than baked in, so a fix to either is a container restart and not a rebuild.
+CMD ["python", "-u", "/vm-watchdog.py"]

docker-compose.yml.example

@@ -91,21 +91,48 @@ services:
       - ./scripts/rotate-logs.sh:/rotate.sh:ro
     command: ["sh", "/rotate.sh"]
 
-  # nginx is the single entry point for all terminal APIs. Routes
-  # /<broker>/<account>/... to mt5:<terminal_port> (per-terminal Python
-  # API process inside the Windows VM, reachable via mt5 container's
-  # iptables DNAT). Auto-generated from config/config.yaml by run.sh.
-  # Bound to 127.0.0.1:8888 so it's loopback-only by default — LAN
-  # exposure is opt-in (change the host bind), tailnet exposure is via
-  # the optional tailscale sidecar below.
-  # Unified MCP endpoint. One MCP session that reaches every terminal, with
-  # broker/account as tool parameters, instead of one endpoint per terminal.
-  # The per-terminal /<broker>/<account>/mcp endpoints keep working unchanged;
-  # nginx routes /mcp/ here.
-  #
-  # Reads the same config/config.yaml that generates the nginx routing, so it
-  # cannot route somewhere nginx does not. It never waits for terminals: a
-  # terminal that is down fails only the calls naming it.
+  # VM crash watchdog. dockurr/windows keeps the container up while the
+  # Windows guest may have crashed internally, so restart: unless-stopped
+  # never fires and every terminal API in that VM stays dead. This sidecar
+  # polls Docker health through the socket and restarts a VM only after its
+  # health has stayed unhealthy for a sustained FailingStreak, with
+  # exponential backoff and bounded retries; state lives on a named volume.
+  # Runs inside the compose project (docker compose up -d), no host cron.
+  # See scripts/vm-watchdog.py.
+  vm-watchdog:
+    # Needs the docker CLI + compose plugin to run scripts/recreate-vm.sh, so
+    # it is built rather than pulled. The base image is digest-pinned inside
+    # the Dockerfile (this container mounts the Docker socket).
+    build:
+      context: .
+      dockerfile: Dockerfile.watchdog
+    restart: unless-stopped
+    command: ["python", "-u", "/vm-watchdog.py"]
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./scripts/vm-watchdog.py:/vm-watchdog.py:ro
+      - vm-watchdog-state:/state
+      # The project itself, at the SAME absolute path the host uses. Compose
+      # resolves the relative bind mounts in this file client-side, so a
+      # different path in here would rewrite every mount to somewhere that
+      # does not exist on the host. run.sh exports MT5_PROJECT_DIR; the
+      # watchdog refuses to act (and says so at startup) if it is unset.
+      - ${MT5_PROJECT_DIR:?MT5_PROJECT_DIR must be the absolute host path of this project (run.sh exports it; otherwise set it in .env)}:${MT5_PROJECT_DIR}:ro
+    environment:
+      WATCHDOG_STATE_DIR: /state
+      WATCHDOG_PROJECT_DIR: ${MT5_PROJECT_DIR}
+      WATCHDOG_RECREATE_SCRIPT: ${MT5_PROJECT_DIR}/scripts/recreate-vm.sh
+      # The helper the watchdog runs calls `docker compose`, which interpolates
+      # ${MT5_PROJECT_DIR:?} in THIS file again. The watchdog sets it in the
+      # helper's environment itself; it is handed through here as well so a
+      # human running `docker compose exec vm-watchdog …/recreate-vm.sh` gets
+      # the same environment the watchdog uses.
+      MT5_PROJECT_DIR: ${MT5_PROJECT_DIR}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
   mcpunifier:
     build:
       context: .
@@ -198,3 +225,8 @@ services:
   #     - NET_RAW
   #   depends_on:
   #     - nginx
+
+volumes:
+  # Persistent per-container watchdog state (last restart, attempts,
+  # healthy-since) so backoff survives the watchdog's own restarts.
+  vm-watchdog-state:

docker-compose.yml.j2

@@ -120,6 +120,48 @@ services:
       - ./scripts/rotate-logs.sh:/rotate.sh:ro
     command: ["sh", "/rotate.sh"]
 
+  # VM crash watchdog. dockurr/windows keeps the container up while the
+  # Windows guest may have crashed internally, so restart: unless-stopped
+  # never fires and every terminal API in that VM stays dead. This sidecar
+  # polls Docker health through the socket and restarts a VM only after its
+  # health has stayed unhealthy for a sustained FailingStreak, with
+  # exponential backoff and bounded retries; state lives on a named volume.
+  # Runs inside the compose project (docker compose up -d), no host cron.
+  # See scripts/vm-watchdog.py.
+  vm-watchdog:
+    # Needs the docker CLI + compose plugin to run scripts/recreate-vm.sh, so
+    # it is built rather than pulled. The base image is digest-pinned inside
+    # the Dockerfile (this container mounts the Docker socket).
+    build:
+      context: .
+      dockerfile: Dockerfile.watchdog
+    restart: unless-stopped
+    command: ["python", "-u", "/vm-watchdog.py"]
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./scripts/vm-watchdog.py:/vm-watchdog.py:ro
+      - vm-watchdog-state:/state
+      # The project itself, at the SAME absolute path the host uses. Compose
+      # resolves the relative bind mounts in this file client-side, so a
+      # different path in here would rewrite every mount to somewhere that
+      # does not exist on the host. run.sh exports MT5_PROJECT_DIR; the
+      # watchdog refuses to act (and says so at startup) if it is unset.
+      - ${MT5_PROJECT_DIR:?MT5_PROJECT_DIR must be the absolute host path of this project (run.sh exports it; otherwise set it in .env)}:${MT5_PROJECT_DIR}:ro
+    environment:
+      WATCHDOG_STATE_DIR: /state
+      WATCHDOG_PROJECT_DIR: ${MT5_PROJECT_DIR}
+      WATCHDOG_RECREATE_SCRIPT: ${MT5_PROJECT_DIR}/scripts/recreate-vm.sh
+      # The helper the watchdog runs calls `docker compose`, which interpolates
+      # ${MT5_PROJECT_DIR:?} in THIS file again. The watchdog sets it in the
+      # helper's environment itself; it is handed through here as well so a
+      # human running `docker compose exec vm-watchdog …/recreate-vm.sh` gets
+      # the same environment the watchdog uses.
+      MT5_PROJECT_DIR: ${MT5_PROJECT_DIR}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
 {% if enable_mcpunifier|default(true) %}
   # Unified MCP endpoint. One MCP session that reaches every terminal, with
   # broker/account as tool parameters, instead of one endpoint per terminal.
@@ -229,3 +271,8 @@ services:
   #     - NET_RAW
   #   depends_on:
   #     - nginx
+
+volumes:
+  # Persistent per-container watchdog state (last restart, attempts,
+  # healthy-since) so backoff survives the watchdog's own restarts.
+  vm-watchdog-state:

docs/operations.md

@@ -10,6 +10,7 @@ The commands, ports, tunnels, locks, queues, and logs you need after the thing b
 - [Cloudflare Tunnel](#cloudflare-tunnel-optional)
 - [Project structure](#project-structure)
 - [Concurrency and backpressure](#concurrency-and-backpressure)
+- [Auto-recovery](#auto-recovery)
 - [Logs](#logs)
 
 ## Make Targets
@@ -251,6 +252,122 @@ or, with the sidecar list discovered from the generated compose file:
 
 This is covered by the real Compose lifecycle regression in
 `tests/integration/test_wickworks_lifecycle.py`.
+## Auto-recovery
+
+The Windows VM(s) run inside `dockurr/windows` containers with a Docker healthcheck
+(`scripts/healthcheck.sh`) that probes every terminal port this VM owns. A crash
+inside the guest — an unexpected shutdown (Event 6008), a wedged terminal, an
+OOM — leaves the **container** up while the **API** is dead, so
+`restart: unless-stopped` never fires and nothing recovers it on its own.
+
+### VM crash watchdog
+
+`vm-watchdog` is a Compose-managed sidecar for exactly that case. It is part of
+the project (`docker compose up -d` brings it up with everything else) — no
+host cron, no systemd unit, no machine-specific checkout path. It polls Docker
+health through the mounted unix socket and uses Docker's own
+`State.Health.FailingStreak` as the source of truth.
+
+Behavior:
+
+- Scopes itself to this Compose project (`com.docker.compose.project`,
+  discovered from its own container labels) and to the `dockurr/windows` VM
+  image, so it only ever recovers the VM containers — never nginx, wickworks,
+  the log rotator, or other sidecars.
+- **Recovers by recreating the VM together with its netns sidecars**, by
+  running `scripts/recreate-vm.sh <service>` — the same helper documented above
+  and covered by `tests/integration/test_wickworks_lifecycle.py`. It does not
+  use `docker restart`: that keeps the container ID but gives the VM a fresh
+  netns on start, which strands the wickworks sidecar exactly as recreating the
+  VM alone does.
+- Acts **only** after its Docker health has stayed `unhealthy`
+  for `WATCHDOG_MIN_FAILING_STREAK` consecutive healthcheck failures (default
+  `10`, i.e. ~5 minutes at the default 30s interval). A container that is
+  healthy or still starting is never touched, so running backtests on a working
+  VM are never interrupted — the healthcheck stays green the whole time a
+  terminal is serving.
+- Keeps a tiny state record per VM on a named volume, keyed by compose
+  project + service (`/state/<project>.<service>.json` — stable across the
+  recreate that recovery performs, unlike a container id): last restart,
+  attempt count, and when the VM
+  was last observed healthy.
+- Enforces exponential backoff between recovery attempts
+  (`WATCHDOG_BACKOFF_ATTEMPTS`, default `300,900,3600` — 5m → 15m → 1h), so a
+  VM that crashes again immediately after recovery is not restarted into a
+  loop.
+- Stops after `WATCHDOG_MAX_ATTEMPTS` consecutive failed recoveries (default
+  `3`) and logs loudly, instead of threshing forever.
+- Resets the attempt budget only after the VM has stayed **continuously**
+  healthy for `WATCHDOG_RESET_SECONDS` (default `1800`), so a VM that recovered
+  then crashed later gets a fresh budget. Any non-healthy observation — a
+  `starting` container after a restart, or an `unhealthy` poll below the streak
+  threshold — restarts that clock; it does not carry over from an earlier
+  healthy run.
+- Never selects itself, whatever `WATCHDOG_IMAGE_FILTER` is set to — it resolves
+  its own full container id at startup and excludes it by equality (falling
+  back to Docker's short-id hostname only if that inspect fails) — and matches
+  the image **repository exactly** (`dockurr/windows`, `dockurr/windows:5.14`,
+  `dockurr/windows@sha256:…` — not `dockurr/windows-something`).
+- `WATCHDOG_DRY_RUN=1` (or `--dry-run`) prints what it would do without
+  touching any container.
+
+Environment overrides: `WATCHDOG_INTERVAL_SECONDS`, `WATCHDOG_MIN_FAILING_STREAK`,
+`WATCHDOG_IMAGE_FILTER`, `WATCHDOG_BACKOFF_ATTEMPTS`, `WATCHDOG_MAX_ATTEMPTS`,
+`WATCHDOG_RESET_SECONDS`, `WATCHDOG_COMPOSE_PROJECT`, `WATCHDOG_STATE_DIR`,
+`WATCHDOG_DOCKER_SOCKET`, `WATCHDOG_DRY_RUN`, `WATCHDOG_PROJECT_DIR`,
+`WATCHDOG_RECREATE_SCRIPT`, `WATCHDOG_RECREATE_TIMEOUT`.
+
+`WATCHDOG_PROJECT_DIR` is the **host** path of this project, and the compose
+service mounts the project through at that same absolute path. Compose resolves
+the relative bind mounts in `docker-compose.yml` client-side, so a
+container-local path would rewrite every mount to something that does not exist
+on the host. If it is missing the watchdog reports it at startup and refuses to
+act, rather than falling back to a restart that looks like recovery and is not.
+
+That path reaches compose as `MT5_PROJECT_DIR`, and compose interpolates it on
+**every** command against `docker-compose.yml`, not only the first `up`:
+
+- `run.sh` exports it for its own run **and writes it to `.env`**, so `make
+  down`, `make logs` and a manual `docker compose …` keep working after `run.sh`
+  has exited. Starting the stack some other way? Put
+  `MT5_PROJECT_DIR=<absolute host path of this directory>` in `.env` yourself.
+- The watchdog sets it explicitly in the environment of the `recreate-vm.sh` it
+  runs (from its own `WATCHDOG_PROJECT_DIR`), because the container is not
+  handed the host's shell variables. `tests/test_vm_watchdog.py` runs the real
+  helper under exactly that environment, and
+  `tests/integration/test_vm_watchdog_lifecycle.py` drives a real recovery
+  through the built sidecar on a disposable Compose project.
+
+### Busy is not dead — and hung is not busy
+
+`healthcheck.sh` reports a port **healthy** when the TCP handshake completes but
+no HTTP answer arrives inside the probe window: something is listening, the
+guest is just saturated (a compile, a Strategy Tester run). Restarting a VM for
+being busy would turn a slow batch into an outage.
+
+That tolerance is **bounded**. A port that accepts TCP but stays silent for
+`HEALTHCHECK_SLOW_GRACE` consecutive checks (default `10`, ≈5 minutes at the 30s
+interval) is reported as `hung` and the check fails — from there the watchdog's
+own streak gate (`WATCHDOG_MIN_FAILING_STREAK`, another ≈5 minutes) applies, so
+a wedged API is recovered in roughly ten minutes rather than never. The
+per-port counters live in `HEALTHCHECK_STATE_DIR` (default `/tmp/healthcheck-slow`
+inside the VM container); an HTTP answer or a refused connection resets a port's
+count, and a recreate starts every count from zero. A refused connection
+(nothing listening) is `DOWN` immediately, as before. If the counters cannot be
+written (a full disk), the bound is off for that check and the verdict says so:
+`ok (slow but listening: …) [slow-state unwritable: hung detection off]`.
+
+**Blast radius.** One hung terminal API is enough to mark the whole VM `DOWN`,
+and the watchdog's recovery is the whole VM — every other terminal on it, and
+whatever they were running, goes with it. That is the same rule the check has
+always applied to a dead port; it is just now applied to a hung one after the
+grace. When you catch a single wedged terminal before the watchdog does,
+`POST /terminal/restart` on that terminal (see `docs/rest-api.md`) is the
+cheaper first response.
+
+This complements the in-VM `MT5AutoReboot` scheduled task, which reboots on a
+fixed timer and can interrupt long-running backtests; operators who disable that
+task still get crash recovery from the watchdog.
 
 ## Logs
 

requirements-watchdog.txt

@@ -0,0 +1,16 @@
+# Dependencies baked into the vm-watchdog image (Dockerfile.watchdog).
+#
+# Pinned by version AND hash: this image mounts the root-equivalent Docker
+# socket, so "whatever PyPI serves today" is not an acceptable input to it -
+# same argument as the base-image digest pin above it. --require-hashes makes
+# pip refuse anything not listed here, including a compromised or yanked
+# re-upload under the same version number.
+#
+# The three hashes are the two musllinux cp312 wheels (python:3.12-alpine on
+# x86_64 / aarch64) plus the sdist as a fallback for any other platform.
+# To update: pick the new version on https://pypi.org/pypi/PyYAML/json and
+# copy the sha256 digests for the matching artifacts.
+pyyaml==6.0.2 \
+    --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \
+    --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
+    --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e

run.sh

@@ -2,6 +2,27 @@
 set -eo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# The vm-watchdog container recovers a crashed VM by running
+# scripts/recreate-vm.sh, which shells out to `docker compose`. Compose
+# resolves this file's relative bind mounts client-side, so the watchdog needs
+# the project at the SAME absolute path the host uses. Exported (not just set)
+# because compose interpolates it into docker-compose.yml at up time.
+export MT5_PROJECT_DIR="${MT5_PROJECT_DIR:-${DIR}}"
+# It must be THIS checkout. A stale export from another clone would be
+# persisted to .env below and then acted on by the watchdog, which would
+# recreate this project's VMs from the other clone's compose file.
+if [ "$(readlink -f "${MT5_PROJECT_DIR}")" != "$(readlink -f "${DIR}")" ]; then
+    echo "ERROR: MT5_PROJECT_DIR is '${MT5_PROJECT_DIR}' but this checkout is '${DIR}'."
+    echo "  Unset it (run.sh derives it) or point it at this directory."
+    exit 1
+fi
+case "${MT5_PROJECT_DIR}" in
+*"'"*)
+    echo "ERROR: the project path contains a single quote, which .env cannot carry: ${MT5_PROJECT_DIR}"
+    exit 1
+    ;;
+esac
 DEBLOAT=0
 for arg in "$@"; do
     if [ "$arg" = "--debloat" ]; then
@@ -134,6 +155,15 @@ done
 # Generate fresh .env each run.
 : >"${DIR}/.env"
 
+# Compose interpolates ${MT5_PROJECT_DIR:?} in docker-compose.yml on EVERY
+# compose command, not only the `up` below - so the value has to outlive this
+# shell. The export above covers this script; this line covers `make down`,
+# `make logs`, a manual `docker compose`, and the vm-watchdog's own recreate,
+# all of which run after it has exited. Written first, so a later failure in
+# this script cannot leave .env without it. Single-quoted: unquoted, a path
+# with `$` or ` #` in it is mangled by compose's dotenv parser.
+echo "MT5_PROJECT_DIR='${MT5_PROJECT_DIR}'" >>"${DIR}/.env"
+
 API_TOKEN=$(python3 "$CFG" api_token)
 if [ -n "${API_TOKEN}" ]; then
     echo "API_TOKEN=${API_TOKEN}" >>"${DIR}/.env"