From b2dce0e722507e98ceda469b6a5d398005bf19b6 Mon Sep 17 00:00:00 2001 From: Lyndz Williams Date: Thu, 9 Jul 2026 16:32:45 +0100 Subject: [PATCH] fix: run hyperhealth migrations once, and never stamp on failure Two problems, one boot path. 1. Migrations ran three times per boot. The Dockerfile CMD runs `alembic upgrade head`, then execs uvicorn with --workers 2. Each worker runs the FastAPI lifespan, and lifespan also shelled out to `alembic upgrade head`. So the CMD migrated once and the two workers migrated again, concurrently, racing each other. Logs showed two hyperhealth.startup and two migrations_completed events on every boot. Migrations belong in the CMD, before uvicorn forks. Removed the subprocess call from lifespan (and the now-unused subprocess import). 2. `|| alembic stamp 001` could mark an unmigrated database as current. 001 is head. `alembic stamp 001` writes the version row and creates nothing else. Proved with alembic's offline SQL mode: alembic upgrade head --sql -> CREATE TABLE alert_policies CREATE TABLE self_heal_policies CREATE TABLE check_definitions CREATE TABLE check_results INSERT INTO alembic_version_hyperhealth ('001') alembic stamp 001 --sql -> INSERT INTO alembic_version_hyperhealth ('001') If `upgrade head` failed for any reason that left the DB reachable -- a lock timeout, a conflicting object, a failing step -- the fallback declared the schema current without building it, and every later `upgrade head` became a no-op. On 2026-06-24 the lifespan copy timed out after 30s on every boot and was swallowed at level=warning. CMD is now `alembic upgrade head && exec uvicorn ...`. On failure the container exits; compose has restart: unless-stopped and waits for postgres condition: service_healthy, so it retries against a live DB instead of lying about the schema. Verified on the running stack after rebuild: migrations_* app events 2 -> 0 alembic invocations 3 -> 1 hyperhealth.startup 2 (unchanged; that is --workers 2) alembic_version_hyperhealth still 001, all four tables present api healthy, worker healthy, 0 restarts The version table is alembic_version_hyperhealth, dedicated in migrations/env.py, so none of this touches core's shared alembic_version. Co-Authored-By: Claude Opus 4.8 --- agents/hyperhealth/Dockerfile | 7 +++++-- agents/hyperhealth/main.py | 23 +++++------------------ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/agents/hyperhealth/Dockerfile b/agents/hyperhealth/Dockerfile index 30ddb78f..3fc08199 100644 --- a/agents/hyperhealth/Dockerfile +++ b/agents/hyperhealth/Dockerfile @@ -46,5 +46,8 @@ EXPOSE 8090 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=5 \ CMD curl -f http://localhost:8090/health || exit 1 -# Run migrations on container start: upgrade head, fallback to stamp if needed -CMD ["sh", "-c", "alembic upgrade head || alembic stamp 001 && exec uvicorn main:app --host 0.0.0.0 --port 8090 --workers 2"] +# Migrate once, here, before uvicorn forks its workers. Never `|| alembic stamp`: +# 001 is head, so stamping on failure marks an unmigrated DB as fully migrated and +# every later `upgrade head` becomes a no-op. On failure we exit; compose has +# restart: unless-stopped and waits for postgres to be healthy, so it retries. +CMD ["sh", "-c", "alembic upgrade head && exec uvicorn main:app --host 0.0.0.0 --port 8090 --workers 2"] diff --git a/agents/hyperhealth/main.py b/agents/hyperhealth/main.py index 5ca4dd8a..0606fa61 100644 --- a/agents/hyperhealth/main.py +++ b/agents/hyperhealth/main.py @@ -7,7 +7,6 @@ import os import sys import uuid -import subprocess from contextlib import asynccontextmanager from datetime import datetime, timedelta from pathlib import Path @@ -105,23 +104,11 @@ def _get_session_factory(): async def lifespan(app: FastAPI): log.info("hyperhealth.startup", environment=ENVIRONMENT) engine = _get_engine() - - # Run Alembic migrations on startup - try: - result = subprocess.run( - ["alembic", "upgrade", "head"], - cwd=_HERE, - capture_output=True, - text=True, - timeout=30 - ) - if result.returncode == 0: - log.info("hyperhealth.migrations_completed") - else: - log.warning("hyperhealth.migrations_warning", stderr=result.stderr) - except Exception as e: - log.warning("hyperhealth.migrations_error", error=str(e)) - + + # Migrations run once in the container CMD, before uvicorn forks its + # workers. lifespan executes per worker, so running alembic here meant + # concurrent `upgrade head` calls racing each other on every boot. + log.info("hyperhealth.db_ready") await _ping_healer() yield