From ce6e924213e38fb2898a73033b82c49f005530df Mon Sep 17 00:00:00 2001 From: MP2EZ <182439403+MP2EZ@users.noreply.github.com> Date: Wed, 17 Jun 2026 21:49:40 -0700 Subject: [PATCH] feat(v1.47): production synthetic monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add active/synthetic monitoring for live campable.co — distinct from the existing playwright.yml release gate, which only tests staging. - Real /healthz endpoint: pings SQLite, returns {status,db,version}, 503 when DB unreachable. Registered before the SPA catch-all so it no longer false-greens through index.html (also fixes the deploy wait-loop check). - prod-smoke.spec.ts: read-only, non-mutating Playwright smoke (homepage, search API, pricing, SEO) safe to run against production. - prod-monitor.yml: daily workflow — curls /healthz + /api/search, runs the smoke vs prod, emails via Resend + opens a 'prod-down' issue on failure. - docs/MONITORING.md: 4-layer model + UptimeRobot + Resend config. Requires repo config: RESEND_API_KEY secret + ALERT_EMAIL_TO var (email step skips gracefully if unset; GitHub issue still fires). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/prod-monitor.yml | 121 +++++++++++++++++++++++++++++ docs/MONITORING.md | 96 +++++++++++++++++++++++ docs/ROADMAP.md | 1 + e2e/tests/prod-smoke.spec.ts | 81 +++++++++++++++++++ src/pnw_campsites/api.py | 36 ++++++++- tests/test_api_endpoints.py | 28 ++++++- 6 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/prod-monitor.yml create mode 100644 docs/MONITORING.md create mode 100644 e2e/tests/prod-smoke.spec.ts diff --git a/.github/workflows/prod-monitor.yml b/.github/workflows/prod-monitor.yml new file mode 100644 index 0000000..79e8c67 --- /dev/null +++ b/.github/workflows/prod-monitor.yml @@ -0,0 +1,121 @@ +name: Prod Monitor + +# Daily synthetic monitor for the LIVE site (https://campable.co). +# +# Distinct from playwright.yml, which is a release GATE: it deploys dev HEAD to +# staging and tests that before promotion. This workflow tests PRODUCTION as it +# actually is, so a Fly outage / TLS expiry / Supabase incident is caught even +# when no real user is around. It performs ZERO mutations (no signup, Stripe, +# or writes) — see e2e/tests/prod-smoke.spec.ts. +# +# Layering (see docs/MONITORING.md): +# UptimeRobot -> "is it down?" every 5 min (external, /healthz) +# THIS -> "do features work?" daily (deep, read-only) +# playwright -> "is the next release safe?" pre-deploy (staging) +# PostHog -> passive real-user error capture +# +# NOTE: the /healthz JSON check only goes green once v1.47 (which adds the real +# /healthz route) is released to prod. Until then that step will fail on manual +# dispatch; the Playwright smoke + /api/search check work against current prod. + +on: + schedule: + - cron: "0 15 * * *" # daily 15:00 UTC (offset from the 08:00 release gate) + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + PROD_URL: https://campable.co + +jobs: + monitor: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Healthz — app + DB readiness + run: | + body=$(curl -sf --max-time 30 "$PROD_URL/healthz") || { + echo "::error::/healthz did not return 2xx"; exit 1; } + echo "$body" + echo "$body" | grep -q '"status":"ok"' || { + echo "::error::/healthz not ok (DB degraded or route not yet deployed)"; exit 1; } + + - name: Search API — DB + provider pipeline + run: | + start=$(date -u -d "+30 days" +%Y-%m-%d) + end=$(date -u -d "+33 days" +%Y-%m-%d) + url="$PROD_URL/api/search?start_date=$start&end_date=$end&state=WA&nights=2&limit=5" + body=$(curl -sf --max-time 60 "$url") || { + echo "::error::/api/search did not return 2xx"; exit 1; } + echo "$body" | grep -q '"campgrounds_checked"' || { + echo "::error::/api/search response missing campgrounds_checked"; exit 1; } + + - name: Install Playwright + browser + working-directory: e2e + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Run read-only prod smoke + working-directory: e2e + env: + E2E_BASE_URL: ${{ env.PROD_URL }} + CI: "true" + run: npx playwright test prod-smoke.spec.ts + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: prod-monitor-report + path: e2e/playwright-report/ + retention-days: 30 + + - name: Email alert on failure (Resend) + if: failure() + env: + RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} + ALERT_EMAIL_FROM: ${{ vars.ALERT_EMAIL_FROM }} + ALERT_EMAIL_TO: ${{ vars.ALERT_EMAIL_TO }} + run: | + if [ -z "$RESEND_API_KEY" ] || [ -z "$ALERT_EMAIL_TO" ]; then + echo "::warning::Resend not configured (RESEND_API_KEY / ALERT_EMAIL_TO); skipping email" + exit 0 + fi + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + from="${ALERT_EMAIL_FROM:-Campable Monitor }" + date_str=$(date -u +%Y-%m-%d) + text=$(printf 'The daily production monitor failed. A feature on the live site may be down.\n\nRun: %s\n\nThe Playwright report + trace are attached to the run as an artifact.' "$run_url") + payload=$(jq -n \ + --arg from "$from" \ + --arg to "$ALERT_EMAIL_TO" \ + --arg subject "🔴 campable.co prod monitor FAILED ($date_str)" \ + --arg text "$text" \ + '{from: $from, to: [$to], subject: $subject, text: $text}') + curl -sf -X POST https://api.resend.com/emails \ + -H "Authorization: Bearer $RESEND_API_KEY" \ + -H "Content-Type: application/json" \ + -d "$payload" + + - name: Open issue on failure + if: failure() + uses: actions/github-script@v7 + with: + script: | + const date = new Date().toISOString().split('T')[0]; + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Prod monitor FAILED — campable.co (${date})`, + body: `The daily production synthetic monitor failed — a live feature may be down.\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}\n\nThe Playwright HTML report + trace are attached as the run's artifact.`, + labels: ['prod-down'], + }); diff --git a/docs/MONITORING.md b/docs/MONITORING.md new file mode 100644 index 0000000..3da5b02 --- /dev/null +++ b/docs/MONITORING.md @@ -0,0 +1,96 @@ +# Monitoring (v1.47) + +How we know campable.co is up and its features work. Four layers, each +answering a different question — none replaces the others. + +| Layer | Question | Cadence | Where | +|-------|----------|---------|-------| +| **UptimeRobot** | Is it down *right now*? | every 5 min | external service (you set up) | +| **Prod monitor** | Do the *features* work on prod? | daily | `.github/workflows/prod-monitor.yml` | +| **Release gate** | Is the *next release* safe? | nightly / per-PR | `.github/workflows/playwright.yml` (staging) | +| **PostHog** | What broke for a *real user*? | passive | PostHog error tracking | + +**Why not just PostHog?** PostHog is *passive* — it only captures an error once +a real user trips it. If the site dies at 3am, PostHog stays quiet until someone +shows up and gets hurt. The first two layers are *active / synthetic*: a robot +drives the live site on a schedule so we hear about breakage first. + +**Why not just the existing nightly?** `playwright.yml` is a release *gate* — it +deploys `dev` HEAD to **staging** and tests that before promotion. It never +touches production, so a Fly outage, expired TLS cert, or Supabase incident on +the *live* site is invisible to it. + +--- + +## Layer 1 — UptimeRobot (you set up, ~5 min) + +Fast "is it down" detection on the real `/healthz` endpoint (added in v1.47 — +pings SQLite, returns `{"status":"ok","db":true,"version":"..."}`, or **503** +when the DB is unreachable). + +1. Create a free account at . +2. **Add New Monitor**: + - Type: **HTTP(s)** + - URL: `https://campable.co/healthz` + - Interval: **5 minutes** + - **Keyword monitoring**: alert if the response does **not** contain + `"status":"ok"`. This way a 503 (DB degraded) *or* a missing keyword both + trigger — not just a hard connection failure. +3. **Alert contact**: add your Outlook address (`…@palouselabs.com`). UptimeRobot + emails on down + recovery. + +> Note: `/healthz` ships with v1.47. Until that release reaches prod, point the +> monitor at `https://campable.co/` (plain reachability) and switch to +> `/healthz` once deployed. + +--- + +## Layer 2 — Daily prod monitor (built in, `.github/workflows/prod-monitor.yml`) + +Runs daily at **15:00 UTC** (and on demand via *Actions → Prod Monitor → Run +workflow*). It checks, against live `https://campable.co`: + +1. `/healthz` returns `"status":"ok"` (app + DB). +2. `/api/search` returns a well-shaped response (DB + provider pipeline). +3. `e2e/tests/prod-smoke.spec.ts` — a **read-only, anonymous** Playwright smoke: + homepage renders, search API works, pricing page renders, SEO state index + renders. + +It is **strictly non-mutating** — no signup, no Stripe, no writes — so it is +safe to point at production. (The four flows in `e2e/tests/` *do* mutate state +and run only against staging.) + +On failure it: uploads the Playwright report, **emails you via Resend**, and +**opens a GitHub issue** labelled `prod-down`. + +### Required repo configuration + +Set under *Settings → Secrets and variables → Actions*: + +| Kind | Name | Value | +|------|------|-------| +| **Secret** | `RESEND_API_KEY` | your Resend API key | +| Variable | `ALERT_EMAIL_TO` | your Outlook address, e.g. `you@palouselabs.com` | +| Variable | `ALERT_EMAIL_FROM` | *(optional)* sender, default `Campable Monitor ` — the domain must be verified in Resend | + +If `RESEND_API_KEY` / `ALERT_EMAIL_TO` are unset, the email step is skipped with +a warning and the GitHub issue still fires (so you're never silently blind). + +> The `ALERT_EMAIL_FROM` domain must be verified in Resend. If `campable.co` +> isn't verified there yet, either verify it or set `ALERT_EMAIL_FROM` to a +> domain that is. + +--- + +## Layer 3 — Release gate (`.github/workflows/playwright.yml`) + +Unchanged by v1.47. Per-PR smoke + nightly full E2E against **staging** +(`campnw-staging`), opening an issue on nightly failure. This is the +pre-promotion safety net, not a production monitor. + +--- + +## Layer 4 — PostHog + +Passive real-user product analytics + error tracking. Keeps the "what did a real +user experience" record that synthetic checks can't reproduce. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5b7c4a7..fc01f35 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -43,6 +43,7 @@ v1.4 [SHIPPED] Monetization Launch — Pro tier gate, Stripe Checkout/Porta v1.41 [SHIPPED] Playwright E2E — Playwright E2E suite — smoke + watch/planner limits + cancel — 4/4 nightly green (2026-05-31) v1.42 -------> Site Polish + Legal — About, Privacy, Terms, footer. Unblocks Stripe live-mode review + Apple App Store URL requirement. v1.45 -------> Native Apps — Capacitor shell, iOS App Store + Google Play, native push/GPS/offline registry +v1.47 -------> Prod Monitoring — Real /healthz (DB probe) + daily read-only prod smoke + UptimeRobot. Catches live-site outages the staging gate can't. See docs/MONITORING.md v2.0 -------> Predictions+ — Statistical model, anomaly alerts, post-mortems (~Q1 2027) ``` diff --git a/e2e/tests/prod-smoke.spec.ts b/e2e/tests/prod-smoke.spec.ts new file mode 100644 index 0000000..b3c46b8 --- /dev/null +++ b/e2e/tests/prod-smoke.spec.ts @@ -0,0 +1,81 @@ +import { test, expect } from "@playwright/test"; + +/** + * Production synthetic monitor — READ-ONLY smoke. + * + * Runs against the LIVE site (E2E_BASE_URL=https://campable.co) on a daily + * schedule via .github/workflows/prod-monitor.yml. Unlike the four flows in + * this directory, it performs ZERO mutations: no signup, no Stripe, no + * POST/DELETE. It only loads anonymous pages and reads the public search API, + * so it is safe to point at production. + * + * Purpose: catch feature regressions on prod (broken homepage, dead search + * pipeline, missing pricing/SEO pages) that the staging release-gate nightly + * cannot see, and that PostHog only surfaces after a real user is harmed. + */ + +// Compute future dates at runtime so the search query never goes stale. +function isoDate(daysFromNow: number): string { + const d = new Date(); + d.setUTCDate(d.getUTCDate() + daysFromNow); + return d.toISOString().slice(0, 10); +} + +test("homepage renders core discovery UI", async ({ page }) => { + await page.goto("/"); + + // The discovery-mode tab switcher (Find a Site / Plan a Trip). + await expect( + page.getByRole("tablist", { name: "Discovery mode" }) + ).toBeVisible(); + + // The structured search form — the app's primary surface. + await expect(page.locator(".search-form")).toBeVisible(); + + // Anonymous header CTA confirms auth wiring rendered. + await expect( + page.getByRole("button", { name: "Sign in" }).first() + ).toBeVisible(); +}); + +test("search API returns a valid response (DB + provider pipeline)", async ({ + request, +}) => { + // The differentiator: a real availability search across the registry. + // We assert the endpoint is healthy and well-shaped — NOT that any site is + // available (external booking systems may legitimately have zero), to avoid + // false alarms from upstream providers. + const res = await request.get("/api/search", { + params: { + start_date: isoDate(30), + end_date: isoDate(33), + state: "WA", + nights: "2", + limit: "5", + }, + timeout: 60_000, + }); + + expect(res.ok()).toBeTruthy(); + const body = await res.json(); + expect(typeof body.campgrounds_checked).toBe("number"); + expect(body.campgrounds_checked).toBeGreaterThan(0); +}); + +test("pricing page renders", async ({ page }) => { + await page.goto("/pricing"); + await expect( + page.getByRole("heading", { name: /pricing/i, level: 1 }) + ).toBeVisible(); + // Both plan cards render. We assert structure, not the CTA: the CTA is + // auth-dependent (anon sees "Sign in to upgrade", free sees "Upgrade to + // Pro") and we stay logged-out / never click it (would hit Stripe). + await expect(page.locator(".pricing-card-free")).toBeVisible(); + await expect(page.locator(".pricing-card-pro")).toBeVisible(); +}); + +test("SEO state index renders", async ({ page }) => { + const res = await page.goto("/campgrounds/wa"); + expect(res?.status()).toBe(200); + await expect(page.getByRole("heading", { level: 1 })).toBeVisible(); +}); diff --git a/src/pnw_campsites/api.py b/src/pnw_campsites/api.py index ab139c6..5a31da7 100644 --- a/src/pnw_campsites/api.py +++ b/src/pnw_campsites/api.py @@ -5,6 +5,7 @@ import asyncio import logging import os +import sqlite3 import time from collections import deque from contextlib import asynccontextmanager @@ -18,7 +19,7 @@ from fastapi.responses import FileResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles -from pnw_campsites.monitor.db import WatchDB +from pnw_campsites.monitor.db import DEFAULT_DB_PATH, WatchDB from pnw_campsites.posthog_client import get_posthog_client from pnw_campsites.providers.goingtocamp import GoingToCampClient from pnw_campsites.providers.recgov import RecGovClient @@ -26,6 +27,9 @@ from pnw_campsites.registry.db import CampgroundRegistry from pnw_campsites.search.engine import SearchEngine +# App version — surfaced by /healthz so monitors can confirm what's deployed. +APP_VERSION = "1.47" + # --------------------------------------------------------------------------- # App state — initialized in lifespan # --------------------------------------------------------------------------- @@ -539,6 +543,36 @@ async def timing_middleware(request: Request, call_next): # takes precedence over /{path:path} app.include_router(seo_router) + +@app.get("/healthz") +async def healthz() -> JSONResponse: + """Liveness + DB readiness probe for synthetic monitoring. + + Returns 200 when SQLite is reachable, 503 otherwise. Defined as an + explicit route so it takes precedence over the SPA catch-all at + /{path:path} — without it, unknown paths fall through to index.html + and any /healthz check would be falsely green during a DB outage. + """ + db_ok = False + try: + conn = sqlite3.connect(str(DEFAULT_DB_PATH)) + try: + conn.execute("SELECT 1") + finally: + conn.close() + db_ok = True + except Exception: + logging.getLogger(__name__).exception("healthz: SQLite check failed") + + return JSONResponse( + { + "status": "ok" if db_ok else "degraded", + "db": db_ok, + "version": APP_VERSION, + }, + status_code=200 if db_ok else 503, + ) + # Mount SEO static assets (tokens.css, seo.css) _seo_static_candidates = [ Path("/app/seo-static"), diff --git a/tests/test_api_endpoints.py b/tests/test_api_endpoints.py index a010cda..63fd7d8 100644 --- a/tests/test_api_endpoints.py +++ b/tests/test_api_endpoints.py @@ -2,7 +2,7 @@ from __future__ import annotations -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch from fastapi.testclient import TestClient @@ -19,6 +19,32 @@ from tests.conftest import make_campground +class TestHealthzEndpoint: + """Tests for GET /healthz — the synthetic-monitoring liveness probe.""" + + def test_healthz_ok_when_db_reachable(self, api_client: TestClient): + """Returns 200 + status:ok when SQLite is reachable.""" + response = api_client.get("/healthz") + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "ok" + assert body["db"] is True + assert body["version"] == api_module.APP_VERSION + + def test_healthz_degraded_when_db_unreachable(self, api_client: TestClient): + """Returns 503 + status:degraded when the SQLite check raises.""" + with patch.object( + api_module.sqlite3, "connect", side_effect=OSError("disk gone") + ): + response = api_client.get("/healthz") + + assert response.status_code == 503 + body = response.json() + assert body["status"] == "degraded" + assert body["db"] is False + + class TestCampgroundsEndpoint: """Tests for GET /api/campgrounds."""