feat(v1.47): production synthetic monitoring

.github/workflows/prod-monitor.yml

@@ -0,0 +1,121 @@
+name: Prod Monitor
+
+# Daily synthetic monitor for the LIVE site (https://campable.co).
+#
+# Distinct from playwright.yml, which is a release GATE: it deploys dev HEAD to
+# staging and tests that before promotion. This workflow tests PRODUCTION as it
+# actually is, so a Fly outage / TLS expiry / Supabase incident is caught even
+# when no real user is around. It performs ZERO mutations (no signup, Stripe,
+# or writes) — see e2e/tests/prod-smoke.spec.ts.
+#
+# Layering (see docs/MONITORING.md):
+#   UptimeRobot  -> "is it down?"        every 5 min  (external, /healthz)
+#   THIS         -> "do features work?"  daily        (deep, read-only)
+#   playwright   -> "is the next release safe?"  pre-deploy (staging)
+#   PostHog      -> passive real-user error capture
+#
+# NOTE: the /healthz JSON check only goes green once v1.47 (which adds the real
+# /healthz route) is released to prod. Until then that step will fail on manual
+# dispatch; the Playwright smoke + /api/search check work against current prod.
+
+on:
+  schedule:
+    - cron: "0 15 * * *" # daily 15:00 UTC (offset from the 08:00 release gate)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+env:
+  PROD_URL: https://campable.co
+
+jobs:
+  monitor:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Healthz — app + DB readiness
+        run: |
+          body=$(curl -sf --max-time 30 "$PROD_URL/healthz") || {
+            echo "::error::/healthz did not return 2xx"; exit 1; }
+          echo "$body"
+          echo "$body" | grep -q '"status":"ok"' || {
+            echo "::error::/healthz not ok (DB degraded or route not yet deployed)"; exit 1; }
+
+      - name: Search API — DB + provider pipeline
+        run: |
+          start=$(date -u -d "+30 days" +%Y-%m-%d)
+          end=$(date -u -d "+33 days" +%Y-%m-%d)
+          url="$PROD_URL/api/search?start_date=$start&end_date=$end&state=WA&nights=2&limit=5"
+          body=$(curl -sf --max-time 60 "$url") || {
+            echo "::error::/api/search did not return 2xx"; exit 1; }
+          echo "$body" | grep -q '"campgrounds_checked"' || {
+            echo "::error::/api/search response missing campgrounds_checked"; exit 1; }
+
+      - name: Install Playwright + browser
+        working-directory: e2e
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Run read-only prod smoke
+        working-directory: e2e
+        env:
+          E2E_BASE_URL: ${{ env.PROD_URL }}
+          CI: "true"
+        run: npx playwright test prod-smoke.spec.ts
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: prod-monitor-report
+          path: e2e/playwright-report/
+          retention-days: 30
+
+      - name: Email alert on failure (Resend)
+        if: failure()
+        env:
+          RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
+          ALERT_EMAIL_FROM: ${{ vars.ALERT_EMAIL_FROM }}
+          ALERT_EMAIL_TO: ${{ vars.ALERT_EMAIL_TO }}
+        run: |
+          if [ -z "$RESEND_API_KEY" ] || [ -z "$ALERT_EMAIL_TO" ]; then
+            echo "::warning::Resend not configured (RESEND_API_KEY / ALERT_EMAIL_TO); skipping email"
+            exit 0
+          fi
+          run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          from="${ALERT_EMAIL_FROM:-Campable Monitor <alerts@campable.co>}"
+          date_str=$(date -u +%Y-%m-%d)
+          text=$(printf 'The daily production monitor failed. A feature on the live site may be down.\n\nRun: %s\n\nThe Playwright report + trace are attached to the run as an artifact.' "$run_url")
+          payload=$(jq -n \
+            --arg from "$from" \
+            --arg to "$ALERT_EMAIL_TO" \
+            --arg subject "🔴 campable.co prod monitor FAILED ($date_str)" \
+            --arg text "$text" \
+            '{from: $from, to: [$to], subject: $subject, text: $text}')
+          curl -sf -X POST https://api.resend.com/emails \
+            -H "Authorization: Bearer $RESEND_API_KEY" \
+            -H "Content-Type: application/json" \
+            -d "$payload"
+
+      - name: Open issue on failure
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const date = new Date().toISOString().split('T')[0];
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `Prod monitor FAILED — campable.co (${date})`,
+              body: `The daily production synthetic monitor failed — a live feature may be down.\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}\n\nThe Playwright HTML report + trace are attached as the run's artifact.`,
+              labels: ['prod-down'],
+            });

docs/MONITORING.md

@@ -0,0 +1,96 @@
+# Monitoring (v1.47)
+
+How we know campable.co is up and its features work. Four layers, each
+answering a different question — none replaces the others.
+
+| Layer | Question | Cadence | Where |
+|-------|----------|---------|-------|
+| **UptimeRobot** | Is it down *right now*? | every 5 min | external service (you set up) |
+| **Prod monitor** | Do the *features* work on prod? | daily | `.github/workflows/prod-monitor.yml` |
+| **Release gate** | Is the *next release* safe? | nightly / per-PR | `.github/workflows/playwright.yml` (staging) |
+| **PostHog** | What broke for a *real user*? | passive | PostHog error tracking |
+
+**Why not just PostHog?** PostHog is *passive* — it only captures an error once
+a real user trips it. If the site dies at 3am, PostHog stays quiet until someone
+shows up and gets hurt. The first two layers are *active / synthetic*: a robot
+drives the live site on a schedule so we hear about breakage first.
+
+**Why not just the existing nightly?** `playwright.yml` is a release *gate* — it
+deploys `dev` HEAD to **staging** and tests that before promotion. It never
+touches production, so a Fly outage, expired TLS cert, or Supabase incident on
+the *live* site is invisible to it.
+
+---
+
+## Layer 1 — UptimeRobot (you set up, ~5 min)
+
+Fast "is it down" detection on the real `/healthz` endpoint (added in v1.47 —
+pings SQLite, returns `{"status":"ok","db":true,"version":"..."}`, or **503**
+when the DB is unreachable).
+
+1. Create a free account at <https://uptimerobot.com>.
+2. **Add New Monitor**:
+   - Type: **HTTP(s)**
+   - URL: `https://campable.co/healthz`
+   - Interval: **5 minutes**
+   - **Keyword monitoring**: alert if the response does **not** contain
+     `"status":"ok"`. This way a 503 (DB degraded) *or* a missing keyword both
+     trigger — not just a hard connection failure.
+3. **Alert contact**: add your Outlook address (`…@palouselabs.com`). UptimeRobot
+   emails on down + recovery.
+
+> Note: `/healthz` ships with v1.47. Until that release reaches prod, point the
+> monitor at `https://campable.co/` (plain reachability) and switch to
+> `/healthz` once deployed.
+
+---
+
+## Layer 2 — Daily prod monitor (built in, `.github/workflows/prod-monitor.yml`)
+
+Runs daily at **15:00 UTC** (and on demand via *Actions → Prod Monitor → Run
+workflow*). It checks, against live `https://campable.co`:
+
+1. `/healthz` returns `"status":"ok"` (app + DB).
+2. `/api/search` returns a well-shaped response (DB + provider pipeline).
+3. `e2e/tests/prod-smoke.spec.ts` — a **read-only, anonymous** Playwright smoke:
+   homepage renders, search API works, pricing page renders, SEO state index
+   renders.
+
+It is **strictly non-mutating** — no signup, no Stripe, no writes — so it is
+safe to point at production. (The four flows in `e2e/tests/` *do* mutate state
+and run only against staging.)
+
+On failure it: uploads the Playwright report, **emails you via Resend**, and
+**opens a GitHub issue** labelled `prod-down`.
+
+### Required repo configuration
+
+Set under *Settings → Secrets and variables → Actions*:
+
+| Kind | Name | Value |
+|------|------|-------|
+| **Secret** | `RESEND_API_KEY` | your Resend API key |
+| Variable | `ALERT_EMAIL_TO` | your Outlook address, e.g. `you@palouselabs.com` |
+| Variable | `ALERT_EMAIL_FROM` | *(optional)* sender, default `Campable Monitor <alerts@campable.co>` — the domain must be verified in Resend |
+
+If `RESEND_API_KEY` / `ALERT_EMAIL_TO` are unset, the email step is skipped with
+a warning and the GitHub issue still fires (so you're never silently blind).
+
+> The `ALERT_EMAIL_FROM` domain must be verified in Resend. If `campable.co`
+> isn't verified there yet, either verify it or set `ALERT_EMAIL_FROM` to a
+> domain that is.
+
+---
+
+## Layer 3 — Release gate (`.github/workflows/playwright.yml`)
+
+Unchanged by v1.47. Per-PR smoke + nightly full E2E against **staging**
+(`campnw-staging`), opening an issue on nightly failure. This is the
+pre-promotion safety net, not a production monitor.
+
+---
+
+## Layer 4 — PostHog
+
+Passive real-user product analytics + error tracking. Keeps the "what did a real
+user experience" record that synthetic checks can't reproduce.

docs/ROADMAP.md

@@ -43,6 +43,7 @@ v1.4    [SHIPPED]  Monetization Launch  — Pro tier gate, Stripe Checkout/Porta
 v1.41   [SHIPPED]  Playwright E2E       — Playwright E2E suite — smoke + watch/planner limits + cancel — 4/4 nightly green (2026-05-31)
 v1.42   ------->   Site Polish + Legal  — About, Privacy, Terms, footer. Unblocks Stripe live-mode review + Apple App Store URL requirement.
 v1.45   ------->   Native Apps          — Capacitor shell, iOS App Store + Google Play, native push/GPS/offline registry
+v1.47   ------->   Prod Monitoring      — Real /healthz (DB probe) + daily read-only prod smoke + UptimeRobot. Catches live-site outages the staging gate can't. See docs/MONITORING.md
 v2.0    ------->   Predictions+        — Statistical model, anomaly alerts, post-mortems (~Q1 2027)
 ```
 

e2e/tests/prod-smoke.spec.ts

@@ -0,0 +1,81 @@
+import { test, expect } from "@playwright/test";
+
+/**
+ * Production synthetic monitor — READ-ONLY smoke.
+ *
+ * Runs against the LIVE site (E2E_BASE_URL=https://campable.co) on a daily
+ * schedule via .github/workflows/prod-monitor.yml. Unlike the four flows in
+ * this directory, it performs ZERO mutations: no signup, no Stripe, no
+ * POST/DELETE. It only loads anonymous pages and reads the public search API,
+ * so it is safe to point at production.
+ *
+ * Purpose: catch feature regressions on prod (broken homepage, dead search
+ * pipeline, missing pricing/SEO pages) that the staging release-gate nightly
+ * cannot see, and that PostHog only surfaces after a real user is harmed.
+ */
+
+// Compute future dates at runtime so the search query never goes stale.
+function isoDate(daysFromNow: number): string {
+  const d = new Date();
+  d.setUTCDate(d.getUTCDate() + daysFromNow);
+  return d.toISOString().slice(0, 10);
+}
+
+test("homepage renders core discovery UI", async ({ page }) => {
+  await page.goto("/");
+
+  // The discovery-mode tab switcher (Find a Site / Plan a Trip).
+  await expect(
+    page.getByRole("tablist", { name: "Discovery mode" })
+  ).toBeVisible();
+
+  // The structured search form — the app's primary surface.
+  await expect(page.locator(".search-form")).toBeVisible();
+
+  // Anonymous header CTA confirms auth wiring rendered.
+  await expect(
+    page.getByRole("button", { name: "Sign in" }).first()
+  ).toBeVisible();
+});
+
+test("search API returns a valid response (DB + provider pipeline)", async ({
+  request,
+}) => {
+  // The differentiator: a real availability search across the registry.
+  // We assert the endpoint is healthy and well-shaped — NOT that any site is
+  // available (external booking systems may legitimately have zero), to avoid
+  // false alarms from upstream providers.
+  const res = await request.get("/api/search", {
+    params: {
+      start_date: isoDate(30),
+      end_date: isoDate(33),
+      state: "WA",
+      nights: "2",
+      limit: "5",
+    },
+    timeout: 60_000,
+  });
+
+  expect(res.ok()).toBeTruthy();
+  const body = await res.json();
+  expect(typeof body.campgrounds_checked).toBe("number");
+  expect(body.campgrounds_checked).toBeGreaterThan(0);
+});
+
+test("pricing page renders", async ({ page }) => {
+  await page.goto("/pricing");
+  await expect(
+    page.getByRole("heading", { name: /pricing/i, level: 1 })
+  ).toBeVisible();
+  // Both plan cards render. We assert structure, not the CTA: the CTA is
+  // auth-dependent (anon sees "Sign in to upgrade", free sees "Upgrade to
+  // Pro") and we stay logged-out / never click it (would hit Stripe).
+  await expect(page.locator(".pricing-card-free")).toBeVisible();
+  await expect(page.locator(".pricing-card-pro")).toBeVisible();
+});
+
+test("SEO state index renders", async ({ page }) => {
+  const res = await page.goto("/campgrounds/wa");
+  expect(res?.status()).toBe(200);
+  await expect(page.getByRole("heading", { level: 1 })).toBeVisible();
+});

src/pnw_campsites/api.py

@@ -5,6 +5,7 @@
 import asyncio
 import logging
 import os
+import sqlite3
 import time
 from collections import deque
 from contextlib import asynccontextmanager
@@ -18,14 +19,17 @@
 from fastapi.responses import FileResponse, JSONResponse, Response
 from fastapi.staticfiles import StaticFiles
 
-from pnw_campsites.monitor.db import WatchDB
+from pnw_campsites.monitor.db import DEFAULT_DB_PATH, WatchDB
 from pnw_campsites.posthog_client import get_posthog_client
 from pnw_campsites.providers.goingtocamp import GoingToCampClient
 from pnw_campsites.providers.recgov import RecGovClient
 from pnw_campsites.providers.reserveamerica import ReserveAmericaClient
 from pnw_campsites.registry.db import CampgroundRegistry
 from pnw_campsites.search.engine import SearchEngine
 
+# App version — surfaced by /healthz so monitors can confirm what's deployed.
+APP_VERSION = "1.47"
+
 # ---------------------------------------------------------------------------
 # App state — initialized in lifespan
 # ---------------------------------------------------------------------------
@@ -539,6 +543,36 @@ async def timing_middleware(request: Request, call_next):
 # takes precedence over /{path:path}
 app.include_router(seo_router)
 
+
+@app.get("/healthz")
+async def healthz() -> JSONResponse:
+    """Liveness + DB readiness probe for synthetic monitoring.
+
+    Returns 200 when SQLite is reachable, 503 otherwise. Defined as an
+    explicit route so it takes precedence over the SPA catch-all at
+    /{path:path} — without it, unknown paths fall through to index.html
+    and any /healthz check would be falsely green during a DB outage.
+    """
+    db_ok = False
+    try:
+        conn = sqlite3.connect(str(DEFAULT_DB_PATH))
+        try:
+            conn.execute("SELECT 1")
+        finally:
+            conn.close()
+        db_ok = True
+    except Exception:
+        logging.getLogger(__name__).exception("healthz: SQLite check failed")
+
+    return JSONResponse(
+        {
+            "status": "ok" if db_ok else "degraded",
+            "db": db_ok,
+            "version": APP_VERSION,
+        },
+        status_code=200 if db_ok else 503,
+    )
+
 # Mount SEO static assets (tokens.css, seo.css)
 _seo_static_candidates = [
     Path("/app/seo-static"),