diff --git a/.github/workflows/deploy-prd.yml b/.github/workflows/deploy-prd.yml index f155d7486..2ba5721b2 100644 --- a/.github/workflows/deploy-prd.yml +++ b/.github/workflows/deploy-prd.yml @@ -70,8 +70,10 @@ jobs: KLIPY_API_KEY: ${{ secrets.KLIPY_API_KEY }} # electric-proxy + self-hosted Electric (DATABASE_URL comes from the stack's replication role) ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET }} - # Maple: an org-admin API key with `ingest_keys:read`; the deploy binds the ingest key onto the Workers + # Maple: an org-admin API key with `ingest_keys:read`, `dashboards:write`, `alerts:write`; the deploy binds the ingest key onto the Workers MAPLE_API_KEY: ${{ secrets.MAPLE_API_KEY }} + # A Hazel channel's Maple webhook URL: production alerts post there (`@hazel/infra/maple-monitoring`) + MAPLE_ALERTS_WEBHOOK_URL: ${{ secrets.MAPLE_ALERTS_WEBHOOK_URL }} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/alchemy.run.ts b/alchemy.run.ts index 19cbbf1a6..e7ce482fa 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -20,6 +20,7 @@ import { } from "@hazel/infra/cloudflare" import { plainWithDefault } from "@hazel/infra/env" import { isMapleDeploy } from "@hazel/infra/maple" +import { HazelMonitoring } from "@hazel/infra/maple-monitoring" import Actors from "./apps/actors/alchemy.run.ts" import ApiLive, { Api } from "./apps/backend/src/worker.ts" import BotGateway from "./apps/bot-gateway/alchemy.run.ts" @@ -107,6 +108,8 @@ export default Alchemy.Stack( // Shared marketing site: prd only (previews and dev run their own dev servers). docs.hazel.sh // stays on Vercel until its DNS record is moved, so the docs site is not deployed here yet. const landing = stage.kind === "prd" ? yield* Landing : undefined + // Maple alerts and dashboard watch production only: see `@hazel/infra/maple-monitoring`. + const monitoring = stage.kind === "prd" && isMapleDeploy() ? yield* HazelMonitoring : undefined const summary = { stage: formatHazelStage(stage), @@ -127,6 +130,8 @@ export default Alchemy.Stack( linkPreviewWorker: linkPreview.workerName, actorsWorker: actors.workerName, landingWorker: landing?.workerName, + mapleDashboard: monitoring?.dashboardId, + mapleAlertRules: monitoring?.alertRules, } }).pipe( // The stack IS the entry point: the one place `HazelStack` is provided. diff --git a/infra/README.md b/infra/README.md index 32d35abc9..e468618bb 100644 --- a/infra/README.md +++ b/infra/README.md @@ -46,6 +46,12 @@ Alchemy reads its own credentials from the environment (or an `alchemy profile`) Effect Workers (api, electric-proxy, bot-gateway), which export traces, logs and metrics through the Maple SDK (`packages/infra/src/cloudflare/maple.ts`). Without it, and under `alchemy dev`, the SDK is a no-op. + On prd it also declares Hazel's monitoring in Maple (`packages/infra/src/cloudflare/maple-monitoring.ts`): + a "Hazel services" dashboard, plus error-rate and p95 alert rules per Worker (needs a key that + is also allowed `dashboards:write` and `alerts:write`). +- `MAPLE_ALERTS_WEBHOOK_URL` (optional, prd): the alert rules' destination, a Hazel channel's + Maple webhook URL (Integrations → Maple in Hazel, `…/webhooks/incoming///maple`). + Without it only the dashboard is declared. Locally, `bunx alchemy profile refresh --profile default --provider Cloudflare` re-authenticates the default profile. @@ -85,4 +91,4 @@ Worker env (secrets are uploaded as Worker secrets, plain values as vars) is lis - Moved to Cloudflare: `api.hazel.sh` (`hazel-api`), `electric.hazel.sh` (`hazel-electric-proxy`), `bot-gateway.hazel.sh`. - Electric itself runs on Railway (service `electric`, volume at `/app/persistent`, PlanetScale role `electric-railway` with replication); `ELECTRIC_URL` = `electric-production-0d89.up.railway.app`. Electric runs with `ELECTRIC_MANUAL_TABLE_PUBLISHING=true` and `ELECTRIC_DB_POOL_SIZE=4` (PS-5 allows 50 connections): the synced tables plus `channel_access` were added to `electric_publication_default` by hand (owned by `postgres`; Electric's role cannot alter it), so a newly synced table must be added there too. The Cloudflare Container was removed: always-on it cost several times more and lost its disk on every restart. `app.hazel.sh` serves `apps/web-foldkit`. - Rollback DNS (all unproxied CNAMEs, detach the Worker custom domain first): `api` → `j0tqzlof.up.railway.app`, `electric` → `2hg74iuk.up.railway.app`, `bot-gateway` → `s3t62x1p.up.railway.app` (was proxied). -- Not done yet: Discord gateway flip (Worker `false`, Railway still runs it), `MAPLE_API_KEY` for Worker telemetry to Maple, docs.hazel.sh stays on Vercel, GitHub `production` environment secrets for CI deploys. Link previews return `INVALID_URL` for every URL (already broken on the June build). +- Not done yet: Discord gateway flip (Worker `false`, Railway still runs it), `MAPLE_API_KEY` for Worker telemetry to Maple and `MAPLE_ALERTS_WEBHOOK_URL` for its alerts, docs.hazel.sh stays on Vercel, GitHub `production` environment secrets for CI deploys. Link previews return `INVALID_URL` for every URL (already broken on the June build). diff --git a/packages/infra/package.json b/packages/infra/package.json index 53f0aad4f..bc3784314 100644 --- a/packages/infra/package.json +++ b/packages/infra/package.json @@ -9,6 +9,7 @@ "./cloudflare": "./src/cloudflare/index.ts", "./env": "./src/env.ts", "./maple": "./src/cloudflare/maple.ts", + "./maple-monitoring": "./src/cloudflare/maple-monitoring.ts", "./worker-http": "./src/cloudflare/worker-http.ts", "./worker-runtime": "./src/cloudflare/worker-runtime.ts", "./cached-recoverable": "./src/cloudflare/cached-recoverable.ts" diff --git a/packages/infra/src/cloudflare/maple-monitoring.ts b/packages/infra/src/cloudflare/maple-monitoring.ts new file mode 100644 index 000000000..d25826f48 --- /dev/null +++ b/packages/infra/src/cloudflare/maple-monitoring.ts @@ -0,0 +1,172 @@ +/** + * Hazel's production monitoring in Maple, declared next to the Workers it watches: per-service + * alert rules and a services dashboard over the telemetry `hazelTelemetry` exports (`./maple.ts`). + * + * Alerts post into a Hazel channel through Hazel's own Maple integration: create the channel's + * Maple webhook in Hazel (Integrations → Maple) and set its URL as `MAPLE_ALERTS_WEBHOOK_URL`. + * Without it the rules are not declared, since a rule with no destination notifies nobody. + * + * Rules are `builder_query`s so they can scope to `deployment.environment = "production"`: preview + * deploys report under the same service names. + */ +import * as Maple from "@maple-dev/alchemy" +import type * as Output from "alchemy/Output" +import { Effect, Option } from "effect" +import { optionalString } from "../config-helpers.ts" + +/** The Workers that export through the Maple SDK (`hazelTelemetry`), by `service.name`. */ +export const HAZEL_TELEMETRY_SERVICES = ["api", "electric-proxy", "bot-gateway"] as const + +const PRODUCTION = `deployment.environment = "production"` + +type TraceAggregation = "count" | "error_rate" | "p95_duration" | "p99_duration" + +/** A traces query-builder draft, the shape alert rules and dashboard widgets share. */ +const tracesQuery = (options: { + readonly where: string + readonly aggregation: TraceAggregation + readonly groupBy?: ReadonlyArray +}) => ({ + id: "q1", + name: "A", + enabled: true, + hidden: false, + whereClause: options.where, + aggregation: options.aggregation, + stepInterval: "", + orderByDirection: "desc", + addOns: { + groupBy: options.groupBy !== undefined, + having: false, + orderBy: false, + limit: false, + legend: false, + }, + groupBy: options.groupBy ?? [], + having: "", + orderBy: "", + limit: "", + legend: "", + dataSource: "traces", +}) + +/** Root spans of one service in production: request-level signals, not internal calls. */ +const serviceRequests = (service: string) => + `service.name = "${service}" AND ${PRODUCTION} AND root_only = "true"` + +const serviceAlerts = (service: string, destinationId: Output.Output) => + Effect.all([ + Maple.AlertRule(`${service}-error-rate`, { + name: `Hazel ${service}: error rate above 5%`, + severity: "critical", + signal_type: "builder_query", + query_builder_draft: tracesQuery({ where: serviceRequests(service), aggregation: "error_rate" }), + comparator: "gt", + threshold: 0.05, + window_minutes: 5, + minimum_sample_count: 20, + consecutive_breaches_required: 2, + destination_ids: [destinationId], + tags: ["hazel", service], + }), + Maple.AlertRule(`${service}-p95-latency`, { + name: `Hazel ${service}: p95 latency above 2s`, + severity: "warning", + signal_type: "builder_query", + query_builder_draft: tracesQuery({ + where: serviceRequests(service), + aggregation: "p95_duration", + }), + comparator: "gt", + threshold: 2000, + window_minutes: 10, + minimum_sample_count: 20, + consecutive_breaches_required: 2, + destination_ids: [destinationId], + tags: ["hazel", service], + }), + ]) + +const chart = (options: { + readonly id: string + readonly title: string + readonly aggregation: TraceAggregation + readonly unit: string + readonly x: number + readonly y: number +}) => ({ + id: options.id, + visualization: "chart", + dataSource: { + kind: "query", + resultShape: "timeseries", + queries: [ + tracesQuery({ + where: `${PRODUCTION} AND root_only = "true"`, + aggregation: options.aggregation, + groupBy: ["service.name"], + }), + ], + }, + display: { + title: options.title, + chartId: "query-builder-line", + unit: options.unit, + chartPresentation: { legend: "visible" }, + }, + layout: { x: options.x, y: options.y, w: 6, h: 6, minW: 2, minH: 2 }, +}) + +/** Golden signals of every Hazel service in production, one line per `service.name`. */ +const servicesDashboard = Maple.Dashboard("services", { + name: "Hazel services", + description: + "Production golden signals of the Hazel Workers (api, electric-proxy, bot-gateway) over root spans. Managed by alchemy.run.ts: edits in the UI are overwritten on deploy.", + tags: ["hazel"], + time_range: { type: "relative", value: "12h" }, + widgets: [ + chart({ id: "throughput", title: "Requests", aggregation: "count", unit: "number", x: 0, y: 0 }), + chart({ + id: "error-rate", + title: "Error rate", + aggregation: "error_rate", + unit: "percent", + x: 6, + y: 0, + }), + chart({ + id: "p95", + title: "p95 latency", + aggregation: "p95_duration", + unit: "duration_ms", + x: 0, + y: 6, + }), + chart({ + id: "p99", + title: "p99 latency", + aggregation: "p99_duration", + unit: "duration_ms", + x: 6, + y: 6, + }), + ], +}) + +/** Declare Hazel's Maple dashboard and, when a Hazel channel webhook is configured, its alerts. */ +export const HazelMonitoring = Effect.gen(function* () { + const dashboard = yield* servicesDashboard + const webhookUrl = yield* optionalString("MAPLE_ALERTS_WEBHOOK_URL") + if (Option.isNone(webhookUrl)) return { dashboardId: dashboard.dashboardId, alertRules: 0 } + + // Hazel's incoming Maple webhook (`/webhooks/incoming/:id/:token/maple`) renders the alert payload. + const destination = yield* Maple.AlertDestination("hazel-alerts", { + type: "webhook", + name: "Hazel #alerts", + url: webhookUrl.value.trim(), + }) + const rules = yield* Effect.forEach(HAZEL_TELEMETRY_SERVICES, (service) => + serviceAlerts(service, destination.destinationId), + ) + return { dashboardId: dashboard.dashboardId, alertRules: rules.flat().length } +})