feat(api): zero-downtime CHECKIN_ENCRYPTION_KEY rotation

apps/api/.env.example

@@ -1,6 +1,8 @@
 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/telemetry
 # Check-in monitoring: 32 random bytes as hex; retain securely with DB backups.
 # CHECKIN_ENCRYPTION_KEY=
+# Optional, during key rotation only (decrypt-only, comma-separated old keys):
+# CHECKIN_ENCRYPTION_KEY_PREVIOUS=
 
 PORT=3001
 HOST=0.0.0.0

apps/api/package.json

@@ -14,6 +14,7 @@
     "alert-rules-evaluator": "tsx src/jobs/run-alert-rules-evaluator.ts",
     "builtin-alert-rules-backfill": "tsx src/jobs/run-builtin-alert-rules-backfill.ts",
     "pii-scrub-backfill": "tsx src/jobs/run-pii-scrub-backfill.ts",
+    "checkin-reencrypt": "tsx src/jobs/run-checkin-reencrypt.ts",
     "brief-worker": "tsx src/jobs/run-brief-worker.ts",
     "alert-webhook-worker": "tsx src/jobs/run-alert-webhook-worker.ts",
     "seed:dev-api-key": "tsx scripts/seed-dev-api-key.ts"

apps/api/src/jobs/checkin-reencrypt.ts

@@ -0,0 +1,66 @@
+/**
+ * Re-encrypt CheckInMonitor.token_encrypted with the current CHECKIN_ENCRYPTION_KEY.
+ * Idempotent: rows already on the current key are skipped. Logs counts only.
+ */
+import type { PrismaClient } from "@prisma/client";
+import { checkInTokenHash, decryptCheckInSecret, encryptCheckInToken, isEncryptedWithCurrentKey } from "../lib/checkin-secrets.js";
+
+export type CheckInReencryptResult = {
+  dryRun: boolean; scanned: number; alreadyCurrent: number; reencrypted: number; failed: number;
+};
+
+type Db = Pick<PrismaClient, "checkInMonitor">;
+
+export function parseCheckInReencryptArgs(argv: string[]): { apply: boolean; batchSize: number; help: boolean; error?: string } {
+  let apply = false, batchSize = 200, help = false;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--apply") apply = true;
+    else if (a === "--dry-run") apply = false;
+    else if (a === "--help" || a === "-h") help = true;
+    else if (a === "--batch-size") {
+      const n = Number(argv[++i]);
+      if (!Number.isInteger(n) || n < 1 || n > 2000) return { apply, batchSize, help, error: "--batch-size must be 1..2000" };
+      batchSize = n;
+    } else if (a !== "--") return { apply, batchSize, help, error: `Unknown argument: ${a}` };
+  }
+  return { apply, batchSize, help };
+}
+
+export async function runCheckInReencrypt(db: Db, opts: { apply?: boolean; batchSize?: number } = {}): Promise<CheckInReencryptResult> {
+  const apply = opts.apply === true;
+  const batchSize = opts.batchSize ?? 200;
+  const result: CheckInReencryptResult = { dryRun: !apply, scanned: 0, alreadyCurrent: 0, reencrypted: 0, failed: 0 };
+  let cursor: string | undefined;
+  for (;;) {
+    const rows = await db.checkInMonitor.findMany({
+      select: { id: true, token_hash: true, token_encrypted: true },
+      orderBy: { id: "asc" }, take: batchSize,
+      ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
+    });
+    if (rows.length === 0) break;
+    for (const row of rows) {
+      result.scanned++;
+      if (isEncryptedWithCurrentKey(row.token_encrypted)) {
+        try { decryptCheckInSecret(row.token_encrypted); result.alreadyCurrent++; } catch { result.failed++; }
+        continue;
+      }
+      let token: string;
+      try { token = decryptCheckInSecret(row.token_encrypted); } catch { result.failed++; continue; }
+      if (checkInTokenHash(token) !== row.token_hash) { result.failed++; continue; }
+      if (apply) {
+        // Compare-and-set so a concurrent token rotation is never overwritten.
+        const updated = await db.checkInMonitor.updateMany({
+          where: { id: row.id, token_encrypted: row.token_encrypted },
+          data: { token_encrypted: encryptCheckInToken(token) },
+        });
+        if (updated.count === 1) result.reencrypted++;
+      } else {
+        result.reencrypted++;
+      }
+    }
+    cursor = rows[rows.length - 1].id;
+    if (rows.length < batchSize) break;
+  }
+  return result;
+}

apps/api/src/jobs/run-checkin-reencrypt.ts

@@ -0,0 +1,30 @@
+/**
+ * CLI: re-encrypt check-in tokens with the current CHECKIN_ENCRYPTION_KEY.
+ *
+ *   node dist/jobs/run-checkin-reencrypt.js            # dry run (default)
+ *   node dist/jobs/run-checkin-reencrypt.js --apply    # write
+ *
+ * Requires DATABASE_URL, CHECKIN_ENCRYPTION_KEY and (during rotation) CHECKIN_ENCRYPTION_KEY_PREVIOUS.
+ * Prints counts only; never prints tokens or keys.
+ */
+import { prisma } from "../lib/db.js";
+import { parseCheckInReencryptArgs, runCheckInReencrypt } from "./checkin-reencrypt.js";
+
+const parsed = parseCheckInReencryptArgs(process.argv.slice(2));
+if (parsed.help) {
+  console.log("Usage: node dist/jobs/run-checkin-reencrypt.js [--apply] [--batch-size <n>]\nDry run by default. Idempotent. See docs/CHECK-INS.md (key rotation).");
+  process.exit(0);
+}
+if (parsed.error) { console.error(parsed.error); process.exit(1); }
+
+runCheckInReencrypt(prisma, { apply: parsed.apply, batchSize: parsed.batchSize })
+  .then(async (result) => {
+    console.log(JSON.stringify({ ok: result.failed === 0, ...result, at: new Date().toISOString() }));
+    await prisma.$disconnect();
+    if (result.failed > 0) process.exit(1);
+  })
+  .catch(async (e: unknown) => {
+    console.error(`[checkin-reencrypt] failed: ${e instanceof Error ? e.message : "unknown error"}`);
+    await prisma.$disconnect().catch(() => {});
+    process.exit(1);
+  });

apps/api/src/lib/checkin-key-rotation.test.ts

@@ -0,0 +1,99 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createCipheriv, randomBytes } from "node:crypto";
+import { checkInTokenHash, createCheckInSecret, decryptCheckInSecret, isEncryptedWithCurrentKey } from "./checkin-secrets.js";
+import { parseCheckInReencryptArgs, runCheckInReencrypt } from "../jobs/checkin-reencrypt.js";
+
+const OLD = "ab".repeat(32), NEW = "cd".repeat(32), OTHER = "ef".repeat(32);
+afterEach(() => vi.unstubAllEnvs());
+
+/** Legacy (pre-rotation) format: bare base64(iv|tag|ct). */
+function legacyEncrypt(keyHex: string, token: string): string {
+  const iv = randomBytes(12);
+  const c = createCipheriv("aes-256-gcm", Buffer.from(keyHex, "hex"), iv);
+  const ct = Buffer.concat([c.update(token, "utf8"), c.final()]);
+  return Buffer.concat([iv, c.getAuthTag(), ct]).toString("base64");
+}
+function setKeys(current: string, previous?: string) {
+  vi.stubEnv("CHECKIN_ENCRYPTION_KEY", current);
+  vi.stubEnv("CHECKIN_ENCRYPTION_KEY_PREVIOUS", previous ?? "");
+}
+function fakeDb(rows: { id: string; token_hash: string; token_encrypted: string }[]) {
+  return {
+    checkInMonitor: {
+      findMany: vi.fn(async (args: { take: number; cursor?: { id: string } }) => {
+        const sorted = [...rows].sort((a, b) => a.id.localeCompare(b.id));
+        const start = args.cursor ? sorted.findIndex((r) => r.id === args.cursor!.id) + 1 : 0;
+        return sorted.slice(start, start + args.take).map((r) => ({ ...r }));
+      }),
+      updateMany: vi.fn(async (args: { where: { id: string; token_encrypted: string }; data: { token_encrypted: string } }) => {
+        const r = rows.find((x) => x.id === args.where.id && x.token_encrypted === args.where.token_encrypted);
+        if (!r) return { count: 0 };
+        r.token_encrypted = args.data.token_encrypted; return { count: 1 };
+      }),
+    },
+  };
+}
+
+describe("check-in encryption key rotation", () => {
+  it("decrypts legacy and versioned old-key data after rotation", () => {
+    setKeys(OLD);
+    const versioned = createCheckInSecret().token_encrypted;
+    const legacy = legacyEncrypt(OLD, "ttc_legacy");
+    setKeys(NEW, `${OTHER}, ${OLD}`);
+    expect(decryptCheckInSecret(legacy)).toBe("ttc_legacy");
+    expect(decryptCheckInSecret(versioned)).toMatch(/^ttc_/);
+    expect(isEncryptedWithCurrentKey(versioned)).toBe(false);
+  });
+  it("encrypts new writes with the current key only", () => {
+    setKeys(NEW, OLD);
+    const enc = createCheckInSecret().token_encrypted;
+    expect(enc.startsWith("v1.")).toBe(true);
+    expect(enc).not.toContain(NEW);
+    expect(isEncryptedWithCurrentKey(enc)).toBe(true);
+    setKeys(NEW);
+    expect(decryptCheckInSecret(enc)).toMatch(/^ttc_/);
+    setKeys(OLD);
+    expect(() => decryptCheckInSecret(enc)).toThrow("could not be decrypted");
+  });
+  it("fails safely without the previous key and rejects malformed previous keys", () => {
+    const legacy = legacyEncrypt(OLD, "ttc_x");
+    setKeys(NEW);
+    expect(() => decryptCheckInSecret(legacy)).toThrow("could not be decrypted");
+    setKeys(NEW, "not-hex");
+    expect(() => decryptCheckInSecret(legacy)).toThrow("CHECKIN_ENCRYPTION_KEY_PREVIOUS");
+    setKeys("", OLD);
+    expect(() => createCheckInSecret()).toThrow("CHECKIN_ENCRYPTION_KEY");
+  });
+  it("re-encrypt is dry-run by default, migrates with --apply and is idempotent", async () => {
+    const rows = Array.from({ length: 5 }, (_, i) => {
+      const token = `ttc_${i}`;
+      return { id: `m${i}`, token_hash: checkInTokenHash(token), token_encrypted: legacyEncrypt(OLD, token) };
+    });
+    setKeys(NEW, OLD);
+    const db = fakeDb(rows);
+    const log = vi.spyOn(console, "log");
+    const dry = await runCheckInReencrypt(db as never, { batchSize: 2 });
+    expect(dry).toEqual({ dryRun: true, scanned: 5, alreadyCurrent: 0, reencrypted: 5, failed: 0 });
+    expect(db.checkInMonitor.updateMany).not.toHaveBeenCalled();
+    const applied = await runCheckInReencrypt(db as never, { apply: true, batchSize: 2 });
+    expect(applied).toMatchObject({ dryRun: false, scanned: 5, reencrypted: 5, failed: 0 });
+    const again = await runCheckInReencrypt(db as never, { apply: true, batchSize: 2 });
+    expect(again).toMatchObject({ scanned: 5, alreadyCurrent: 5, reencrypted: 0, failed: 0 });
+    setKeys(NEW);
+    rows.forEach((r, i) => expect(decryptCheckInSecret(r.token_encrypted)).toBe(`ttc_${i}`));
+    expect(log).not.toHaveBeenCalled();
+  });
+  it("re-encrypt counts undecryptable rows as failed and leaves them unchanged", async () => {
+    const row = { id: "m1", token_hash: checkInTokenHash("ttc_a"), token_encrypted: legacyEncrypt(OLD, "ttc_a") };
+    setKeys(NEW);
+    const before = row.token_encrypted;
+    const r = await runCheckInReencrypt(fakeDb([row]) as never, { apply: true });
+    expect(r).toMatchObject({ failed: 1, reencrypted: 0 });
+    expect(row.token_encrypted).toBe(before);
+  });
+  it("parses CLI args with dry-run default", () => {
+    expect(parseCheckInReencryptArgs([])).toMatchObject({ apply: false, batchSize: 200 });
+    expect(parseCheckInReencryptArgs(["--apply", "--batch-size", "50"])).toMatchObject({ apply: true, batchSize: 50 });
+    expect(parseCheckInReencryptArgs(["--nope"]).error).toBeTruthy();
+  });
+});

apps/api/src/lib/checkin-secrets.ts

@@ -3,27 +3,83 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:
 export function checkInTokenHash(token: string): string {
   return createHash("sha256").update(token).digest("hex");
 }
-function encryptionKey(): Buffer {
-  const raw = process.env.CHECKIN_ENCRYPTION_KEY ?? "";
-  if (!/^[a-f0-9]{64}$/i.test(raw)) throw new Error("CHECKIN_ENCRYPTION_KEY must be 32 bytes in hex");
-  return Buffer.from(raw, "hex");
+
+/**
+ * Encryption keys for check-in tokens.
+ * - CHECKIN_ENCRYPTION_KEY: current key (32 bytes hex), used for all new encryption.
+ * - CHECKIN_ENCRYPTION_KEY_PREVIOUS: optional comma-separated old keys, decryption only.
+ * Ciphertext format: `v1.<kid>.<base64(iv|tag|ct)>` where kid is a short SHA-256 fingerprint
+ * of the key (not the key). Legacy rows are bare base64 and are tried against every key;
+ * AES-256-GCM authentication rejects a wrong key.
+ */
+type CheckInKey = { id: string; key: Buffer };
+const KEY_RE = /^[a-f0-9]{64}$/i;
+const PREFIX = "v1.";
+
+function toKey(raw: string): CheckInKey {
+  const key = Buffer.from(raw, "hex");
+  return { id: createHash("sha256").update(key).digest("hex").slice(0, 16), key };
 }
-export function createCheckInSecret() {
-  const token = `ttc_${randomBytes(32).toString("hex")}`;
+function currentKey(): CheckInKey {
+  const raw = (process.env.CHECKIN_ENCRYPTION_KEY ?? "").trim();
+  if (!KEY_RE.test(raw)) throw new Error("CHECKIN_ENCRYPTION_KEY must be 32 bytes in hex");
+  return toKey(raw);
+}
+function previousKeys(): CheckInKey[] {
+  const raw = process.env.CHECKIN_ENCRYPTION_KEY_PREVIOUS ?? "";
+  const parts = raw.split(",").map((s) => s.trim()).filter(Boolean);
+  for (const p of parts) {
+    if (!KEY_RE.test(p)) throw new Error("CHECKIN_ENCRYPTION_KEY_PREVIOUS must be comma-separated 32-byte hex keys");
+  }
+  return parts.map(toKey);
+}
+/** Current key first, then previous keys (deduplicated). */
+function decryptionKeys(): CheckInKey[] {
+  const keys = [currentKey(), ...previousKeys()];
+  return keys.filter((k, i) => keys.findIndex((o) => o.id === k.id) === i);
+}
+
+function encryptWith(k: CheckInKey, plaintext: string): string {
   const iv = randomBytes(12);
-  const cipher = createCipheriv("aes-256-gcm", encryptionKey(), iv);
-  const ciphertext = Buffer.concat([cipher.update(token, "utf8"), cipher.final()]);
-  return {
-    token_hash: checkInTokenHash(token),
-    token_encrypted: Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString("base64"),
-  };
+  const cipher = createCipheriv("aes-256-gcm", k.key, iv);
+  const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
+  return `${PREFIX}${k.id}.${Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString("base64")}`;
 }
-export function decryptCheckInSecret(encrypted: string): string {
-  const bytes = Buffer.from(encrypted, "base64");
-  const decipher = createDecipheriv("aes-256-gcm", encryptionKey(), bytes.subarray(0, 12));
+function decryptWith(k: CheckInKey, payload: string): string {
+  const bytes = Buffer.from(payload, "base64");
+  const decipher = createDecipheriv("aes-256-gcm", k.key, bytes.subarray(0, 12));
   decipher.setAuthTag(bytes.subarray(12, 28));
   return Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString("utf8");
 }
+function parse(encrypted: string): { kid: string | null; payload: string } {
+  if (encrypted.startsWith(PREFIX)) {
+    const rest = encrypted.slice(PREFIX.length);
+    const dot = rest.indexOf(".");
+    if (dot > 0) return { kid: rest.slice(0, dot), payload: rest.slice(dot + 1) };
+  }
+  return { kid: null, payload: encrypted };
+}
+
+export function encryptCheckInToken(token: string): string {
+  return encryptWith(currentKey(), token);
+}
+export function createCheckInSecret() {
+  const token = `ttc_${randomBytes(32).toString("hex")}`;
+  return { token_hash: checkInTokenHash(token), token_encrypted: encryptCheckInToken(token) };
+}
+export function decryptCheckInSecret(encrypted: string): string {
+  const { kid, payload } = parse(encrypted);
+  const keys = decryptionKeys();
+  const ordered = kid ? [...keys.filter((k) => k.id === kid), ...keys.filter((k) => k.id !== kid)] : keys;
+  for (const k of ordered) {
+    try { return decryptWith(k, payload); } catch { /* wrong key or tampered: try next */ }
+  }
+  throw new Error("Check-in secret could not be decrypted with the configured keys");
+}
+/** True when the value is already encrypted with the current key (versioned format). */
+export function isEncryptedWithCurrentKey(encrypted: string): boolean {
+  return parse(encrypted).kid === currentKey().id;
+}
 export function redactCheckInSecrets(text: string): string {
   return text.replace(/ttc_[a-zA-Z0-9_%.-]+/g, "[redacted]")
     .replace(/(\/checkin\/)[^\s/?#"<>]+/gi, "$1[redacted]");

docs/CHECK-INS.md

@@ -165,6 +165,26 @@ viewable URL; SHA-256 of the 256-bit random token provides indexed authenticatio
 Losing/changing this encryption key makes existing URLs unreadable in the dashboard;
 rotate affected monitor URLs afterwards. Do not reuse an ingest key or a public key.
 
+### Rotating CHECKIN_ENCRYPTION_KEY without data loss
+
+Pings authenticate by SHA-256 hash, so they keep working throughout. The key only
+protects the copyable URL shown in the dashboard. New encryption always uses
+`CHECKIN_ENCRYPTION_KEY`; `CHECKIN_ENCRYPTION_KEY_PREVIOUS` (comma-separated, optional)
+is used only to decrypt older values. Never paste key values into tickets or logs.
+
+1. Deploy a release that contains key-rotation support (this change).
+2. Generate a new key (`openssl rand -hex 32`). On the API service **and on any job
+   service that shares API code**, set `CHECKIN_ENCRYPTION_KEY_PREVIOUS` to the old key and
+   `CHECKIN_ENCRYPTION_KEY` to the new key. Deploy.
+3. Dry run (default, counts only): `node dist/jobs/run-checkin-reencrypt.js`
+4. Apply: `node dist/jobs/run-checkin-reencrypt.js --apply` (batched, idempotent, safe to re-run).
+5. Verify: re-run the dry run; expect `reencrypted: 0`, `failed: 0`, `alreadyCurrent` equal to
+   `scanned`. Open a monitor in the dashboard and confirm its URL is shown.
+6. Remove `CHECKIN_ENCRYPTION_KEY_PREVIOUS` and deploy. Then retire the old key.
+
+If the script reports `failed > 0`, keep `CHECKIN_ENCRYPTION_KEY_PREVIOUS` set and investigate;
+those rows cannot be decrypted with any configured key (rotate those monitor URLs instead).
+
 Set dashboard **CHECKIN_PUBLIC_BASE_URL** to the publicly reachable API origin
 if `API_URL` is an internal address. Otherwise `API_URL` is used for copied URLs.