diff --git a/apps/api/.env.example b/apps/api/.env.example index bcc4d158..e593b93c 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -1,6 +1,8 @@ DATABASE_URL=postgresql://postgres:postgres@localhost:5432/telemetry # Check-in monitoring: 32 random bytes as hex; retain securely with DB backups. # CHECKIN_ENCRYPTION_KEY= +# Optional, during key rotation only (decrypt-only, comma-separated old keys): +# CHECKIN_ENCRYPTION_KEY_PREVIOUS= PORT=3001 HOST=0.0.0.0 diff --git a/apps/api/package.json b/apps/api/package.json index 069ab4f0..a795c922 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -14,6 +14,7 @@ "alert-rules-evaluator": "tsx src/jobs/run-alert-rules-evaluator.ts", "builtin-alert-rules-backfill": "tsx src/jobs/run-builtin-alert-rules-backfill.ts", "pii-scrub-backfill": "tsx src/jobs/run-pii-scrub-backfill.ts", + "checkin-reencrypt": "tsx src/jobs/run-checkin-reencrypt.ts", "brief-worker": "tsx src/jobs/run-brief-worker.ts", "alert-webhook-worker": "tsx src/jobs/run-alert-webhook-worker.ts", "seed:dev-api-key": "tsx scripts/seed-dev-api-key.ts" diff --git a/apps/api/src/jobs/checkin-reencrypt.ts b/apps/api/src/jobs/checkin-reencrypt.ts new file mode 100644 index 00000000..bd12b19d --- /dev/null +++ b/apps/api/src/jobs/checkin-reencrypt.ts @@ -0,0 +1,66 @@ +/** + * Re-encrypt CheckInMonitor.token_encrypted with the current CHECKIN_ENCRYPTION_KEY. + * Idempotent: rows already on the current key are skipped. Logs counts only. + */ +import type { PrismaClient } from "@prisma/client"; +import { checkInTokenHash, decryptCheckInSecret, encryptCheckInToken, isEncryptedWithCurrentKey } from "../lib/checkin-secrets.js"; + +export type CheckInReencryptResult = { + dryRun: boolean; scanned: number; alreadyCurrent: number; reencrypted: number; failed: number; +}; + +type Db = Pick; + +export function parseCheckInReencryptArgs(argv: string[]): { apply: boolean; batchSize: number; help: boolean; error?: string } { + let apply = false, batchSize = 200, help = false; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--apply") apply = true; + else if (a === "--dry-run") apply = false; + else if (a === "--help" || a === "-h") help = true; + else if (a === "--batch-size") { + const n = Number(argv[++i]); + if (!Number.isInteger(n) || n < 1 || n > 2000) return { apply, batchSize, help, error: "--batch-size must be 1..2000" }; + batchSize = n; + } else if (a !== "--") return { apply, batchSize, help, error: `Unknown argument: ${a}` }; + } + return { apply, batchSize, help }; +} + +export async function runCheckInReencrypt(db: Db, opts: { apply?: boolean; batchSize?: number } = {}): Promise { + const apply = opts.apply === true; + const batchSize = opts.batchSize ?? 200; + const result: CheckInReencryptResult = { dryRun: !apply, scanned: 0, alreadyCurrent: 0, reencrypted: 0, failed: 0 }; + let cursor: string | undefined; + for (;;) { + const rows = await db.checkInMonitor.findMany({ + select: { id: true, token_hash: true, token_encrypted: true }, + orderBy: { id: "asc" }, take: batchSize, + ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}), + }); + if (rows.length === 0) break; + for (const row of rows) { + result.scanned++; + if (isEncryptedWithCurrentKey(row.token_encrypted)) { + try { decryptCheckInSecret(row.token_encrypted); result.alreadyCurrent++; } catch { result.failed++; } + continue; + } + let token: string; + try { token = decryptCheckInSecret(row.token_encrypted); } catch { result.failed++; continue; } + if (checkInTokenHash(token) !== row.token_hash) { result.failed++; continue; } + if (apply) { + // Compare-and-set so a concurrent token rotation is never overwritten. + const updated = await db.checkInMonitor.updateMany({ + where: { id: row.id, token_encrypted: row.token_encrypted }, + data: { token_encrypted: encryptCheckInToken(token) }, + }); + if (updated.count === 1) result.reencrypted++; + } else { + result.reencrypted++; + } + } + cursor = rows[rows.length - 1].id; + if (rows.length < batchSize) break; + } + return result; +} diff --git a/apps/api/src/jobs/run-checkin-reencrypt.ts b/apps/api/src/jobs/run-checkin-reencrypt.ts new file mode 100644 index 00000000..30233d68 --- /dev/null +++ b/apps/api/src/jobs/run-checkin-reencrypt.ts @@ -0,0 +1,30 @@ +/** + * CLI: re-encrypt check-in tokens with the current CHECKIN_ENCRYPTION_KEY. + * + * node dist/jobs/run-checkin-reencrypt.js # dry run (default) + * node dist/jobs/run-checkin-reencrypt.js --apply # write + * + * Requires DATABASE_URL, CHECKIN_ENCRYPTION_KEY and (during rotation) CHECKIN_ENCRYPTION_KEY_PREVIOUS. + * Prints counts only; never prints tokens or keys. + */ +import { prisma } from "../lib/db.js"; +import { parseCheckInReencryptArgs, runCheckInReencrypt } from "./checkin-reencrypt.js"; + +const parsed = parseCheckInReencryptArgs(process.argv.slice(2)); +if (parsed.help) { + console.log("Usage: node dist/jobs/run-checkin-reencrypt.js [--apply] [--batch-size ]\nDry run by default. Idempotent. See docs/CHECK-INS.md (key rotation)."); + process.exit(0); +} +if (parsed.error) { console.error(parsed.error); process.exit(1); } + +runCheckInReencrypt(prisma, { apply: parsed.apply, batchSize: parsed.batchSize }) + .then(async (result) => { + console.log(JSON.stringify({ ok: result.failed === 0, ...result, at: new Date().toISOString() })); + await prisma.$disconnect(); + if (result.failed > 0) process.exit(1); + }) + .catch(async (e: unknown) => { + console.error(`[checkin-reencrypt] failed: ${e instanceof Error ? e.message : "unknown error"}`); + await prisma.$disconnect().catch(() => {}); + process.exit(1); + }); diff --git a/apps/api/src/lib/checkin-key-rotation.test.ts b/apps/api/src/lib/checkin-key-rotation.test.ts new file mode 100644 index 00000000..2bfa2069 --- /dev/null +++ b/apps/api/src/lib/checkin-key-rotation.test.ts @@ -0,0 +1,99 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createCipheriv, randomBytes } from "node:crypto"; +import { checkInTokenHash, createCheckInSecret, decryptCheckInSecret, isEncryptedWithCurrentKey } from "./checkin-secrets.js"; +import { parseCheckInReencryptArgs, runCheckInReencrypt } from "../jobs/checkin-reencrypt.js"; + +const OLD = "ab".repeat(32), NEW = "cd".repeat(32), OTHER = "ef".repeat(32); +afterEach(() => vi.unstubAllEnvs()); + +/** Legacy (pre-rotation) format: bare base64(iv|tag|ct). */ +function legacyEncrypt(keyHex: string, token: string): string { + const iv = randomBytes(12); + const c = createCipheriv("aes-256-gcm", Buffer.from(keyHex, "hex"), iv); + const ct = Buffer.concat([c.update(token, "utf8"), c.final()]); + return Buffer.concat([iv, c.getAuthTag(), ct]).toString("base64"); +} +function setKeys(current: string, previous?: string) { + vi.stubEnv("CHECKIN_ENCRYPTION_KEY", current); + vi.stubEnv("CHECKIN_ENCRYPTION_KEY_PREVIOUS", previous ?? ""); +} +function fakeDb(rows: { id: string; token_hash: string; token_encrypted: string }[]) { + return { + checkInMonitor: { + findMany: vi.fn(async (args: { take: number; cursor?: { id: string } }) => { + const sorted = [...rows].sort((a, b) => a.id.localeCompare(b.id)); + const start = args.cursor ? sorted.findIndex((r) => r.id === args.cursor!.id) + 1 : 0; + return sorted.slice(start, start + args.take).map((r) => ({ ...r })); + }), + updateMany: vi.fn(async (args: { where: { id: string; token_encrypted: string }; data: { token_encrypted: string } }) => { + const r = rows.find((x) => x.id === args.where.id && x.token_encrypted === args.where.token_encrypted); + if (!r) return { count: 0 }; + r.token_encrypted = args.data.token_encrypted; return { count: 1 }; + }), + }, + }; +} + +describe("check-in encryption key rotation", () => { + it("decrypts legacy and versioned old-key data after rotation", () => { + setKeys(OLD); + const versioned = createCheckInSecret().token_encrypted; + const legacy = legacyEncrypt(OLD, "ttc_legacy"); + setKeys(NEW, `${OTHER}, ${OLD}`); + expect(decryptCheckInSecret(legacy)).toBe("ttc_legacy"); + expect(decryptCheckInSecret(versioned)).toMatch(/^ttc_/); + expect(isEncryptedWithCurrentKey(versioned)).toBe(false); + }); + it("encrypts new writes with the current key only", () => { + setKeys(NEW, OLD); + const enc = createCheckInSecret().token_encrypted; + expect(enc.startsWith("v1.")).toBe(true); + expect(enc).not.toContain(NEW); + expect(isEncryptedWithCurrentKey(enc)).toBe(true); + setKeys(NEW); + expect(decryptCheckInSecret(enc)).toMatch(/^ttc_/); + setKeys(OLD); + expect(() => decryptCheckInSecret(enc)).toThrow("could not be decrypted"); + }); + it("fails safely without the previous key and rejects malformed previous keys", () => { + const legacy = legacyEncrypt(OLD, "ttc_x"); + setKeys(NEW); + expect(() => decryptCheckInSecret(legacy)).toThrow("could not be decrypted"); + setKeys(NEW, "not-hex"); + expect(() => decryptCheckInSecret(legacy)).toThrow("CHECKIN_ENCRYPTION_KEY_PREVIOUS"); + setKeys("", OLD); + expect(() => createCheckInSecret()).toThrow("CHECKIN_ENCRYPTION_KEY"); + }); + it("re-encrypt is dry-run by default, migrates with --apply and is idempotent", async () => { + const rows = Array.from({ length: 5 }, (_, i) => { + const token = `ttc_${i}`; + return { id: `m${i}`, token_hash: checkInTokenHash(token), token_encrypted: legacyEncrypt(OLD, token) }; + }); + setKeys(NEW, OLD); + const db = fakeDb(rows); + const log = vi.spyOn(console, "log"); + const dry = await runCheckInReencrypt(db as never, { batchSize: 2 }); + expect(dry).toEqual({ dryRun: true, scanned: 5, alreadyCurrent: 0, reencrypted: 5, failed: 0 }); + expect(db.checkInMonitor.updateMany).not.toHaveBeenCalled(); + const applied = await runCheckInReencrypt(db as never, { apply: true, batchSize: 2 }); + expect(applied).toMatchObject({ dryRun: false, scanned: 5, reencrypted: 5, failed: 0 }); + const again = await runCheckInReencrypt(db as never, { apply: true, batchSize: 2 }); + expect(again).toMatchObject({ scanned: 5, alreadyCurrent: 5, reencrypted: 0, failed: 0 }); + setKeys(NEW); + rows.forEach((r, i) => expect(decryptCheckInSecret(r.token_encrypted)).toBe(`ttc_${i}`)); + expect(log).not.toHaveBeenCalled(); + }); + it("re-encrypt counts undecryptable rows as failed and leaves them unchanged", async () => { + const row = { id: "m1", token_hash: checkInTokenHash("ttc_a"), token_encrypted: legacyEncrypt(OLD, "ttc_a") }; + setKeys(NEW); + const before = row.token_encrypted; + const r = await runCheckInReencrypt(fakeDb([row]) as never, { apply: true }); + expect(r).toMatchObject({ failed: 1, reencrypted: 0 }); + expect(row.token_encrypted).toBe(before); + }); + it("parses CLI args with dry-run default", () => { + expect(parseCheckInReencryptArgs([])).toMatchObject({ apply: false, batchSize: 200 }); + expect(parseCheckInReencryptArgs(["--apply", "--batch-size", "50"])).toMatchObject({ apply: true, batchSize: 50 }); + expect(parseCheckInReencryptArgs(["--nope"]).error).toBeTruthy(); + }); +}); diff --git a/apps/api/src/lib/checkin-secrets.ts b/apps/api/src/lib/checkin-secrets.ts index 91e8c2b6..2395c1c4 100644 --- a/apps/api/src/lib/checkin-secrets.ts +++ b/apps/api/src/lib/checkin-secrets.ts @@ -3,27 +3,83 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node: export function checkInTokenHash(token: string): string { return createHash("sha256").update(token).digest("hex"); } -function encryptionKey(): Buffer { - const raw = process.env.CHECKIN_ENCRYPTION_KEY ?? ""; - if (!/^[a-f0-9]{64}$/i.test(raw)) throw new Error("CHECKIN_ENCRYPTION_KEY must be 32 bytes in hex"); - return Buffer.from(raw, "hex"); + +/** + * Encryption keys for check-in tokens. + * - CHECKIN_ENCRYPTION_KEY: current key (32 bytes hex), used for all new encryption. + * - CHECKIN_ENCRYPTION_KEY_PREVIOUS: optional comma-separated old keys, decryption only. + * Ciphertext format: `v1..` where kid is a short SHA-256 fingerprint + * of the key (not the key). Legacy rows are bare base64 and are tried against every key; + * AES-256-GCM authentication rejects a wrong key. + */ +type CheckInKey = { id: string; key: Buffer }; +const KEY_RE = /^[a-f0-9]{64}$/i; +const PREFIX = "v1."; + +function toKey(raw: string): CheckInKey { + const key = Buffer.from(raw, "hex"); + return { id: createHash("sha256").update(key).digest("hex").slice(0, 16), key }; } -export function createCheckInSecret() { - const token = `ttc_${randomBytes(32).toString("hex")}`; +function currentKey(): CheckInKey { + const raw = (process.env.CHECKIN_ENCRYPTION_KEY ?? "").trim(); + if (!KEY_RE.test(raw)) throw new Error("CHECKIN_ENCRYPTION_KEY must be 32 bytes in hex"); + return toKey(raw); +} +function previousKeys(): CheckInKey[] { + const raw = process.env.CHECKIN_ENCRYPTION_KEY_PREVIOUS ?? ""; + const parts = raw.split(",").map((s) => s.trim()).filter(Boolean); + for (const p of parts) { + if (!KEY_RE.test(p)) throw new Error("CHECKIN_ENCRYPTION_KEY_PREVIOUS must be comma-separated 32-byte hex keys"); + } + return parts.map(toKey); +} +/** Current key first, then previous keys (deduplicated). */ +function decryptionKeys(): CheckInKey[] { + const keys = [currentKey(), ...previousKeys()]; + return keys.filter((k, i) => keys.findIndex((o) => o.id === k.id) === i); +} + +function encryptWith(k: CheckInKey, plaintext: string): string { const iv = randomBytes(12); - const cipher = createCipheriv("aes-256-gcm", encryptionKey(), iv); - const ciphertext = Buffer.concat([cipher.update(token, "utf8"), cipher.final()]); - return { - token_hash: checkInTokenHash(token), - token_encrypted: Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString("base64"), - }; + const cipher = createCipheriv("aes-256-gcm", k.key, iv); + const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + return `${PREFIX}${k.id}.${Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString("base64")}`; } -export function decryptCheckInSecret(encrypted: string): string { - const bytes = Buffer.from(encrypted, "base64"); - const decipher = createDecipheriv("aes-256-gcm", encryptionKey(), bytes.subarray(0, 12)); +function decryptWith(k: CheckInKey, payload: string): string { + const bytes = Buffer.from(payload, "base64"); + const decipher = createDecipheriv("aes-256-gcm", k.key, bytes.subarray(0, 12)); decipher.setAuthTag(bytes.subarray(12, 28)); return Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString("utf8"); } +function parse(encrypted: string): { kid: string | null; payload: string } { + if (encrypted.startsWith(PREFIX)) { + const rest = encrypted.slice(PREFIX.length); + const dot = rest.indexOf("."); + if (dot > 0) return { kid: rest.slice(0, dot), payload: rest.slice(dot + 1) }; + } + return { kid: null, payload: encrypted }; +} + +export function encryptCheckInToken(token: string): string { + return encryptWith(currentKey(), token); +} +export function createCheckInSecret() { + const token = `ttc_${randomBytes(32).toString("hex")}`; + return { token_hash: checkInTokenHash(token), token_encrypted: encryptCheckInToken(token) }; +} +export function decryptCheckInSecret(encrypted: string): string { + const { kid, payload } = parse(encrypted); + const keys = decryptionKeys(); + const ordered = kid ? [...keys.filter((k) => k.id === kid), ...keys.filter((k) => k.id !== kid)] : keys; + for (const k of ordered) { + try { return decryptWith(k, payload); } catch { /* wrong key or tampered: try next */ } + } + throw new Error("Check-in secret could not be decrypted with the configured keys"); +} +/** True when the value is already encrypted with the current key (versioned format). */ +export function isEncryptedWithCurrentKey(encrypted: string): boolean { + return parse(encrypted).kid === currentKey().id; +} export function redactCheckInSecrets(text: string): string { return text.replace(/ttc_[a-zA-Z0-9_%.-]+/g, "[redacted]") .replace(/(\/checkin\/)[^\s/?#"<>]+/gi, "$1[redacted]"); diff --git a/docs/CHECK-INS.md b/docs/CHECK-INS.md index 9b110a33..8ad2a75d 100644 --- a/docs/CHECK-INS.md +++ b/docs/CHECK-INS.md @@ -165,6 +165,26 @@ viewable URL; SHA-256 of the 256-bit random token provides indexed authenticatio Losing/changing this encryption key makes existing URLs unreadable in the dashboard; rotate affected monitor URLs afterwards. Do not reuse an ingest key or a public key. +### Rotating CHECKIN_ENCRYPTION_KEY without data loss + +Pings authenticate by SHA-256 hash, so they keep working throughout. The key only +protects the copyable URL shown in the dashboard. New encryption always uses +`CHECKIN_ENCRYPTION_KEY`; `CHECKIN_ENCRYPTION_KEY_PREVIOUS` (comma-separated, optional) +is used only to decrypt older values. Never paste key values into tickets or logs. + +1. Deploy a release that contains key-rotation support (this change). +2. Generate a new key (`openssl rand -hex 32`). On the API service **and on any job + service that shares API code**, set `CHECKIN_ENCRYPTION_KEY_PREVIOUS` to the old key and + `CHECKIN_ENCRYPTION_KEY` to the new key. Deploy. +3. Dry run (default, counts only): `node dist/jobs/run-checkin-reencrypt.js` +4. Apply: `node dist/jobs/run-checkin-reencrypt.js --apply` (batched, idempotent, safe to re-run). +5. Verify: re-run the dry run; expect `reencrypted: 0`, `failed: 0`, `alreadyCurrent` equal to + `scanned`. Open a monitor in the dashboard and confirm its URL is shown. +6. Remove `CHECKIN_ENCRYPTION_KEY_PREVIOUS` and deploy. Then retire the old key. + +If the script reports `failed > 0`, keep `CHECKIN_ENCRYPTION_KEY_PREVIOUS` set and investigate; +those rows cannot be decrypted with any configured key (rotate those monitor URLs instead). + Set dashboard **CHECKIN_PUBLIC_BASE_URL** to the publicly reachable API origin if `API_URL` is an internal address. Otherwise `API_URL` is used for copied URLs.