feat(api): add observability readiness harness

README.md

@@ -569,6 +569,8 @@ AUTH_CHALLENGE_TTL_SECONDS=300
 AUTH_COOKIE_DOMAIN=
 AUTH_COOKIE_SECURE=true
 HIVE_AUTH_AUDIENCE="hivelore-local-api"
+ERROR_TRACKING_ENABLED=false
+ERROR_TRACKING_WEBHOOK_URL=
 
 # Hive integration defaults to public development endpoints.
 HIVE_RPC_URL="https://api.hive.blog"
@@ -758,10 +760,13 @@ INDEXER_NAME=hivelore-haf
 INDEXER_START_BLOCK=1
 INDEXER_BATCH_SIZE=100
 INDEXER_MAX_BLOCKS_PER_RUN=1000
+INDEXER_MAX_READY_LAG_BLOCKS=1200
 ```
 
 Run `npm run db:generate` after pulling schema changes, then apply the committed Prisma migrations before running it against a real database.
 
+`GET /health` is the lightweight liveness endpoint. `GET /ready` verifies PostgreSQL connectivity and reports HAF indexer lag against `INDEXER_MAX_READY_LAG_BLOCKS`; it returns `503` when any dependency is degraded. API logs are structured JSON with an `x-request-id` request identifier, and `ERROR_TRACKING_ENABLED=true` posts unhandled error summaries to `ERROR_TRACKING_WEBHOOK_URL` without request headers or secret-bearing bodies.
+
 ## Run Locally
 
 ```bash
@@ -773,6 +778,7 @@ Local services:
 - Web: http://localhost:3000
 - API: http://localhost:3001
 - Health: http://localhost:3001/health
+- Readiness: http://localhost:3001/ready
 
 Run one application at a time:
 
@@ -869,7 +875,7 @@ hivelore/
 Directory roles:
 
 - `apps/web` is the only frontend application. It is a Next.js App Router scaffold.
-- `apps/api` is the only backend application. It is a Fastify TypeScript scaffold with `GET /health`.
+- `apps/api` is the only backend application. It is a Fastify TypeScript API with liveness and readiness endpoints.
 - `apps/api/prisma` contains the canonical API Prisma schema, migrations, and development seed.
 - `packages` is reserved for shared workspace packages. `packages/config` is intentionally minimal until real shared configuration is needed.
 - `.github/workflows` contains CI checks for formatting, linting, type-checking, and builds.

apps/api/.env.example

@@ -24,6 +24,8 @@ AUTH_COOKIE_SECURE=true
 PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE=10000
 PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX=5
 PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS=60
+ERROR_TRACKING_ENABLED=false
+ERROR_TRACKING_WEBHOOK_URL=
 
 # Hive integration is mainnet-only and defaults to public read/write endpoints for development.
 # The chain ID is validated positively against Hive mainnet.
@@ -66,6 +68,7 @@ INDEXER_NAME=hivelore-haf
 INDEXER_START_BLOCK=1
 INDEXER_BATCH_SIZE=100
 INDEXER_MAX_BLOCKS_PER_RUN=1000
+INDEXER_MAX_READY_LAG_BLOCKS=1200
 
 # PostgreSQL application connection through the Supabase pooler.
 DATABASE_URL="postgresql://USER:PASSWORD@POOLER_HOST:6543/postgres?pgbouncer=true"

apps/api/src/app.ts

@@ -1,9 +1,12 @@
 import cors from '@fastify/cors';
 import rateLimit from '@fastify/rate-limit';
 import Fastify from 'fastify';
+import { randomUUID } from 'node:crypto';
 
 import { env } from './config/env.js';
 import { createPrismaRateLimitStore } from './lib/auth-rate-limit-store.js';
+import { reportUnhandledError } from './lib/error-tracking.js';
+import { createHafClient } from './lib/hive/client.js';
 import { prisma } from './lib/prisma.js';
 import { registerAuthRoutes } from './routes/auth.js';
 import { registerContributionRoutes } from './routes/contributions.js';
@@ -15,6 +18,7 @@ import { registerWorldRoutes } from './routes/worlds.js';
 
 export async function buildApp() {
   const app = Fastify({
+    genReqId: () => randomUUID(),
     logger: {
       level: env.NODE_ENV === 'production' ? 'info' : 'debug',
       redact: [
@@ -31,9 +35,34 @@ export async function buildApp() {
         'body.refresh_token',
       ],
     },
+    requestIdHeader: 'x-request-id',
     trustProxy: env.TRUST_PROXY,
   });
 
+  app.addHook('onRequest', async (request, reply) => {
+    reply.header('x-request-id', request.id);
+  });
+
+  app.addHook('onError', async (request, _reply, error) => {
+    // Deliberately NOT awaited: Fastify waits for this hook before sending the
+    // error response, so awaiting webhook delivery would let a slow tracker
+    // delay - or with no timeout, indefinitely stall - every failing request.
+    // reportUnhandledError swallows its own errors, so this cannot reject.
+    void reportUnhandledError(
+      {
+        error,
+        method: request.method,
+        requestId: request.id,
+        url: request.url,
+      },
+      {
+        enabled: env.ERROR_TRACKING_ENABLED,
+        logger: request.log,
+        webhookUrl: env.ERROR_TRACKING_WEBHOOK_URL,
+      },
+    );
+  });
+
   await app.register(cors, {
     credentials: true,
     origin: env.NODE_ENV === 'production' ? env.CORS_ORIGIN : [env.CORS_ORIGIN],
@@ -47,7 +76,12 @@ export async function buildApp() {
     store: createPrismaRateLimitStore(prisma),
   });
 
-  await registerHealthRoute(app);
+  await registerHealthRoute(app, {
+    database: prisma,
+    getHeadBlock: () => createHafClient().getHeadBlock(),
+    indexerLagThresholdBlocks: env.INDEXER_MAX_READY_LAG_BLOCKS,
+    indexerName: env.INDEXER_NAME,
+  });
   await registerAuthRoutes(app);
   await registerWorldRoutes(app);
   await registerLoreRoutes(app);

apps/api/src/config/env.test.ts

@@ -87,6 +87,28 @@ describe('API environment validation', () => {
     assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE, 10_000);
     assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX, 5);
     assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS, 60);
+    assert.equal(env.ERROR_TRACKING_ENABLED, false);
+    assert.equal(env.INDEXER_MAX_READY_LAG_BLOCKS, 1_200);
+  });
+
+  test('requires an error tracking webhook only when alerting is enabled', () => {
+    assert.throws(
+      () =>
+        parseEnv({
+          ERROR_TRACKING_ENABLED: 'true',
+          NODE_ENV: 'test',
+        }),
+      /ERROR_TRACKING_WEBHOOK_URL/,
+    );
+
+    assert.equal(
+      parseEnv({
+        ERROR_TRACKING_ENABLED: 'true',
+        ERROR_TRACKING_WEBHOOK_URL: 'https://example.com/hivelore-errors',
+        NODE_ENV: 'test',
+      }).ERROR_TRACKING_ENABLED,
+      true,
+    );
   });
 
   test('rejects invalid Hive network configuration', () => {

apps/api/src/config/env.ts

@@ -37,6 +37,8 @@ const envSchema = z.object({
   PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE: z.coerce.number().int().positive().default(10_000),
   PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX: z.coerce.number().int().positive().default(5),
   PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS: z.coerce.number().int().positive().default(60),
+  ERROR_TRACKING_ENABLED: booleanEnv(false),
+  ERROR_TRACKING_WEBHOOK_URL: z.string().url().optional(),
   HIVE_RPC_URL: z.string().url().default('https://api.hive.blog'),
   HIVE_MAINNET_CHAIN_ID: z
     .string()
@@ -105,6 +107,7 @@ const envSchema = z.object({
   INDEXER_START_BLOCK: z.coerce.number().int().positive().default(1),
   INDEXER_BATCH_SIZE: z.coerce.number().int().positive().max(1_000).default(100),
   INDEXER_MAX_BLOCKS_PER_RUN: z.coerce.number().int().positive().default(1_000),
+  INDEXER_MAX_READY_LAG_BLOCKS: z.coerce.number().int().nonnegative().default(1_200),
   GOOGLE_AUTH_ENABLED: booleanEnv(false),
   GOOGLE_CLIENT_ID: z.string().optional(),
   GOOGLE_CLIENT_SECRET: z.string().optional(),
@@ -174,6 +177,10 @@ export function parseEnv(environment: NodeJS.ProcessEnv) {
     throw new Error('Hive RC delegation is enabled but delegation configuration is missing.');
   }
 
+  if (parsedEnv.ERROR_TRACKING_ENABLED && !parsedEnv.ERROR_TRACKING_WEBHOOK_URL) {
+    throw new Error('Error tracking is enabled but ERROR_TRACKING_WEBHOOK_URL is missing.');
+  }
+
   if (!parsedEnv.AUTH_JWT_SECRET || !parsedEnv.AUTH_REFRESH_SECRET) {
     throw new Error('Authentication secrets must be set.');
   }

apps/api/src/lib/canon-voting-concurrency.test.ts

@@ -1,11 +1,6 @@
 import assert from 'node:assert/strict';
-import { spawnSync } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
 import { test } from 'node:test';
 
-import { PrismaPg } from '@prisma/adapter-pg';
-import pg from 'pg';
-
 import { PrismaClient } from '../generated/prisma/client.js';
 import {
   PlatformRole,
@@ -19,60 +14,11 @@ import { confirmCanonTransaction, CanonVotingError } from './canon-voting.js';
 import { hashCanonicalJson } from './canon-voting-policy.js';
 import { HIVELORE_CUSTOM_JSON_ID } from './hive/constants.js';
 import { buildHiveLoreCustomJsonOperation } from './hive/operations.js';
-
-const { Client } = pg;
-
-const adminUrl = process.env.TEST_DATABASE_ADMIN_URL;
-
-function disposableDatabaseName() {
-  return `hivelore_confirm_race_${Date.now()}_${process.pid}_${randomUUID().replaceAll('-', '').slice(0, 8)}`;
-}
-
-function databaseUrlFor(adminConnectionUrl: string, databaseName: string) {
-  const databaseUrl = new URL(adminConnectionUrl);
-  databaseUrl.pathname = `/${databaseName}`;
-
-  return databaseUrl.toString();
-}
-
-async function withPgClient<T>(
-  connectionString: string,
-  callback: (client: pg.Client) => Promise<T>,
-) {
-  const client = new Client({ connectionString });
-  await client.connect();
-
-  try {
-    return await callback(client);
-  } finally {
-    await client.end();
-  }
-}
-
-function runPrismaMigrateDeploy(databaseUrl: string) {
-  const executable = process.platform === 'win32' ? 'npx.cmd' : 'npx';
-  const result = spawnSync(executable, ['prisma', 'migrate', 'deploy'], {
-    cwd: new URL('../..', import.meta.url),
-    env: {
-      ...process.env,
-      DATABASE_URL: databaseUrl,
-      DIRECT_URL: databaseUrl,
-      NODE_ENV: 'test',
-    },
-    shell: false,
-    stdio: 'inherit',
-  });
-
-  if (result.status !== 0) {
-    throw new Error(`prisma migrate deploy failed with exit code ${result.status}`);
-  }
-}
-
-function createPrismaClient(connectionString: string) {
-  return new PrismaClient({
-    adapter: new PrismaPg({ connectionString }),
-  });
-}
+import {
+  createPrismaClient,
+  testDatabaseAdminUrl,
+  withDisposablePrismaDatabase,
+} from '../test/integration-db.js';
 
 function createBarrier(parties: number) {
   let arrived = 0;
@@ -197,26 +143,10 @@ async function seedConfirmedProposal(database: PrismaClient) {
 
 test(
   'canon decision confirmation is atomic across concurrent PostgreSQL clients',
-  { skip: !adminUrl ? 'TEST_DATABASE_ADMIN_URL is not configured.' : false },
+  { skip: !testDatabaseAdminUrl ? 'TEST_DATABASE_ADMIN_URL is not configured.' : false },
   async () => {
-    assert.ok(adminUrl);
-    const databaseName = disposableDatabaseName();
-
-    if (!databaseName.startsWith('hivelore_confirm_race_')) {
-      throw new Error(`Refusing to use non-disposable database name: ${databaseName}`);
-    }
-
-    const directUrl = databaseUrlFor(adminUrl, databaseName);
-
-    await withPgClient(adminUrl, (client) => client.query(`CREATE DATABASE "${databaseName}"`));
-
-    try {
-      runPrismaMigrateDeploy(directUrl);
-
-      const seedClient = createPrismaClient(directUrl);
+    await withDisposablePrismaDatabase('hivelore_confirm_race', async (seedClient, directUrl) => {
       await seedConfirmedProposal(seedClient);
-      await seedClient.$disconnect();
-
       const firstClient = createPrismaClient(directUrl);
       const secondClient = createPrismaClient(directUrl);
       const synchronizeLookup = createBarrier(2);
@@ -288,19 +218,6 @@ test(
         secondClient.$disconnect(),
         verificationClient.$disconnect(),
       ]);
-    } finally {
-      await withPgClient(adminUrl, async (client) => {
-        await client.query(
-          `
-            SELECT pg_terminate_backend(pid)
-            FROM pg_stat_activity
-            WHERE datname = $1
-              AND pid <> pg_backend_pid()
-          `,
-          [databaseName],
-        );
-        await client.query(`DROP DATABASE IF EXISTS "${databaseName}"`);
-      });
-    }
+    });
   },
 );

apps/api/src/lib/error-tracking.test.ts

@@ -0,0 +1,67 @@
+import { createServer } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import assert from 'node:assert/strict';
+import { describe, test } from 'node:test';
+
+import { reportUnhandledError, serializeError } from './error-tracking.js';
+
+describe('error tracking', () => {
+  test('serializes errors without request secrets', () => {
+    const serialized = serializeError(new Error('boom'));
+
+    assert.equal(serialized.name, 'Error');
+    assert.equal(serialized.message, 'boom');
+    assert.equal('headers' in serialized, false);
+  });
+
+  test('logs unhandled errors and skips webhook delivery when disabled', async () => {
+    const logs: unknown[] = [];
+
+    await reportUnhandledError(
+      {
+        error: new Error('boom'),
+        method: 'GET',
+        requestId: 'request-1',
+        url: '/explode',
+      },
+      {
+        enabled: false,
+        logger: {
+          error(payload: unknown) {
+            logs.push(payload);
+          },
+        } as never,
+        webhookUrl: 'https://example.com/errors',
+      },
+    );
+
+    assert.equal(logs.length, 1);
+    assert.equal((logs[0] as { requestId?: string }).requestId, 'request-1');
+  });
+
+  test('gives up on an unresponsive webhook instead of hanging the error path', async () => {
+    // Regression guard. Fastify awaits the onError hook, so an unbounded fetch
+    // to a stalled webhook would keep every failing request open until OS-level
+    // TCP timeouts. Against the unbounded version this test never resolves.
+    const stalled = createServer(() => {
+      // accept the connection, never reply
+    });
+    await new Promise<void>((resolve) => stalled.listen(0, resolve));
+    const address = stalled.address() as AddressInfo;
+
+    const startedAt = Date.now();
+    await reportUnhandledError(
+      { error: new Error('boom'), method: 'GET', requestId: 'req-stall', url: '/x' },
+      {
+        enabled: true,
+        timeoutMs: 150,
+        webhookUrl: `http://127.0.0.1:${address.port}/hook`,
+      },
+    );
+    const elapsed = Date.now() - startedAt;
+
+    stalled.close();
+
+    assert.ok(elapsed < 2_000, `expected the webhook to be abandoned quickly, took ${elapsed}ms`);
+  });
+});