From 1a3eab46581db41150fdb5ceb38e6ba9630e1937 Mon Sep 17 00:00:00 2001 From: Kareem Naous <62202089+KareemN1936@users.noreply.github.com> Date: Sun, 6 Sep 2026 02:17:32 +0300 Subject: [PATCH 1/2] feat(api): add observability readiness harness --- README.md | 8 +- apps/api/.env.example | 3 + apps/api/src/app.ts | 32 +++- apps/api/src/config/env.test.ts | 22 +++ apps/api/src/config/env.ts | 7 + .../src/lib/canon-voting-concurrency.test.ts | 99 +----------- apps/api/src/lib/error-tracking.test.ts | 39 +++++ apps/api/src/lib/error-tracking.ts | 67 ++++++++ apps/api/src/routes/health.test.ts | 144 ++++++++++++++++++ apps/api/src/routes/health.ts | 126 ++++++++++++++- apps/api/src/test/integration-db.ts | 108 +++++++++++++ 11 files changed, 561 insertions(+), 94 deletions(-) create mode 100644 apps/api/src/lib/error-tracking.test.ts create mode 100644 apps/api/src/lib/error-tracking.ts create mode 100644 apps/api/src/routes/health.test.ts create mode 100644 apps/api/src/test/integration-db.ts diff --git a/README.md b/README.md index 453a0d9..7917ff3 100644 --- a/README.md +++ b/README.md @@ -569,6 +569,8 @@ AUTH_CHALLENGE_TTL_SECONDS=300 AUTH_COOKIE_DOMAIN= AUTH_COOKIE_SECURE=true HIVE_AUTH_AUDIENCE="hivelore-local-api" +ERROR_TRACKING_ENABLED=false +ERROR_TRACKING_WEBHOOK_URL= # Hive integration defaults to public development endpoints. HIVE_RPC_URL="https://api.hive.blog" @@ -758,10 +760,13 @@ INDEXER_NAME=hivelore-haf INDEXER_START_BLOCK=1 INDEXER_BATCH_SIZE=100 INDEXER_MAX_BLOCKS_PER_RUN=1000 +INDEXER_MAX_READY_LAG_BLOCKS=1200 ``` Run `npm run db:generate` after pulling schema changes, then apply the committed Prisma migrations before running it against a real database. +`GET /health` is the lightweight liveness endpoint. `GET /ready` verifies PostgreSQL connectivity and reports HAF indexer lag against `INDEXER_MAX_READY_LAG_BLOCKS`; it returns `503` when any dependency is degraded. API logs are structured JSON with an `x-request-id` request identifier, and `ERROR_TRACKING_ENABLED=true` posts unhandled error summaries to `ERROR_TRACKING_WEBHOOK_URL` without request headers or secret-bearing bodies. + ## Run Locally ```bash @@ -773,6 +778,7 @@ Local services: - Web: http://localhost:3000 - API: http://localhost:3001 - Health: http://localhost:3001/health +- Readiness: http://localhost:3001/ready Run one application at a time: @@ -869,7 +875,7 @@ hivelore/ Directory roles: - `apps/web` is the only frontend application. It is a Next.js App Router scaffold. -- `apps/api` is the only backend application. It is a Fastify TypeScript scaffold with `GET /health`. +- `apps/api` is the only backend application. It is a Fastify TypeScript API with liveness and readiness endpoints. - `apps/api/prisma` contains the canonical API Prisma schema, migrations, and development seed. - `packages` is reserved for shared workspace packages. `packages/config` is intentionally minimal until real shared configuration is needed. - `.github/workflows` contains CI checks for formatting, linting, type-checking, and builds. diff --git a/apps/api/.env.example b/apps/api/.env.example index e9d58f8..bc5cac4 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -24,6 +24,8 @@ AUTH_COOKIE_SECURE=true PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE=10000 PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX=5 PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS=60 +ERROR_TRACKING_ENABLED=false +ERROR_TRACKING_WEBHOOK_URL= # Hive integration is mainnet-only and defaults to public read/write endpoints for development. # The chain ID is validated positively against Hive mainnet. @@ -66,6 +68,7 @@ INDEXER_NAME=hivelore-haf INDEXER_START_BLOCK=1 INDEXER_BATCH_SIZE=100 INDEXER_MAX_BLOCKS_PER_RUN=1000 +INDEXER_MAX_READY_LAG_BLOCKS=1200 # PostgreSQL application connection through the Supabase pooler. DATABASE_URL="postgresql://USER:PASSWORD@POOLER_HOST:6543/postgres?pgbouncer=true" diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 756fc61..61158b0 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -1,9 +1,12 @@ import cors from '@fastify/cors'; import rateLimit from '@fastify/rate-limit'; import Fastify from 'fastify'; +import { randomUUID } from 'node:crypto'; import { env } from './config/env.js'; import { createPrismaRateLimitStore } from './lib/auth-rate-limit-store.js'; +import { reportUnhandledError } from './lib/error-tracking.js'; +import { createHafClient } from './lib/hive/client.js'; import { prisma } from './lib/prisma.js'; import { registerAuthRoutes } from './routes/auth.js'; import { registerContributionRoutes } from './routes/contributions.js'; @@ -15,6 +18,7 @@ import { registerWorldRoutes } from './routes/worlds.js'; export async function buildApp() { const app = Fastify({ + genReqId: () => randomUUID(), logger: { level: env.NODE_ENV === 'production' ? 'info' : 'debug', redact: [ @@ -31,9 +35,30 @@ export async function buildApp() { 'body.refresh_token', ], }, + requestIdHeader: 'x-request-id', trustProxy: env.TRUST_PROXY, }); + app.addHook('onRequest', async (request, reply) => { + reply.header('x-request-id', request.id); + }); + + app.addHook('onError', async (request, _reply, error) => { + await reportUnhandledError( + { + error, + method: request.method, + requestId: request.id, + url: request.url, + }, + { + enabled: env.ERROR_TRACKING_ENABLED, + logger: request.log, + webhookUrl: env.ERROR_TRACKING_WEBHOOK_URL, + }, + ); + }); + await app.register(cors, { credentials: true, origin: env.NODE_ENV === 'production' ? env.CORS_ORIGIN : [env.CORS_ORIGIN], @@ -47,7 +72,12 @@ export async function buildApp() { store: createPrismaRateLimitStore(prisma), }); - await registerHealthRoute(app); + await registerHealthRoute(app, { + database: prisma, + getHeadBlock: () => createHafClient().getHeadBlock(), + indexerLagThresholdBlocks: env.INDEXER_MAX_READY_LAG_BLOCKS, + indexerName: env.INDEXER_NAME, + }); await registerAuthRoutes(app); await registerWorldRoutes(app); await registerLoreRoutes(app); diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.ts index 2353997..ac9d618 100644 --- a/apps/api/src/config/env.test.ts +++ b/apps/api/src/config/env.test.ts @@ -87,6 +87,28 @@ describe('API environment validation', () => { assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE, 10_000); assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX, 5); assert.equal(env.PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS, 60); + assert.equal(env.ERROR_TRACKING_ENABLED, false); + assert.equal(env.INDEXER_MAX_READY_LAG_BLOCKS, 1_200); + }); + + test('requires an error tracking webhook only when alerting is enabled', () => { + assert.throws( + () => + parseEnv({ + ERROR_TRACKING_ENABLED: 'true', + NODE_ENV: 'test', + }), + /ERROR_TRACKING_WEBHOOK_URL/, + ); + + assert.equal( + parseEnv({ + ERROR_TRACKING_ENABLED: 'true', + ERROR_TRACKING_WEBHOOK_URL: 'https://example.com/hivelore-errors', + NODE_ENV: 'test', + }).ERROR_TRACKING_ENABLED, + true, + ); }); test('rejects invalid Hive network configuration', () => { diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index 8939fa1..a0b9f67 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -37,6 +37,8 @@ const envSchema = z.object({ PROPOSAL_COMMENT_WRITE_RATE_LIMIT_CACHE: z.coerce.number().int().positive().default(10_000), PROPOSAL_COMMENT_WRITE_RATE_LIMIT_MAX: z.coerce.number().int().positive().default(5), PROPOSAL_COMMENT_WRITE_RATE_LIMIT_WINDOW_SECONDS: z.coerce.number().int().positive().default(60), + ERROR_TRACKING_ENABLED: booleanEnv(false), + ERROR_TRACKING_WEBHOOK_URL: z.string().url().optional(), HIVE_RPC_URL: z.string().url().default('https://api.hive.blog'), HIVE_MAINNET_CHAIN_ID: z .string() @@ -105,6 +107,7 @@ const envSchema = z.object({ INDEXER_START_BLOCK: z.coerce.number().int().positive().default(1), INDEXER_BATCH_SIZE: z.coerce.number().int().positive().max(1_000).default(100), INDEXER_MAX_BLOCKS_PER_RUN: z.coerce.number().int().positive().default(1_000), + INDEXER_MAX_READY_LAG_BLOCKS: z.coerce.number().int().nonnegative().default(1_200), GOOGLE_AUTH_ENABLED: booleanEnv(false), GOOGLE_CLIENT_ID: z.string().optional(), GOOGLE_CLIENT_SECRET: z.string().optional(), @@ -174,6 +177,10 @@ export function parseEnv(environment: NodeJS.ProcessEnv) { throw new Error('Hive RC delegation is enabled but delegation configuration is missing.'); } + if (parsedEnv.ERROR_TRACKING_ENABLED && !parsedEnv.ERROR_TRACKING_WEBHOOK_URL) { + throw new Error('Error tracking is enabled but ERROR_TRACKING_WEBHOOK_URL is missing.'); + } + if (!parsedEnv.AUTH_JWT_SECRET || !parsedEnv.AUTH_REFRESH_SECRET) { throw new Error('Authentication secrets must be set.'); } diff --git a/apps/api/src/lib/canon-voting-concurrency.test.ts b/apps/api/src/lib/canon-voting-concurrency.test.ts index bc3b05e..ea7ce56 100644 --- a/apps/api/src/lib/canon-voting-concurrency.test.ts +++ b/apps/api/src/lib/canon-voting-concurrency.test.ts @@ -1,11 +1,6 @@ import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { randomUUID } from 'node:crypto'; import { test } from 'node:test'; -import { PrismaPg } from '@prisma/adapter-pg'; -import pg from 'pg'; - import { PrismaClient } from '../generated/prisma/client.js'; import { PlatformRole, @@ -19,60 +14,11 @@ import { confirmCanonTransaction, CanonVotingError } from './canon-voting.js'; import { hashCanonicalJson } from './canon-voting-policy.js'; import { HIVELORE_CUSTOM_JSON_ID } from './hive/constants.js'; import { buildHiveLoreCustomJsonOperation } from './hive/operations.js'; - -const { Client } = pg; - -const adminUrl = process.env.TEST_DATABASE_ADMIN_URL; - -function disposableDatabaseName() { - return `hivelore_confirm_race_${Date.now()}_${process.pid}_${randomUUID().replaceAll('-', '').slice(0, 8)}`; -} - -function databaseUrlFor(adminConnectionUrl: string, databaseName: string) { - const databaseUrl = new URL(adminConnectionUrl); - databaseUrl.pathname = `/${databaseName}`; - - return databaseUrl.toString(); -} - -async function withPgClient( - connectionString: string, - callback: (client: pg.Client) => Promise, -) { - const client = new Client({ connectionString }); - await client.connect(); - - try { - return await callback(client); - } finally { - await client.end(); - } -} - -function runPrismaMigrateDeploy(databaseUrl: string) { - const executable = process.platform === 'win32' ? 'npx.cmd' : 'npx'; - const result = spawnSync(executable, ['prisma', 'migrate', 'deploy'], { - cwd: new URL('../..', import.meta.url), - env: { - ...process.env, - DATABASE_URL: databaseUrl, - DIRECT_URL: databaseUrl, - NODE_ENV: 'test', - }, - shell: false, - stdio: 'inherit', - }); - - if (result.status !== 0) { - throw new Error(`prisma migrate deploy failed with exit code ${result.status}`); - } -} - -function createPrismaClient(connectionString: string) { - return new PrismaClient({ - adapter: new PrismaPg({ connectionString }), - }); -} +import { + createPrismaClient, + testDatabaseAdminUrl, + withDisposablePrismaDatabase, +} from '../test/integration-db.js'; function createBarrier(parties: number) { let arrived = 0; @@ -197,26 +143,10 @@ async function seedConfirmedProposal(database: PrismaClient) { test( 'canon decision confirmation is atomic across concurrent PostgreSQL clients', - { skip: !adminUrl ? 'TEST_DATABASE_ADMIN_URL is not configured.' : false }, + { skip: !testDatabaseAdminUrl ? 'TEST_DATABASE_ADMIN_URL is not configured.' : false }, async () => { - assert.ok(adminUrl); - const databaseName = disposableDatabaseName(); - - if (!databaseName.startsWith('hivelore_confirm_race_')) { - throw new Error(`Refusing to use non-disposable database name: ${databaseName}`); - } - - const directUrl = databaseUrlFor(adminUrl, databaseName); - - await withPgClient(adminUrl, (client) => client.query(`CREATE DATABASE "${databaseName}"`)); - - try { - runPrismaMigrateDeploy(directUrl); - - const seedClient = createPrismaClient(directUrl); + await withDisposablePrismaDatabase('hivelore_confirm_race', async (seedClient, directUrl) => { await seedConfirmedProposal(seedClient); - await seedClient.$disconnect(); - const firstClient = createPrismaClient(directUrl); const secondClient = createPrismaClient(directUrl); const synchronizeLookup = createBarrier(2); @@ -288,19 +218,6 @@ test( secondClient.$disconnect(), verificationClient.$disconnect(), ]); - } finally { - await withPgClient(adminUrl, async (client) => { - await client.query( - ` - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = $1 - AND pid <> pg_backend_pid() - `, - [databaseName], - ); - await client.query(`DROP DATABASE IF EXISTS "${databaseName}"`); - }); - } + }); }, ); diff --git a/apps/api/src/lib/error-tracking.test.ts b/apps/api/src/lib/error-tracking.test.ts new file mode 100644 index 0000000..37f293a --- /dev/null +++ b/apps/api/src/lib/error-tracking.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; + +import { reportUnhandledError, serializeError } from './error-tracking.js'; + +describe('error tracking', () => { + test('serializes errors without request secrets', () => { + const serialized = serializeError(new Error('boom')); + + assert.equal(serialized.name, 'Error'); + assert.equal(serialized.message, 'boom'); + assert.equal('headers' in serialized, false); + }); + + test('logs unhandled errors and skips webhook delivery when disabled', async () => { + const logs: unknown[] = []; + + await reportUnhandledError( + { + error: new Error('boom'), + method: 'GET', + requestId: 'request-1', + url: '/explode', + }, + { + enabled: false, + logger: { + error(payload: unknown) { + logs.push(payload); + }, + } as never, + webhookUrl: 'https://example.com/errors', + }, + ); + + assert.equal(logs.length, 1); + assert.equal((logs[0] as { requestId?: string }).requestId, 'request-1'); + }); +}); diff --git a/apps/api/src/lib/error-tracking.ts b/apps/api/src/lib/error-tracking.ts new file mode 100644 index 0000000..56dc3e8 --- /dev/null +++ b/apps/api/src/lib/error-tracking.ts @@ -0,0 +1,67 @@ +import type { FastifyBaseLogger } from 'fastify'; + +type ErrorTrackingEvent = { + error: unknown; + method?: string | undefined; + requestId?: string | undefined; + url?: string | undefined; +}; + +type ErrorTrackerOptions = { + enabled?: boolean; + logger?: FastifyBaseLogger; + webhookUrl?: string | undefined; +}; + +export function serializeError(error: unknown) { + if (error instanceof Error) { + return { + message: error.message, + name: error.name, + stack: error.stack, + }; + } + + return { + message: String(error), + name: 'UnknownError', + }; +} + +export async function reportUnhandledError( + event: ErrorTrackingEvent, + options: ErrorTrackerOptions = {}, +) { + const payload = { + error: serializeError(event.error), + method: event.method, + requestId: event.requestId, + service: 'hivelore-api', + timestamp: new Date().toISOString(), + url: event.url, + }; + + options.logger?.error(payload, 'Unhandled API error'); + + if (!options.enabled || !options.webhookUrl) { + return; + } + + try { + await fetch(options.webhookUrl, { + body: JSON.stringify(payload), + headers: { + 'content-type': 'application/json', + }, + method: 'POST', + }); + } catch (error) { + options.logger?.warn( + { + error: serializeError(error), + requestId: event.requestId, + }, + 'Error tracking webhook delivery failed', + ); + } +} diff --git a/apps/api/src/routes/health.test.ts b/apps/api/src/routes/health.test.ts new file mode 100644 index 0000000..1197a32 --- /dev/null +++ b/apps/api/src/routes/health.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; + +import Fastify from 'fastify'; + +import { registerHealthRoute } from './health.js'; +import { testDatabaseAdminUrl, withDisposablePrismaDatabase } from '../test/integration-db.js'; + +function createHealthDatabase(input: { + failDatabase?: boolean; + lastProcessedBlock?: bigint; + lastProcessedOperationIndex?: number; +}) { + return { + async $queryRaw() { + if (input.failDatabase) { + throw new Error('database unavailable'); + } + + return [{ ok: 1 }]; + }, + indexerWatermark: { + async findUnique() { + return { + lastProcessedBlock: input.lastProcessedBlock ?? 100n, + lastProcessedOperationIndex: input.lastProcessedOperationIndex ?? 2, + lastRunFinishedAt: new Date('2026-09-05T12:02:00.000Z'), + lastRunStartedAt: new Date('2026-09-05T12:00:00.000Z'), + }; + }, + }, + }; +} + +async function createApp(options: Parameters[1]) { + const app = Fastify(); + await registerHealthRoute(app, options); + return app; +} + +describe('health routes', () => { + test('health is a lightweight liveness check', async () => { + const app = await createApp({ + database: createHealthDatabase({ failDatabase: true }) as never, + getHeadBlock: async () => { + throw new Error('head lookup should not run'); + }, + }); + + const response = await app.inject({ + method: 'GET', + url: '/health', + }); + + assert.equal(response.statusCode, 200); + assert.deepEqual(response.json(), { + service: 'hivelore-api', + status: 'ok', + }); + await app.close(); + }); + + test('ready reports database connectivity and indexer lag', async () => { + const app = await createApp({ + database: createHealthDatabase({ + lastProcessedBlock: 100n, + lastProcessedOperationIndex: 3, + }) as never, + getHeadBlock: async () => 105, + indexerLagThresholdBlocks: 10, + indexerName: 'hivelore-haf', + }); + + const response = await app.inject({ + method: 'GET', + url: '/ready', + }); + + assert.equal(response.statusCode, 200); + assert.equal(response.json().status, 'ok'); + assert.equal(response.json().checks.database.status, 'ok'); + assert.equal(response.json().checks.indexer.status, 'ok'); + assert.equal(response.json().checks.indexer.lagBlocks, 5); + assert.equal(response.json().checks.indexer.lastProcessedOperationIndex, 3); + await app.close(); + }); + + test('ready returns degraded when dependencies fail or lag exceeds the threshold', async () => { + const app = await createApp({ + database: createHealthDatabase({ + failDatabase: true, + lastProcessedBlock: 50n, + }) as never, + getHeadBlock: async () => 105, + indexerLagThresholdBlocks: 10, + }); + + const response = await app.inject({ + method: 'GET', + url: '/ready', + }); + + assert.equal(response.statusCode, 503); + assert.equal(response.json().status, 'degraded'); + assert.equal(response.json().checks.database.status, 'degraded'); + assert.equal(response.json().checks.indexer.status, 'degraded'); + assert.equal(response.json().checks.indexer.lagBlocks, 55); + await app.close(); + }); + + test( + 'ready succeeds against a migrated PostgreSQL test database', + { skip: !testDatabaseAdminUrl ? 'TEST_DATABASE_ADMIN_URL is not configured.' : false }, + async () => { + await withDisposablePrismaDatabase('hivelore_health_ready', async (database) => { + await database.indexerWatermark.create({ + data: { + lastProcessedBlock: 90n, + lastProcessedOperationIndex: 4, + lastRunFinishedAt: new Date('2026-09-05T12:01:00.000Z'), + lastRunStartedAt: new Date('2026-09-05T12:00:00.000Z'), + name: 'hivelore-haf', + }, + }); + + const app = await createApp({ + database: database as never, + getHeadBlock: async () => 100, + indexerLagThresholdBlocks: 10, + indexerName: 'hivelore-haf', + }); + const response = await app.inject({ + method: 'GET', + url: '/ready', + }); + + assert.equal(response.statusCode, 200, response.body); + assert.equal(response.json().checks.database.status, 'ok'); + assert.equal(response.json().checks.indexer.lagBlocks, 10); + await app.close(); + }); + }, + ); +}); diff --git a/apps/api/src/routes/health.ts b/apps/api/src/routes/health.ts index 301cb7e..53edfac 100644 --- a/apps/api/src/routes/health.ts +++ b/apps/api/src/routes/health.ts @@ -1,10 +1,134 @@ import type { FastifyInstance } from 'fastify'; -export async function registerHealthRoute(app: FastifyInstance) { +import { env } from '../config/env.js'; +import { prisma } from '../lib/prisma.js'; + +type HealthDatabase = { + $queryRaw(query: TemplateStringsArray, ...values: unknown[]): Promise; + indexerWatermark: { + findUnique(args: { + where: { + name: string; + }; + select: { + lastProcessedBlock: true; + lastProcessedOperationIndex: true; + lastRunFinishedAt: true; + lastRunStartedAt: true; + }; + }): Promise<{ + lastProcessedBlock: bigint; + lastProcessedOperationIndex: number; + lastRunFinishedAt: Date | null; + lastRunStartedAt: Date | null; + } | null>; + }; +}; + +type RegisterHealthRouteOptions = { + database?: HealthDatabase; + getHeadBlock?: () => Promise; + indexerLagThresholdBlocks?: number; + indexerName?: string; +}; + +type CheckStatus = 'ok' | 'degraded'; + +async function checkDatabase(database: HealthDatabase) { + await database.$queryRaw`SELECT 1`; + + return { + status: 'ok' as CheckStatus, + }; +} + +async function checkIndexer( + database: HealthDatabase, + getHeadBlock: () => Promise, + input: { + indexerLagThresholdBlocks: number; + indexerName: string; + }, +) { + const [headBlock, watermark] = await Promise.all([ + getHeadBlock(), + database.indexerWatermark.findUnique({ + select: { + lastProcessedBlock: true, + lastProcessedOperationIndex: true, + lastRunFinishedAt: true, + lastRunStartedAt: true, + }, + where: { + name: input.indexerName, + }, + }), + ]); + const lastProcessedBlock = Number(watermark?.lastProcessedBlock ?? 0n); + const lagBlocks = Math.max(headBlock - lastProcessedBlock, 0); + + return { + headBlock, + lagBlocks, + lastProcessedBlock, + lastProcessedOperationIndex: watermark?.lastProcessedOperationIndex ?? -1, + lastRunFinishedAt: watermark?.lastRunFinishedAt?.toISOString() ?? null, + lastRunStartedAt: watermark?.lastRunStartedAt?.toISOString() ?? null, + status: lagBlocks <= input.indexerLagThresholdBlocks ? ('ok' as CheckStatus) : 'degraded', + thresholdBlocks: input.indexerLagThresholdBlocks, + }; +} + +export async function registerHealthRoute( + app: FastifyInstance, + options: RegisterHealthRouteOptions = {}, +) { + const database = options.database ?? prisma; + const getHeadBlock = options.getHeadBlock ?? (() => Promise.resolve(0)); + const indexerName = options.indexerName ?? env.INDEXER_NAME; + const indexerLagThresholdBlocks = + options.indexerLagThresholdBlocks ?? env.INDEXER_MAX_READY_LAG_BLOCKS; + app.get('/health', async () => { return { status: 'ok', service: 'hivelore-api', }; }); + + app.get('/ready', async (_request, reply) => { + const checks: { + database?: Awaited> | { status: 'degraded'; error: string }; + indexer?: Awaited> | { status: 'degraded'; error: string }; + } = {}; + + try { + checks.database = await checkDatabase(database); + } catch (error) { + checks.database = { + error: error instanceof Error ? error.message : 'Database readiness check failed.', + status: 'degraded', + }; + } + + try { + checks.indexer = await checkIndexer(database, getHeadBlock, { + indexerLagThresholdBlocks, + indexerName, + }); + } catch (error) { + checks.indexer = { + error: error instanceof Error ? error.message : 'Indexer readiness check failed.', + status: 'degraded', + }; + } + + const ready = Object.values(checks).every((check) => check?.status === 'ok'); + + return reply.code(ready ? 200 : 503).send({ + checks, + service: 'hivelore-api', + status: ready ? 'ok' : 'degraded', + }); + }); } diff --git a/apps/api/src/test/integration-db.ts b/apps/api/src/test/integration-db.ts new file mode 100644 index 0000000..7fe3e35 --- /dev/null +++ b/apps/api/src/test/integration-db.ts @@ -0,0 +1,108 @@ +import { spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import process from 'node:process'; + +import { PrismaPg } from '@prisma/adapter-pg'; +import pg from 'pg'; + +import { PrismaClient } from '../generated/prisma/client.js'; + +const { Client } = pg; + +export const testDatabaseAdminUrl = process.env.TEST_DATABASE_ADMIN_URL; + +function disposableDatabaseName(prefix: string) { + return `${prefix}_${Date.now()}_${process.pid}_${randomUUID().replaceAll('-', '').slice(0, 8)}`; +} + +function databaseUrlFor(adminConnectionUrl: string, databaseName: string) { + const databaseUrl = new URL(adminConnectionUrl); + databaseUrl.pathname = `/${databaseName}`; + + return databaseUrl.toString(); +} + +async function withPgClient( + connectionString: string, + callback: (client: pg.Client) => Promise, +) { + const client = new Client({ connectionString }); + await client.connect(); + + try { + return await callback(client); + } finally { + await client.end(); + } +} + +function runPrismaMigrateDeploy(databaseUrl: string) { + const executable = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + const result = spawnSync(executable, ['prisma', 'migrate', 'deploy'], { + cwd: new URL('../..', import.meta.url), + env: { + ...process.env, + DATABASE_URL: databaseUrl, + DIRECT_URL: databaseUrl, + NODE_ENV: 'test', + }, + shell: false, + stdio: 'inherit', + }); + + if (result.status !== 0) { + throw new Error(`prisma migrate deploy failed with exit code ${result.status}`); + } +} + +export function createPrismaClient(connectionString: string) { + return new PrismaClient({ + adapter: new PrismaPg({ connectionString }), + }); +} + +export async function withDisposablePrismaDatabase( + prefix: string, + callback: (database: PrismaClient, databaseUrl: string) => Promise, +) { + if (!testDatabaseAdminUrl) { + throw new Error('TEST_DATABASE_ADMIN_URL is not configured.'); + } + + const databaseName = disposableDatabaseName(prefix); + + if (!databaseName.startsWith(`${prefix}_`)) { + throw new Error(`Refusing to use non-disposable database name: ${databaseName}`); + } + + const directUrl = databaseUrlFor(testDatabaseAdminUrl, databaseName); + + await withPgClient(testDatabaseAdminUrl, (client) => + client.query(`CREATE DATABASE "${databaseName}"`), + ); + + try { + runPrismaMigrateDeploy(directUrl); + + const database = createPrismaClient(directUrl); + + try { + return await callback(database, directUrl); + } finally { + await database.$disconnect(); + } + } finally { + await withPgClient(testDatabaseAdminUrl, async (client) => { + await client.query( + ` + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = $1 + AND pid <> pg_backend_pid() + `, + [databaseName], + ); + await client.query(`DROP DATABASE IF EXISTS "${databaseName}"`); + }); + } +} From e3585c206ae2986b3ecc6383848d0a64841404c0 Mon Sep 17 00:00:00 2001 From: Mohammad Farhat Date: Sun, 6 Sep 2026 10:14:31 +0000 Subject: [PATCH 2/2] fix(observability): stop a stalled webhook from hanging every failing request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fastify awaits the onError hook, and reportUnhandledError awaited a fetch with no timeout. A webhook that accepts the connection and never replies therefore kept every erroring request open until OS-level TCP timeouts — minutes. Proven against a deliberately unresponsive server: the call was still pending after 8s. Observability taking the service down with it is the worst version of this, and it would have bitten on the first day ERROR_TRACKING_ENABLED was turned on. - bound delivery with AbortSignal.timeout, default 2s, overridable per call - stop awaiting delivery in the onError hook, so the error response never waits on telemetry at all (reportUnhandledError swallows its own errors, so the floating promise cannot reject) - add a regression test using a server that accepts and never replies; it hangs against the unbounded version and passes in 150ms against this one Co-Authored-By: Claude Opus 5 --- apps/api/src/app.ts | 6 +++++- apps/api/src/lib/error-tracking.test.ts | 28 +++++++++++++++++++++++++ apps/api/src/lib/error-tracking.ts | 10 +++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 61158b0..85bc8ec 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -44,7 +44,11 @@ export async function buildApp() { }); app.addHook('onError', async (request, _reply, error) => { - await reportUnhandledError( + // Deliberately NOT awaited: Fastify waits for this hook before sending the + // error response, so awaiting webhook delivery would let a slow tracker + // delay - or with no timeout, indefinitely stall - every failing request. + // reportUnhandledError swallows its own errors, so this cannot reject. + void reportUnhandledError( { error, method: request.method, diff --git a/apps/api/src/lib/error-tracking.test.ts b/apps/api/src/lib/error-tracking.test.ts index 37f293a..3fa3e99 100644 --- a/apps/api/src/lib/error-tracking.test.ts +++ b/apps/api/src/lib/error-tracking.test.ts @@ -1,3 +1,5 @@ +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; @@ -36,4 +38,30 @@ describe('error tracking', () => { assert.equal(logs.length, 1); assert.equal((logs[0] as { requestId?: string }).requestId, 'request-1'); }); + + test('gives up on an unresponsive webhook instead of hanging the error path', async () => { + // Regression guard. Fastify awaits the onError hook, so an unbounded fetch + // to a stalled webhook would keep every failing request open until OS-level + // TCP timeouts. Against the unbounded version this test never resolves. + const stalled = createServer(() => { + // accept the connection, never reply + }); + await new Promise((resolve) => stalled.listen(0, resolve)); + const address = stalled.address() as AddressInfo; + + const startedAt = Date.now(); + await reportUnhandledError( + { error: new Error('boom'), method: 'GET', requestId: 'req-stall', url: '/x' }, + { + enabled: true, + timeoutMs: 150, + webhookUrl: `http://127.0.0.1:${address.port}/hook`, + }, + ); + const elapsed = Date.now() - startedAt; + + stalled.close(); + + assert.ok(elapsed < 2_000, `expected the webhook to be abandoned quickly, took ${elapsed}ms`); + }); }); diff --git a/apps/api/src/lib/error-tracking.ts b/apps/api/src/lib/error-tracking.ts index 56dc3e8..1335c2f 100644 --- a/apps/api/src/lib/error-tracking.ts +++ b/apps/api/src/lib/error-tracking.ts @@ -10,9 +10,16 @@ type ErrorTrackingEvent = { type ErrorTrackerOptions = { enabled?: boolean; logger?: FastifyBaseLogger; + timeoutMs?: number; webhookUrl?: string | undefined; }; +// Telemetry must never outlive the request it describes. Without a bound, a +// webhook that accepts the connection and never replies keeps this promise +// pending until OS-level TCP timeouts - minutes - and Fastify's onError hook +// would wait on it. +const DEFAULT_WEBHOOK_TIMEOUT_MS = 2_000; + export function serializeError(error: unknown) { if (error instanceof Error) { return { @@ -47,6 +54,8 @@ export async function reportUnhandledError( return; } + const timeoutMs = options.timeoutMs ?? DEFAULT_WEBHOOK_TIMEOUT_MS; + try { await fetch(options.webhookUrl, { body: JSON.stringify(payload), @@ -54,6 +63,7 @@ export async function reportUnhandledError( 'content-type': 'application/json', }, method: 'POST', + signal: AbortSignal.timeout(timeoutMs), }); } catch (error) { options.logger?.warn(