diff --git a/scripts/ci/production-diagnostics.mjs b/scripts/ci/production-diagnostics.mjs new file mode 100644 index 0000000..1fcbdad --- /dev/null +++ b/scripts/ci/production-diagnostics.mjs @@ -0,0 +1,215 @@ +import { spawn } from 'node:child_process'; + +const BODY_BYTE_LIMIT = 2048; +const BODY_READ_TIMEOUT_MS = 2_000; +const RESPONSE_METADATA_NAMES = ['content-type', 'retry-after']; + +const JWT_SHAPE = /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g; + +export function processSnapshot(child) { + return { + pid: child?.pid ?? null, + exitCode: child?.exitCode ?? null, + signal: child?.signalCode ?? null, + }; +} + +function selectedResponseMetadata(headers) { + const selected = {}; + if (!headers) return selected; + for (const name of RESPONSE_METADATA_NAMES) { + const value = + typeof headers.get === 'function' ? headers.get(name) : headers[name]; + if (value) selected[name] = String(value); + } + return selected; +} + +function redactAuthenticationJwts(text) { + return String(text ?? '').replace(JWT_SHAPE, '[redacted-jwt]'); +} + +export async function readBoundedBody( + response, + { byteLimit = BODY_BYTE_LIMIT, timeoutMs = BODY_READ_TIMEOUT_MS } = {}, +) { + const empty = { text: '', bytes: 0, truncated: false, timedOut: false }; + if (!response?.body || typeof response.body.getReader !== 'function') { + return empty; + } + const reader = response.body.getReader(); + const chunks = []; + let bytes = 0; + let truncated = false; + let timedOut = false; + let timeoutId; + const expired = new Promise((resolve) => { + timeoutId = setTimeout(() => { + timedOut = true; + resolve(); + }, timeoutMs); + }); + try { + while (bytes < byteLimit && !timedOut) { + const outcome = await Promise.race([ + reader.read().then((chunk) => ({ kind: 'chunk', chunk })), + expired.then(() => ({ kind: 'timeout' })), + ]); + if (outcome.kind === 'timeout') { + timedOut = true; + break; + } + const { done, value } = outcome.chunk; + if (done) break; + if (!value) continue; + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + const room = byteLimit - bytes; + if (chunk.byteLength > room) { + chunks.push(chunk.subarray(0, room)); + bytes += room; + truncated = true; + break; + } + chunks.push(chunk); + bytes += chunk.byteLength; + } + } finally { + const cancel = Promise.resolve(reader.cancel()).catch(() => {}); + await Promise.race([cancel, expired]); + clearTimeout(timeoutId); + } + return { + text: Buffer.concat(chunks.map((part) => Buffer.from(part))).toString( + 'utf8', + ), + bytes, + truncated, + timedOut, + }; +} + +export function describeUnexpectedResponse({ + operation, + status, + body, + truncated = false, + timedOut = false, + bodyBytes, + headers, + processState, +}) { + const meta = selectedResponseMetadata(headers); + const state = processState ?? {}; + const safeBody = redactAuthenticationJwts(body); + const flags = `${truncated ? ' truncated' : ''}${timedOut ? ' timed-out' : ''}`; + const lines = [ + '=== unexpected response ===', + `operation: ${operation}`, + `status: ${status}`, + ]; + for (const name of RESPONSE_METADATA_NAMES) { + if (meta[name]) lines.push(`${name}: ${meta[name]}`); + } + lines.push( + `body-bytes: ${bodyBytes ?? Buffer.byteLength(String(body ?? ''))}${flags}`, + ); + lines.push('body:'); + lines.push(safeBody); + lines.push( + `process: pid=${state.pid ?? null} exit=${state.exitCode ?? null} signal=${state.signal ?? null}`, + ); + lines.push('=== end unexpected response ==='); + return `${lines.join('\n')}\n`; +} + +function stdioOpen(server) { + return [server?.stdout, server?.stderr].some( + (stream) => stream && !stream.destroyed && !stream.readableEnded, + ); +} + +function waitForClose(server, timeoutMs) { + if (!server) return Promise.resolve(); + if ( + !stdioOpen(server) && + (server.exitCode !== null || server.signalCode !== null) + ) { + return Promise.resolve(); + } + return new Promise((accept) => { + let settled = false; + const done = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + accept(); + }; + const timer = setTimeout(done, timeoutMs); + server.once('close', done); + if ( + !stdioOpen(server) && + (server.exitCode !== null || server.signalCode !== null) + ) { + done(); + } + }); +} + +function groupAlive(server) { + if (!server?.pid) return false; + if (process.platform === 'win32') { + return server.exitCode === null && server.signalCode === null; + } + try { + process.kill(-server.pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + +async function signalGroup(server, posixSignal) { + if (!server?.pid) return; + if (process.platform === 'win32') { + await new Promise((accept) => { + const stop = spawn('taskkill', ['/pid', String(server.pid), '/T', '/F'], { + windowsHide: true, + stdio: 'ignore', + }); + stop.on('exit', accept); + stop.on('error', accept); + }); + return; + } + try { + process.kill(-server.pid, posixSignal); + } catch (error) { + if (error.code !== 'ESRCH') { + console.error('Unable to stop test worker:', error); + process.exitCode = 1; + } + } +} + +export async function finishProductionServer( + server, + log, + { timeoutMs = 5_000 } = {}, +) { + const closed = waitForClose(server, timeoutMs); + await signalGroup(server, 'SIGTERM'); + await closed; + if (groupAlive(server)) { + const last = waitForClose(server, timeoutMs); + await signalGroup(server, 'SIGKILL'); + await last; + } + if (log && !log.writableEnded) { + await new Promise((accept, reject) => { + log.once('finish', accept); + log.once('error', reject); + log.end(); + }); + } +} diff --git a/scripts/ci/run-production.mjs b/scripts/ci/run-production.mjs index f80bb3f..5f6e767 100644 --- a/scripts/ci/run-production.mjs +++ b/scripts/ci/run-production.mjs @@ -7,6 +7,12 @@ import { resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { exportJWK, generateKeyPair, SignJWT } from 'jose'; import { releaseConfig } from '../release/config.mjs'; +import { + describeUnexpectedResponse, + finishProductionServer, + processSnapshot, + readBoundedBody, +} from './production-diagnostics.mjs'; // Only the test entry supplies a local verification key. The compiled app, // authentication handler, assets and SQL migrations are the release versions. @@ -123,6 +129,25 @@ let startupError; server.on('error', (error) => { startupError = error; }); +async function expectStatus(response, expected, operation) { + if (response.status === expected) return response; + const body = await readBoundedBody(response); + const detail = describeUnexpectedResponse({ + operation, + status: response.status, + body: body.text, + truncated: body.truncated, + timedOut: body.timedOut, + bodyBytes: body.bytes, + headers: response.headers, + processState: processSnapshot(server), + }); + await new Promise((accept, reject) => { + log.write(detail, (error) => (error ? reject(error) : accept())); + }); + assert.equal(response.status, expected, detail); + return response; +} try { const deadline = Date.now() + 120_000; @@ -163,34 +188,40 @@ try { }, ...(body ? { body: JSON.stringify(body) } : {}), }); - assert.equal( - ( - await call(null, undefined, { - 'oai-authenticated-user-id': 'cloudflare:owner-a', - }) - ).status, + await expectStatus( + await call(null, undefined, { + 'oai-authenticated-user-id': 'cloudflare:owner-a', + }), 401, + 'spoofed identity header without Access token', ); const htmlResponse = await fetch(base, { headers: { 'Cf-Access-Jwt-Assertion': first }, }); - assert.equal(htmlResponse.status, 200, await htmlResponse.clone().text()); + await expectStatus(htmlResponse, 200, 'authenticated document'); const html = await htmlResponse.text(); assert.match(html, /Make your next move/); const scriptPath = html.match(/]+src="([^"]+\.js[^"]*)"/)?.[1]; assert.ok(scriptPath, 'Compiled page must load its client JavaScript'); - assert.equal( - (await fetch(new URL(scriptPath, base))).status, + await expectStatus( + await fetch(new URL(scriptPath, base)), 401, - 'Assets must pass through authentication', + 'unauthenticated asset', + ); + const script = await expectStatus( + await fetch(new URL(scriptPath, base), { + headers: { 'Cf-Access-Jwt-Assertion': first }, + }), + 200, + 'authenticated asset', ); - const script = await fetch(new URL(scriptPath, base), { - headers: { 'Cf-Access-Jwt-Assertion': first }, - }); - assert.equal(script.status, 200); assert.match(script.headers.get('content-type'), /javascript/); assert.deepEqual((await (await call(first)).json()).jobs, []); - assert.equal((await call(first, { action: 'bootstrap' })).status, 200); + await expectStatus( + await call(first, { action: 'bootstrap' }), + 200, + 'owner-a bootstrap', + ); const firstWorkspace = await (await call(first)).json(); assert.equal(firstWorkspace.jobs.length, 3); assert.deepEqual( @@ -209,23 +240,18 @@ try { draft: 'A fictional draft verified through the built application.', blocker: '', }; - assert.equal( - (await call(second, update)).status, + await expectStatus( + await call(second, update), 404, - 'Second owner cannot mutate the first owner record', + 'owner-b save of owner-a record', ); - assert.equal( - (await call(first, update, { origin: 'https://untrusted.example.com' })) - .status, + await expectStatus( + await call(first, update, { origin: 'https://untrusted.example.com' }), 403, + 'owner-a save with untrusted origin', ); - assert.equal((await call(first, update)).status, 200); - const stale = await call(first, update); - assert.equal( - stale.status, - 409, - `Stale writes must fail in the actual worker: ${await stale.text()}`, - ); + await expectStatus(await call(first, update), 200, 'owner-a save'); + await expectStatus(await call(first, update), 409, 'owner-a stale save'); const saved = await ( await call(first, undefined, { 'oai-authenticated-user-id': 'cloudflare:owner-b', @@ -248,27 +274,5 @@ try { 'PASS: built app rendering/assets, signed identity, persistence, tenant isolation, forged headers, request origin, exact acceptance and stale-write integrity.', ); } finally { - if (server.pid && server.exitCode === null) { - if (process.platform === 'win32') { - await new Promise((accept) => { - const stop = spawn( - 'taskkill', - ['/pid', String(server.pid), '/T', '/F'], - { windowsHide: true, stdio: 'ignore' }, - ); - stop.on('exit', accept); - stop.on('error', accept); - }); - } else { - try { - process.kill(-server.pid, 'SIGTERM'); - } catch (error) { - if (error.code !== 'ESRCH') { - console.error('Unable to stop test worker:', error); - process.exitCode = 1; - } - } - } - } - log.end(); + await finishProductionServer(server, log); } diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs new file mode 100644 index 0000000..15254c2 --- /dev/null +++ b/tests/production-diagnostics.test.mjs @@ -0,0 +1,379 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { createWriteStream, readFileSync } from 'node:fs'; +import { mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { Writable } from 'node:stream'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + describeUnexpectedResponse, + finishProductionServer, + processSnapshot, + readBoundedBody, +} from '../scripts/ci/production-diagnostics.mjs'; + +const fictionalJwt = + 'eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJvd25lci1hIn0.fictional'; + +function headers(extra = {}) { + return new Headers({ + 'content-type': 'text/plain', + 'retry-after': '2', + authorization: `Bearer ${fictionalJwt}`, + 'cf-access-jwt-assertion': fictionalJwt, + 'set-cookie': 'session=secret', + ...extra, + }); +} + +void test('unexpected response keeps selected metadata and omits request authentication', () => { + const detail = describeUnexpectedResponse({ + operation: 'owner-a save', + status: 503, + body: `Worker failed\n${fictionalJwt}`, + truncated: false, + timedOut: false, + bodyBytes: 32, + headers: headers(), + processState: { pid: 4242, exitCode: null, signal: null }, + }); + assert.match(detail, /operation: owner-a save/); + assert.match(detail, /status: 503/); + assert.match(detail, /content-type: text\/plain/); + assert.match(detail, /retry-after: 2/); + assert.match(detail, /body-bytes: 32/); + assert.match(detail, /process: pid=4242 exit=null signal=null/); + assert.doesNotMatch(detail, /authorization/i); + assert.doesNotMatch(detail, /cf-access-jwt-assertion/i); + assert.doesNotMatch(detail, /set-cookie/i); + assert.doesNotMatch(detail, /Bearer /); + assert.doesNotMatch(detail, /eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9/); + assert.match(detail, /\[redacted-jwt\]/); +}); + +void test('503 body mentioning restart is copied, not interpreted as a process restart', () => { + const detail = describeUnexpectedResponse({ + operation: 'owner-a save', + status: 503, + body: 'Detected a change, restarting local server...', + headers: new Headers({ 'content-type': 'text/html' }), + processState: processSnapshot({ pid: 7, exitCode: null, signalCode: null }), + }); + assert.match(detail, /Detected a change, restarting local server/); + assert.doesNotMatch(detail, /^restart:/m); + assert.doesNotMatch(detail, /process restarted/i); + assert.match(detail, /exit=null signal=null/); +}); + +void test('body read stops at the byte limit', async () => { + const response = new Response('n'.repeat(5000), { + status: 503, + headers: { 'content-type': 'text/plain' }, + }); + const body = await readBoundedBody(response, { + byteLimit: 64, + timeoutMs: 500, + }); + assert.equal(body.truncated, true); + assert.equal(body.timedOut, false); + assert.equal(body.bytes, 64); + assert.equal(Buffer.byteLength(body.text), 64); +}); + +void test('body read stops at the deadline instead of hanging', async () => { + const body = new ReadableStream({ + pull() { + return new Promise(() => {}); + }, + }); + const response = new Response(body, { status: 503 }); + const started = Date.now(); + const result = await readBoundedBody(response, { + byteLimit: 2048, + timeoutMs: 75, + }); + assert.ok(Date.now() - started < 500); + assert.equal(result.timedOut, true); +}); + +function timeoutHandles() { + return process.getActiveResourcesInfo().filter((name) => name === 'Timeout') + .length; +} + +void test('pending cancel cannot exceed the overall deadline', async () => { + const response = new Response( + new ReadableStream({ + pull() { + return new Promise(() => {}); + }, + cancel() { + return new Promise(() => {}); + }, + }), + { status: 503 }, + ); + const result = await Promise.race([ + readBoundedBody(response, { timeoutMs: 25 }).then((body) => ({ + kind: 'returned', + body, + })), + delay(150).then(() => ({ kind: 'HUNG' })), + ]); + assert.equal(result.kind, 'returned'); + assert.equal(result.body.timedOut, true); +}); + +void test('chunked body does not leave dangling deadline timers', async () => { + const before = timeoutHandles(); + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < 20; i++) controller.enqueue(new Uint8Array([120])); + controller.close(); + }, + }); + const body = await readBoundedBody(new Response(stream, { status: 503 }), { + byteLimit: 2048, + timeoutMs: 500, + }); + assert.equal(body.bytes, 20); + assert.equal(body.text, 'x'.repeat(20)); + assert.equal(body.timedOut, false); + assert.equal(timeoutHandles(), before); +}); + +void test('detached process-group teardown captures post-SIGTERM output and reaps nested children', async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-log-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const nested = ` + process.on('SIGTERM', () => { + process.stdout.write('after-term\\n', () => process.exit(0)); + }); + const nested = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + windowsHide: true, + }); + process.stdout.write('grandchild=' + nested.pid + '\\n'); + setInterval(() => {}, 1000); + `; + const child = spawn( + process.execPath, + ['-e', `import { spawn } from 'node:child_process';${nested}`], + { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + windowsHide: true, + }, + ); + let seen = ''; + child.stdout.on('data', (chunk) => { + seen += chunk.toString(); + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + const deadline = Date.now() + 2_000; + while (!seen.includes('grandchild=') && Date.now() < deadline) + await delay(10); + const grandchild = Number(seen.match(/grandchild=(\d+)/)?.[1]); + assert.ok(grandchild, `nested child pid missing: ${seen}`); + try { + await finishProductionServer(child, log, { timeoutMs: 1_000 }); + const text = await readFile(logPath, 'utf8'); + if (process.platform !== 'win32') assert.match(text, /after-term/); + assert.throws(() => process.kill(grandchild, 0), { code: 'ESRCH' }); + } finally { + const pid = child.pid; + if (typeof pid === 'number') { + try { + if (process.platform === 'win32') { + spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { + windowsHide: true, + stdio: 'ignore', + }); + } else { + process.kill(-pid, 'SIGKILL'); + } + } catch { + /* Best-effort cleanup after the assertion. */ + } + } + try { + process.kill(grandchild, 'SIGKILL'); + } catch { + /* Nested child already reaped. */ + } + } +}); + +void test('teardown still finishes when the leader ignores SIGTERM', async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-kill-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const child = spawn( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stdout.write("ready\\n"); setInterval(() => {}, 1000);', + ], + { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + windowsHide: true, + }, + ); + let seen = ''; + child.stdout.on('data', (chunk) => { + seen += chunk.toString(); + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + const readyUntil = Date.now() + 2_000; + while (!seen.includes('ready') && Date.now() < readyUntil) await delay(10); + assert.match(seen, /ready/); + const started = Date.now(); + try { + await finishProductionServer(child, log, { timeoutMs: 200 }); + if (process.platform !== 'win32') assert.ok(Date.now() - started >= 150); + assert.ok(Date.now() - started < 2_000); + const pid = child.pid; + assert.ok(typeof pid === 'number'); + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }); + await readFile(logPath); + } finally { + const pid = child.pid; + if (typeof pid === 'number') { + try { + process.kill(process.platform === 'win32' ? pid : -pid, 'SIGKILL'); + } catch { + /* Best-effort cleanup after the assertion. */ + } + } + } +}); + +function posixTerminated(pid) { + try { + process.kill(pid, 0); + } catch (error) { + if (error.code === 'ESRCH') return true; + throw error; + } + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + return stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z'); + } catch { + return true; + } +} + +void test( + 'exited group leader does not skip TERM/KILL of a TERM-ignoring descendant', + { skip: process.platform === 'win32' }, + async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-group-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const descendant = + "process.on('SIGTERM',()=>{process.stdout.write('GRANDCHILD_TERM_STDOUT\\n');process.stderr.write('GRANDCHILD_TERM_STDERR\\n');});setInterval(()=>{},1000);process.send('ready');"; + const leader = + "const {spawn}=require('node:child_process'); const c=spawn(process.execPath,['-e'," + + JSON.stringify(descendant) + + "],{stdio:['ignore','inherit','inherit','ipc']});c.once('message',()=>process.stdout.write('grandchild='+c.pid+'\\n',()=>process.exit(0)));"; + const child = spawn(process.execPath, ['-e', leader], { + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let output = ''; + let closed = false; + const closePromise = once(child, 'close').then(() => { + closed = true; + }); + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let grandchild; + try { + await once(child, 'exit'); + grandchild = Number(output.match(/grandchild=(\d+)/)?.[1]); + assert.ok(grandchild, `nested child pid missing: ${output}`); + assert.equal(child.exitCode, 0); + assert.equal(closed, false); + await finishProductionServer(child, log, { timeoutMs: 100 }); + assert.equal(closed, true); + const text = await readFile(logPath, 'utf8'); + assert.match(text, /GRANDCHILD_TERM_STDOUT/); + assert.match(text, /GRANDCHILD_TERM_STDERR/); + assert.equal(posixTerminated(grandchild), true); + } finally { + const pid = child.pid; + if (typeof pid === 'number') { + try { + process.kill(-pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } + } + await closePromise; + if (!log.writableEnded) log.end(); + } + }, +); + +void test( + 'TERM-ignoring process is KILLed after it closes stdio', + { skip: process.platform === 'win32' }, + async () => { + const child = spawn( + process.execPath, + [ + '-e', + "process.on('SIGTERM',()=>{});setInterval(()=>{},1000);process.stdout.end(()=>process.stderr.end(()=>process.send('ready')));", + ], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + windowsHide: true, + }, + ); + const closed = once(child, 'close'); + const log = new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + try { + await Promise.all([ + once(child, 'message'), + once(child.stdout, 'end'), + once(child.stderr, 'end'), + ]); + assert.equal(child.stdout.readableEnded, true); + assert.equal(child.stderr.readableEnded, true); + assert.equal(child.exitCode, null); + await finishProductionServer(child, log, { timeoutMs: 100 }); + const pid = child.pid; + assert.ok(typeof pid === 'number'); + assert.equal(posixTerminated(pid), true); + } finally { + const pid = child.pid; + if (typeof pid === 'number') { + try { + process.kill(-pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } + } + await closed; + if (!log.writableEnded) log.end(); + } + }, +);