From 10a962e5c414b2839626e6d4b4079000ffabf4c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 00:55:25 +0000 Subject: [PATCH 1/6] Record bounded unexpected responses in production CI Failure-only diagnostics capture status, selected headers, a deadline-bounded body, and process exit/signal without printing request headers or JWTs. POSIX teardown still signals the detached process group; Windows keeps taskkill /T. Related to #22. Co-authored-by: Seth Carlson --- scripts/ci/production-diagnostics.mjs | 190 +++++++++++++++++++++++ scripts/ci/run-production.mjs | 106 ++++++------- tests/production-diagnostics.test.mjs | 208 ++++++++++++++++++++++++++ 3 files changed, 453 insertions(+), 51 deletions(-) create mode 100644 scripts/ci/production-diagnostics.mjs create mode 100644 tests/production-diagnostics.test.mjs diff --git a/scripts/ci/production-diagnostics.mjs b/scripts/ci/production-diagnostics.mjs new file mode 100644 index 0000000..4b63657 --- /dev/null +++ b/scripts/ci/production-diagnostics.mjs @@ -0,0 +1,190 @@ +import { spawn } from 'node:child_process'; +import { setTimeout as delay } from 'node:timers/promises'; + +const BODY_BYTE_LIMIT = 2048; +const BODY_READ_TIMEOUT_MS = 2_000; +const RESPONSE_METADATA_NAMES = ['content-type', 'retry-after']; + +const JWT_SHAPE = /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g; + +export function processSnapshot(child) { + return { + pid: child?.pid ?? null, + exitCode: child?.exitCode ?? null, + signal: child?.signalCode ?? null, + }; +} + +function selectedResponseMetadata(headers) { + const selected = {}; + if (!headers) return selected; + for (const name of RESPONSE_METADATA_NAMES) { + const value = + typeof headers.get === 'function' ? headers.get(name) : headers[name]; + if (value) selected[name] = String(value); + } + return selected; +} + +function redactAuthenticationJwts(text) { + return String(text ?? '').replace(JWT_SHAPE, '[redacted-jwt]'); +} + +export async function readBoundedBody( + response, + { byteLimit = BODY_BYTE_LIMIT, timeoutMs = BODY_READ_TIMEOUT_MS } = {}, +) { + const empty = { text: '', bytes: 0, truncated: false, timedOut: false }; + if (!response?.body || typeof response.body.getReader !== 'function') { + return empty; + } + const reader = response.body.getReader(); + const chunks = []; + let bytes = 0; + let truncated = false; + let timedOut = false; + const deadline = Date.now() + timeoutMs; + try { + while (bytes < byteLimit) { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + timedOut = true; + break; + } + const outcome = await Promise.race([ + reader.read().then((chunk) => ({ kind: 'chunk', chunk })), + delay(remaining).then(() => ({ kind: 'timeout' })), + ]); + if (outcome.kind === 'timeout') { + timedOut = true; + break; + } + const { done, value } = outcome.chunk; + if (done) break; + if (!value) continue; + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + const room = byteLimit - bytes; + if (chunk.byteLength > room) { + chunks.push(chunk.subarray(0, room)); + bytes += room; + truncated = true; + break; + } + chunks.push(chunk); + bytes += chunk.byteLength; + } + } finally { + try { + await reader.cancel(); + } catch { + /* The stream may already be closed. */ + } + } + return { + text: Buffer.concat(chunks.map((part) => Buffer.from(part))).toString( + 'utf8', + ), + bytes, + truncated, + timedOut, + }; +} + +export function describeUnexpectedResponse({ + operation, + status, + body, + truncated = false, + timedOut = false, + bodyBytes, + headers, + processState, +}) { + const meta = selectedResponseMetadata(headers); + const state = processState ?? {}; + const safeBody = redactAuthenticationJwts(body); + const flags = `${truncated ? ' truncated' : ''}${timedOut ? ' timed-out' : ''}`; + const lines = [ + '=== unexpected response ===', + `operation: ${operation}`, + `status: ${status}`, + ]; + for (const name of RESPONSE_METADATA_NAMES) { + if (meta[name]) lines.push(`${name}: ${meta[name]}`); + } + lines.push( + `body-bytes: ${bodyBytes ?? Buffer.byteLength(String(body ?? ''))}${flags}`, + ); + lines.push('body:'); + lines.push(safeBody); + lines.push( + `process: pid=${state.pid ?? null} exit=${state.exitCode ?? null} signal=${state.signal ?? null}`, + ); + lines.push('=== end unexpected response ==='); + return `${lines.join('\n')}\n`; +} + +function hasStopped(server) { + return !server || server.exitCode !== null || server.signalCode !== null; +} + +function waitForClose(server, timeoutMs) { + if (hasStopped(server)) return Promise.resolve(); + return new Promise((accept) => { + const timer = setTimeout(accept, timeoutMs); + let settled = false; + const done = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + accept(); + }; + server.once('close', done); + if (hasStopped(server)) done(); + }); +} + +async function signalGroup(server, posixSignal) { + if (!server?.pid || hasStopped(server)) return; + if (process.platform === 'win32') { + await new Promise((accept) => { + const stop = spawn('taskkill', ['/pid', String(server.pid), '/T', '/F'], { + windowsHide: true, + stdio: 'ignore', + }); + stop.on('exit', accept); + stop.on('error', accept); + }); + return; + } + try { + process.kill(-server.pid, posixSignal); + } catch (error) { + if (error.code !== 'ESRCH') { + console.error('Unable to stop test worker:', error); + process.exitCode = 1; + } + } +} + +export async function finishProductionServer( + server, + log, + { timeoutMs = 5_000 } = {}, +) { + const closed = waitForClose(server, timeoutMs); + await signalGroup(server, 'SIGTERM'); + await closed; + if (server?.pid && !hasStopped(server)) { + const last = waitForClose(server, timeoutMs); + await signalGroup(server, 'SIGKILL'); + await last; + } + if (log && !log.writableEnded) { + await new Promise((accept, reject) => { + log.once('finish', accept); + log.once('error', reject); + log.end(); + }); + } +} diff --git a/scripts/ci/run-production.mjs b/scripts/ci/run-production.mjs index f80bb3f..5f6e767 100644 --- a/scripts/ci/run-production.mjs +++ b/scripts/ci/run-production.mjs @@ -7,6 +7,12 @@ import { resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { exportJWK, generateKeyPair, SignJWT } from 'jose'; import { releaseConfig } from '../release/config.mjs'; +import { + describeUnexpectedResponse, + finishProductionServer, + processSnapshot, + readBoundedBody, +} from './production-diagnostics.mjs'; // Only the test entry supplies a local verification key. The compiled app, // authentication handler, assets and SQL migrations are the release versions. @@ -123,6 +129,25 @@ let startupError; server.on('error', (error) => { startupError = error; }); +async function expectStatus(response, expected, operation) { + if (response.status === expected) return response; + const body = await readBoundedBody(response); + const detail = describeUnexpectedResponse({ + operation, + status: response.status, + body: body.text, + truncated: body.truncated, + timedOut: body.timedOut, + bodyBytes: body.bytes, + headers: response.headers, + processState: processSnapshot(server), + }); + await new Promise((accept, reject) => { + log.write(detail, (error) => (error ? reject(error) : accept())); + }); + assert.equal(response.status, expected, detail); + return response; +} try { const deadline = Date.now() + 120_000; @@ -163,34 +188,40 @@ try { }, ...(body ? { body: JSON.stringify(body) } : {}), }); - assert.equal( - ( - await call(null, undefined, { - 'oai-authenticated-user-id': 'cloudflare:owner-a', - }) - ).status, + await expectStatus( + await call(null, undefined, { + 'oai-authenticated-user-id': 'cloudflare:owner-a', + }), 401, + 'spoofed identity header without Access token', ); const htmlResponse = await fetch(base, { headers: { 'Cf-Access-Jwt-Assertion': first }, }); - assert.equal(htmlResponse.status, 200, await htmlResponse.clone().text()); + await expectStatus(htmlResponse, 200, 'authenticated document'); const html = await htmlResponse.text(); assert.match(html, /Make your next move/); const scriptPath = html.match(/]+src="([^"]+\.js[^"]*)"/)?.[1]; assert.ok(scriptPath, 'Compiled page must load its client JavaScript'); - assert.equal( - (await fetch(new URL(scriptPath, base))).status, + await expectStatus( + await fetch(new URL(scriptPath, base)), 401, - 'Assets must pass through authentication', + 'unauthenticated asset', + ); + const script = await expectStatus( + await fetch(new URL(scriptPath, base), { + headers: { 'Cf-Access-Jwt-Assertion': first }, + }), + 200, + 'authenticated asset', ); - const script = await fetch(new URL(scriptPath, base), { - headers: { 'Cf-Access-Jwt-Assertion': first }, - }); - assert.equal(script.status, 200); assert.match(script.headers.get('content-type'), /javascript/); assert.deepEqual((await (await call(first)).json()).jobs, []); - assert.equal((await call(first, { action: 'bootstrap' })).status, 200); + await expectStatus( + await call(first, { action: 'bootstrap' }), + 200, + 'owner-a bootstrap', + ); const firstWorkspace = await (await call(first)).json(); assert.equal(firstWorkspace.jobs.length, 3); assert.deepEqual( @@ -209,23 +240,18 @@ try { draft: 'A fictional draft verified through the built application.', blocker: '', }; - assert.equal( - (await call(second, update)).status, + await expectStatus( + await call(second, update), 404, - 'Second owner cannot mutate the first owner record', + 'owner-b save of owner-a record', ); - assert.equal( - (await call(first, update, { origin: 'https://untrusted.example.com' })) - .status, + await expectStatus( + await call(first, update, { origin: 'https://untrusted.example.com' }), 403, + 'owner-a save with untrusted origin', ); - assert.equal((await call(first, update)).status, 200); - const stale = await call(first, update); - assert.equal( - stale.status, - 409, - `Stale writes must fail in the actual worker: ${await stale.text()}`, - ); + await expectStatus(await call(first, update), 200, 'owner-a save'); + await expectStatus(await call(first, update), 409, 'owner-a stale save'); const saved = await ( await call(first, undefined, { 'oai-authenticated-user-id': 'cloudflare:owner-b', @@ -248,27 +274,5 @@ try { 'PASS: built app rendering/assets, signed identity, persistence, tenant isolation, forged headers, request origin, exact acceptance and stale-write integrity.', ); } finally { - if (server.pid && server.exitCode === null) { - if (process.platform === 'win32') { - await new Promise((accept) => { - const stop = spawn( - 'taskkill', - ['/pid', String(server.pid), '/T', '/F'], - { windowsHide: true, stdio: 'ignore' }, - ); - stop.on('exit', accept); - stop.on('error', accept); - }); - } else { - try { - process.kill(-server.pid, 'SIGTERM'); - } catch (error) { - if (error.code !== 'ESRCH') { - console.error('Unable to stop test worker:', error); - process.exitCode = 1; - } - } - } - } - log.end(); + await finishProductionServer(server, log); } diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs new file mode 100644 index 0000000..bbc0e61 --- /dev/null +++ b/tests/production-diagnostics.test.mjs @@ -0,0 +1,208 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createWriteStream } from 'node:fs'; +import { mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + describeUnexpectedResponse, + finishProductionServer, + processSnapshot, + readBoundedBody, +} from '../scripts/ci/production-diagnostics.mjs'; + +const fictionalJwt = + 'eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJvd25lci1hIn0.fictional'; + +function headers(extra = {}) { + return new Headers({ + 'content-type': 'text/plain', + 'retry-after': '2', + authorization: `Bearer ${fictionalJwt}`, + 'cf-access-jwt-assertion': fictionalJwt, + 'set-cookie': 'session=secret', + ...extra, + }); +} + +test('unexpected response keeps selected metadata and omits request authentication', () => { + const detail = describeUnexpectedResponse({ + operation: 'owner-a save', + status: 503, + body: `Worker failed\n${fictionalJwt}`, + truncated: false, + timedOut: false, + bodyBytes: 32, + headers: headers(), + processState: { pid: 4242, exitCode: null, signal: null }, + }); + assert.match(detail, /operation: owner-a save/); + assert.match(detail, /status: 503/); + assert.match(detail, /content-type: text\/plain/); + assert.match(detail, /retry-after: 2/); + assert.match(detail, /body-bytes: 32/); + assert.match(detail, /process: pid=4242 exit=null signal=null/); + assert.doesNotMatch(detail, /authorization/i); + assert.doesNotMatch(detail, /cf-access-jwt-assertion/i); + assert.doesNotMatch(detail, /set-cookie/i); + assert.doesNotMatch(detail, /Bearer /); + assert.doesNotMatch(detail, /eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9/); + assert.match(detail, /\[redacted-jwt\]/); +}); + +test('503 body mentioning restart is copied, not interpreted as a process restart', () => { + const detail = describeUnexpectedResponse({ + operation: 'owner-a save', + status: 503, + body: 'Detected a change, restarting local server...', + headers: new Headers({ 'content-type': 'text/html' }), + processState: processSnapshot({ pid: 7, exitCode: null, signalCode: null }), + }); + assert.match(detail, /Detected a change, restarting local server/); + assert.doesNotMatch(detail, /^restart:/m); + assert.doesNotMatch(detail, /process restarted/i); + assert.match(detail, /exit=null signal=null/); +}); + +test('body read stops at the byte limit', async () => { + const response = new Response('n'.repeat(5000), { + status: 503, + headers: { 'content-type': 'text/plain' }, + }); + const body = await readBoundedBody(response, { + byteLimit: 64, + timeoutMs: 500, + }); + assert.equal(body.truncated, true); + assert.equal(body.timedOut, false); + assert.equal(body.bytes, 64); + assert.equal(Buffer.byteLength(body.text), 64); +}); + +test('body read stops at the deadline instead of hanging', async () => { + const body = new ReadableStream({ + pull() { + return new Promise(() => {}); + }, + }); + const response = new Response(body, { status: 503 }); + const started = Date.now(); + const result = await readBoundedBody(response, { + byteLimit: 2048, + timeoutMs: 75, + }); + assert.ok(Date.now() - started < 500); + assert.equal(result.timedOut, true); +}); + +test('detached process-group teardown captures post-SIGTERM output and reaps nested children', async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-log-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const nested = ` + const nested = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + windowsHide: true, + }); + process.stdout.write('grandchild=' + nested.pid + '\\n'); + process.on('SIGTERM', () => { + process.stdout.write('after-term\\n'); + process.exit(0); + }); + setInterval(() => {}, 1000); + `; + const child = spawn( + process.execPath, + ['-e', `import { spawn } from 'node:child_process';${nested}`], + { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + windowsHide: true, + }, + ); + let seen = ''; + child.stdout.on('data', (chunk) => { + seen += chunk.toString(); + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + const deadline = Date.now() + 2_000; + while (!seen.includes('grandchild=') && Date.now() < deadline) + await delay(10); + const grandchild = Number(seen.match(/grandchild=(\d+)/)?.[1]); + assert.ok(grandchild, `nested child pid missing: ${seen}`); + try { + await finishProductionServer(child, log, { timeoutMs: 1_000 }); + const text = await readFile(logPath, 'utf8'); + if (process.platform !== 'win32') assert.match(text, /after-term/); + assert.throws(() => process.kill(grandchild, 0), { code: 'ESRCH' }); + } finally { + if (child.pid) { + try { + if (process.platform === 'win32') { + spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { + windowsHide: true, + stdio: 'ignore', + }); + } else { + process.kill(-child.pid, 'SIGKILL'); + } + } catch { + /* Best-effort cleanup after the assertion. */ + } + } + try { + process.kill(grandchild, 'SIGKILL'); + } catch { + /* Nested child already reaped. */ + } + } +}); + +test('teardown still finishes when the leader ignores SIGTERM', async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-kill-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const child = spawn( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stdout.write("ready\\n"); setInterval(() => {}, 1000);', + ], + { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + windowsHide: true, + }, + ); + let seen = ''; + child.stdout.on('data', (chunk) => { + seen += chunk.toString(); + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + const readyUntil = Date.now() + 2_000; + while (!seen.includes('ready') && Date.now() < readyUntil) await delay(10); + assert.match(seen, /ready/); + const started = Date.now(); + try { + await finishProductionServer(child, log, { timeoutMs: 200 }); + if (process.platform !== 'win32') assert.ok(Date.now() - started >= 150); + assert.ok(Date.now() - started < 2_000); + assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' }); + await readFile(logPath); + } finally { + if (child.pid) { + try { + process.kill( + process.platform === 'win32' ? child.pid : -child.pid, + 'SIGKILL', + ); + } catch { + /* Best-effort cleanup after the assertion. */ + } + } + } +}); From 7acccc79e3445e3a06d1331871586cc12c5a8f17 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 01:03:10 +0000 Subject: [PATCH 2/6] Bound production body reads to one cancellable deadline Pending reader.cancel() no longer outlives the timeout, and chunked reads no longer leave a delay timer per chunk. Redaction and success-path body handling are unchanged. Co-authored-by: Seth Carlson --- scripts/ci/production-diagnostics.mjs | 26 +++++++-------- tests/production-diagnostics.test.mjs | 46 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/scripts/ci/production-diagnostics.mjs b/scripts/ci/production-diagnostics.mjs index 4b63657..0936508 100644 --- a/scripts/ci/production-diagnostics.mjs +++ b/scripts/ci/production-diagnostics.mjs @@ -1,5 +1,4 @@ import { spawn } from 'node:child_process'; -import { setTimeout as delay } from 'node:timers/promises'; const BODY_BYTE_LIMIT = 2048; const BODY_READ_TIMEOUT_MS = 2_000; @@ -43,17 +42,18 @@ export async function readBoundedBody( let bytes = 0; let truncated = false; let timedOut = false; - const deadline = Date.now() + timeoutMs; + let timeoutId; + const expired = new Promise((resolve) => { + timeoutId = setTimeout(() => { + timedOut = true; + resolve(); + }, timeoutMs); + }); try { - while (bytes < byteLimit) { - const remaining = deadline - Date.now(); - if (remaining <= 0) { - timedOut = true; - break; - } + while (bytes < byteLimit && !timedOut) { const outcome = await Promise.race([ reader.read().then((chunk) => ({ kind: 'chunk', chunk })), - delay(remaining).then(() => ({ kind: 'timeout' })), + expired.then(() => ({ kind: 'timeout' })), ]); if (outcome.kind === 'timeout') { timedOut = true; @@ -74,11 +74,9 @@ export async function readBoundedBody( bytes += chunk.byteLength; } } finally { - try { - await reader.cancel(); - } catch { - /* The stream may already be closed. */ - } + const cancel = Promise.resolve(reader.cancel()).catch(() => {}); + await Promise.race([cancel, expired]); + clearTimeout(timeoutId); } return { text: Buffer.concat(chunks.map((part) => Buffer.from(part))).toString( diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs index bbc0e61..e86f180 100644 --- a/tests/production-diagnostics.test.mjs +++ b/tests/production-diagnostics.test.mjs @@ -97,6 +97,52 @@ test('body read stops at the deadline instead of hanging', async () => { assert.equal(result.timedOut, true); }); +function timeoutHandles() { + return process.getActiveResourcesInfo().filter((name) => name === 'Timeout') + .length; +} + +test('pending cancel cannot exceed the overall deadline', async () => { + const response = new Response( + new ReadableStream({ + pull() { + return new Promise(() => {}); + }, + cancel() { + return new Promise(() => {}); + }, + }), + { status: 503 }, + ); + const result = await Promise.race([ + readBoundedBody(response, { timeoutMs: 25 }).then((body) => ({ + kind: 'returned', + body, + })), + delay(150).then(() => ({ kind: 'HUNG' })), + ]); + assert.equal(result.kind, 'returned'); + assert.equal(result.body.timedOut, true); +}); + +test('chunked body does not leave dangling deadline timers', async () => { + const before = timeoutHandles(); + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < 20; i++) controller.enqueue(new Uint8Array([120])); + controller.close(); + }, + }); + const body = await readBoundedBody(new Response(stream, { status: 503 }), { + byteLimit: 2048, + timeoutMs: 500, + }); + assert.equal(body.bytes, 20); + assert.equal(body.text, 'x'.repeat(20)); + assert.equal(body.timedOut, false); + assert.equal(timeoutHandles(), before); +}); + test('detached process-group teardown captures post-SIGTERM output and reaps nested children', async () => { const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-log-')); const logPath = resolve(directory, 'production-server.log'); From 4aa68e563970916ba833350c0f63c2afe48d9198 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 01:07:39 +0000 Subject: [PATCH 3/6] Keep process-group teardown after the leader exits exitCode and signalCode no longer skip SIGTERM/SIGKILL of remaining group members or waiting for both stdio streams to close. Related to #22. Co-authored-by: Seth Carlson --- scripts/ci/production-diagnostics.mjs | 27 +++++++--- tests/production-diagnostics.test.mjs | 71 ++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/scripts/ci/production-diagnostics.mjs b/scripts/ci/production-diagnostics.mjs index 0936508..2d2b2a0 100644 --- a/scripts/ci/production-diagnostics.mjs +++ b/scripts/ci/production-diagnostics.mjs @@ -122,14 +122,21 @@ export function describeUnexpectedResponse({ return `${lines.join('\n')}\n`; } -function hasStopped(server) { - return !server || server.exitCode !== null || server.signalCode !== null; +function stdioOpen(server) { + return [server?.stdout, server?.stderr].some( + (stream) => stream && !stream.destroyed && !stream.readableEnded, + ); } function waitForClose(server, timeoutMs) { - if (hasStopped(server)) return Promise.resolve(); + if (!server) return Promise.resolve(); + if ( + !stdioOpen(server) && + (server.exitCode !== null || server.signalCode !== null) + ) { + return Promise.resolve(); + } return new Promise((accept) => { - const timer = setTimeout(accept, timeoutMs); let settled = false; const done = () => { if (settled) return; @@ -137,13 +144,19 @@ function waitForClose(server, timeoutMs) { clearTimeout(timer); accept(); }; + const timer = setTimeout(done, timeoutMs); server.once('close', done); - if (hasStopped(server)) done(); + if ( + !stdioOpen(server) && + (server.exitCode !== null || server.signalCode !== null) + ) { + done(); + } }); } async function signalGroup(server, posixSignal) { - if (!server?.pid || hasStopped(server)) return; + if (!server?.pid) return; if (process.platform === 'win32') { await new Promise((accept) => { const stop = spawn('taskkill', ['/pid', String(server.pid), '/T', '/F'], { @@ -173,7 +186,7 @@ export async function finishProductionServer( const closed = waitForClose(server, timeoutMs); await signalGroup(server, 'SIGTERM'); await closed; - if (server?.pid && !hasStopped(server)) { + if (server?.pid && stdioOpen(server)) { const last = waitForClose(server, timeoutMs); await signalGroup(server, 'SIGKILL'); await last; diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs index e86f180..0123d75 100644 --- a/tests/production-diagnostics.test.mjs +++ b/tests/production-diagnostics.test.mjs @@ -1,7 +1,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; -import { createWriteStream } from 'node:fs'; +import { once } from 'node:events'; +import { createWriteStream, readFileSync } from 'node:fs'; import { mkdtemp, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; @@ -252,3 +253,71 @@ test('teardown still finishes when the leader ignores SIGTERM', async () => { } } }); + +function posixTerminated(pid) { + try { + process.kill(pid, 0); + } catch (error) { + if (error.code === 'ESRCH') return true; + throw error; + } + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + return stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z'); + } catch { + return true; + } +} + +test( + 'exited group leader does not skip TERM/KILL of a TERM-ignoring descendant', + { skip: process.platform === 'win32' }, + async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-group-')); + const logPath = resolve(directory, 'production-server.log'); + const log = createWriteStream(logPath); + const descendant = + "process.on('SIGTERM',()=>{process.stdout.write('GRANDCHILD_TERM_STDOUT\\n');process.stderr.write('GRANDCHILD_TERM_STDERR\\n');});setInterval(()=>{},1000);process.send('ready');"; + const leader = + "const {spawn}=require('node:child_process'); const c=spawn(process.execPath,['-e'," + + JSON.stringify(descendant) + + "],{stdio:['ignore','inherit','inherit','ipc']});c.once('message',()=>process.stdout.write('grandchild='+c.pid+'\\n',()=>process.exit(0)));"; + const child = spawn(process.execPath, ['-e', leader], { + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let output = ''; + let closed = false; + const closePromise = once(child, 'close').then(() => { + closed = true; + }); + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let grandchild; + try { + await once(child, 'exit'); + grandchild = Number(output.match(/grandchild=(\d+)/)?.[1]); + assert.ok(grandchild, `nested child pid missing: ${output}`); + assert.equal(child.exitCode, 0); + assert.equal(closed, false); + await finishProductionServer(child, log, { timeoutMs: 100 }); + assert.equal(closed, true); + const text = await readFile(logPath, 'utf8'); + assert.match(text, /GRANDCHILD_TERM_STDOUT/); + assert.match(text, /GRANDCHILD_TERM_STDERR/); + assert.equal(posixTerminated(grandchild), true); + } finally { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } + await closePromise; + if (!log.writableEnded) log.end(); + } + }, +); From 3d65c2d5fd8ac725e4b00b1dcc9642d2ce6c072c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 01:14:21 +0000 Subject: [PATCH 4/6] Escalate production teardown KILL on group liveness A TERM-ignoring process can close stdout and stderr and still be alive. SIGKILL now follows process-group existence, not stream state. Related to #22. Co-authored-by: Seth Carlson --- scripts/ci/production-diagnostics.mjs | 16 ++++++++- tests/production-diagnostics.test.mjs | 48 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/scripts/ci/production-diagnostics.mjs b/scripts/ci/production-diagnostics.mjs index 2d2b2a0..1fcbdad 100644 --- a/scripts/ci/production-diagnostics.mjs +++ b/scripts/ci/production-diagnostics.mjs @@ -155,6 +155,20 @@ function waitForClose(server, timeoutMs) { }); } +function groupAlive(server) { + if (!server?.pid) return false; + if (process.platform === 'win32') { + return server.exitCode === null && server.signalCode === null; + } + try { + process.kill(-server.pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + async function signalGroup(server, posixSignal) { if (!server?.pid) return; if (process.platform === 'win32') { @@ -186,7 +200,7 @@ export async function finishProductionServer( const closed = waitForClose(server, timeoutMs); await signalGroup(server, 'SIGTERM'); await closed; - if (server?.pid && stdioOpen(server)) { + if (groupAlive(server)) { const last = waitForClose(server, timeoutMs); await signalGroup(server, 'SIGKILL'); await last; diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs index 0123d75..b8e7506 100644 --- a/tests/production-diagnostics.test.mjs +++ b/tests/production-diagnostics.test.mjs @@ -6,6 +6,7 @@ import { createWriteStream, readFileSync } from 'node:fs'; import { mkdtemp, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; +import { Writable } from 'node:stream'; import { setTimeout as delay } from 'node:timers/promises'; import { describeUnexpectedResponse, @@ -321,3 +322,50 @@ test( } }, ); + +test( + 'TERM-ignoring process is KILLed after it closes stdio', + { skip: process.platform === 'win32' }, + async () => { + const child = spawn( + process.execPath, + [ + '-e', + "process.on('SIGTERM',()=>{});setInterval(()=>{},1000);process.stdout.end(()=>process.stderr.end(()=>process.send('ready')));", + ], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + windowsHide: true, + }, + ); + const closed = once(child, 'close'); + const log = new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + try { + await Promise.all([ + once(child, 'message'), + once(child.stdout, 'end'), + once(child.stderr, 'end'), + ]); + assert.equal(child.stdout.readableEnded, true); + assert.equal(child.stderr.readableEnded, true); + assert.equal(child.exitCode, null); + await finishProductionServer(child, log, { timeoutMs: 100 }); + assert.equal(posixTerminated(child.pid), true); + } finally { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } + await closed; + if (!log.writableEnded) log.end(); + } + }, +); From dda4090ab7749dfa0238eca66497f89e71acef46 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 01:26:34 +0000 Subject: [PATCH 5/6] Install SIGTERM handler before fixture readiness The process-group fixture published grandchild= before the leader handled SIGTERM, so teardown could default-kill and drop after-term. Flush that line in the write callback before exit. Related to #22. Co-authored-by: Seth Carlson --- tests/production-diagnostics.test.mjs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs index b8e7506..0230800 100644 --- a/tests/production-diagnostics.test.mjs +++ b/tests/production-diagnostics.test.mjs @@ -150,15 +150,14 @@ test('detached process-group teardown captures post-SIGTERM output and reaps nes const logPath = resolve(directory, 'production-server.log'); const log = createWriteStream(logPath); const nested = ` + process.on('SIGTERM', () => { + process.stdout.write('after-term\\n', () => process.exit(0)); + }); const nested = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', windowsHide: true, }); process.stdout.write('grandchild=' + nested.pid + '\\n'); - process.on('SIGTERM', () => { - process.stdout.write('after-term\\n'); - process.exit(0); - }); setInterval(() => {}, 1000); `; const child = spawn( From 7b1f4cc3a7d9b82f20cc5e240e544acfb90d0d9f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 01:31:40 +0000 Subject: [PATCH 6/6] Satisfy oxlint on production diagnostic tests Quality failed because node:test registrations were un-awaited and unary minus was applied to ChildProcess.pid (number | undefined). Match the existing void test() style and narrow pids before SIGKILL. No helper or runtime change. Co-authored-by: Seth Carlson --- tests/production-diagnostics.test.mjs | 65 +++++++++++++++------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/tests/production-diagnostics.test.mjs b/tests/production-diagnostics.test.mjs index 0230800..15254c2 100644 --- a/tests/production-diagnostics.test.mjs +++ b/tests/production-diagnostics.test.mjs @@ -29,7 +29,7 @@ function headers(extra = {}) { }); } -test('unexpected response keeps selected metadata and omits request authentication', () => { +void test('unexpected response keeps selected metadata and omits request authentication', () => { const detail = describeUnexpectedResponse({ operation: 'owner-a save', status: 503, @@ -54,7 +54,7 @@ test('unexpected response keeps selected metadata and omits request authenticati assert.match(detail, /\[redacted-jwt\]/); }); -test('503 body mentioning restart is copied, not interpreted as a process restart', () => { +void test('503 body mentioning restart is copied, not interpreted as a process restart', () => { const detail = describeUnexpectedResponse({ operation: 'owner-a save', status: 503, @@ -68,7 +68,7 @@ test('503 body mentioning restart is copied, not interpreted as a process restar assert.match(detail, /exit=null signal=null/); }); -test('body read stops at the byte limit', async () => { +void test('body read stops at the byte limit', async () => { const response = new Response('n'.repeat(5000), { status: 503, headers: { 'content-type': 'text/plain' }, @@ -83,7 +83,7 @@ test('body read stops at the byte limit', async () => { assert.equal(Buffer.byteLength(body.text), 64); }); -test('body read stops at the deadline instead of hanging', async () => { +void test('body read stops at the deadline instead of hanging', async () => { const body = new ReadableStream({ pull() { return new Promise(() => {}); @@ -104,7 +104,7 @@ function timeoutHandles() { .length; } -test('pending cancel cannot exceed the overall deadline', async () => { +void test('pending cancel cannot exceed the overall deadline', async () => { const response = new Response( new ReadableStream({ pull() { @@ -127,7 +127,7 @@ test('pending cancel cannot exceed the overall deadline', async () => { assert.equal(result.body.timedOut, true); }); -test('chunked body does not leave dangling deadline timers', async () => { +void test('chunked body does not leave dangling deadline timers', async () => { const before = timeoutHandles(); const stream = new ReadableStream({ start(controller) { @@ -145,7 +145,7 @@ test('chunked body does not leave dangling deadline timers', async () => { assert.equal(timeoutHandles(), before); }); -test('detached process-group teardown captures post-SIGTERM output and reaps nested children', async () => { +void test('detached process-group teardown captures post-SIGTERM output and reaps nested children', async () => { const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-log-')); const logPath = resolve(directory, 'production-server.log'); const log = createWriteStream(logPath); @@ -186,15 +186,16 @@ test('detached process-group teardown captures post-SIGTERM output and reaps nes if (process.platform !== 'win32') assert.match(text, /after-term/); assert.throws(() => process.kill(grandchild, 0), { code: 'ESRCH' }); } finally { - if (child.pid) { + const pid = child.pid; + if (typeof pid === 'number') { try { if (process.platform === 'win32') { - spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { + spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore', }); } else { - process.kill(-child.pid, 'SIGKILL'); + process.kill(-pid, 'SIGKILL'); } } catch { /* Best-effort cleanup after the assertion. */ @@ -208,7 +209,7 @@ test('detached process-group teardown captures post-SIGTERM output and reaps nes } }); -test('teardown still finishes when the leader ignores SIGTERM', async () => { +void test('teardown still finishes when the leader ignores SIGTERM', async () => { const directory = await mkdtemp(resolve(tmpdir(), 'relay-prod-kill-')); const logPath = resolve(directory, 'production-server.log'); const log = createWriteStream(logPath); @@ -238,15 +239,15 @@ test('teardown still finishes when the leader ignores SIGTERM', async () => { await finishProductionServer(child, log, { timeoutMs: 200 }); if (process.platform !== 'win32') assert.ok(Date.now() - started >= 150); assert.ok(Date.now() - started < 2_000); - assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' }); + const pid = child.pid; + assert.ok(typeof pid === 'number'); + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }); await readFile(logPath); } finally { - if (child.pid) { + const pid = child.pid; + if (typeof pid === 'number') { try { - process.kill( - process.platform === 'win32' ? child.pid : -child.pid, - 'SIGKILL', - ); + process.kill(process.platform === 'win32' ? pid : -pid, 'SIGKILL'); } catch { /* Best-effort cleanup after the assertion. */ } @@ -269,7 +270,7 @@ function posixTerminated(pid) { } } -test( +void test( 'exited group leader does not skip TERM/KILL of a TERM-ignoring descendant', { skip: process.platform === 'win32' }, async () => { @@ -311,10 +312,13 @@ test( assert.match(text, /GRANDCHILD_TERM_STDERR/); assert.equal(posixTerminated(grandchild), true); } finally { - try { - process.kill(-child.pid, 'SIGKILL'); - } catch { - /* Best-effort group cleanup after the assertion. */ + const pid = child.pid; + if (typeof pid === 'number') { + try { + process.kill(-pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } } await closePromise; if (!log.writableEnded) log.end(); @@ -322,7 +326,7 @@ test( }, ); -test( +void test( 'TERM-ignoring process is KILLed after it closes stdio', { skip: process.platform === 'win32' }, async () => { @@ -356,12 +360,17 @@ test( assert.equal(child.stderr.readableEnded, true); assert.equal(child.exitCode, null); await finishProductionServer(child, log, { timeoutMs: 100 }); - assert.equal(posixTerminated(child.pid), true); + const pid = child.pid; + assert.ok(typeof pid === 'number'); + assert.equal(posixTerminated(pid), true); } finally { - try { - process.kill(-child.pid, 'SIGKILL'); - } catch { - /* Best-effort group cleanup after the assertion. */ + const pid = child.pid; + if (typeof pid === 'number') { + try { + process.kill(-pid, 'SIGKILL'); + } catch { + /* Best-effort group cleanup after the assertion. */ + } } await closed; if (!log.writableEnded) log.end();