Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 215 additions & 0 deletions scripts/ci/production-diagnostics.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
import { spawn } from 'node:child_process';

const BODY_BYTE_LIMIT = 2048;
const BODY_READ_TIMEOUT_MS = 2_000;
const RESPONSE_METADATA_NAMES = ['content-type', 'retry-after'];

const JWT_SHAPE = /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;

export function processSnapshot(child) {
return {
pid: child?.pid ?? null,
exitCode: child?.exitCode ?? null,
signal: child?.signalCode ?? null,
};
}

function selectedResponseMetadata(headers) {
const selected = {};
if (!headers) return selected;
for (const name of RESPONSE_METADATA_NAMES) {
const value =
typeof headers.get === 'function' ? headers.get(name) : headers[name];
if (value) selected[name] = String(value);
}
return selected;
}

function redactAuthenticationJwts(text) {
return String(text ?? '').replace(JWT_SHAPE, '[redacted-jwt]');
}

export async function readBoundedBody(
response,
{ byteLimit = BODY_BYTE_LIMIT, timeoutMs = BODY_READ_TIMEOUT_MS } = {},
) {
const empty = { text: '', bytes: 0, truncated: false, timedOut: false };
if (!response?.body || typeof response.body.getReader !== 'function') {
return empty;
}
const reader = response.body.getReader();
const chunks = [];
let bytes = 0;
let truncated = false;
let timedOut = false;
let timeoutId;
const expired = new Promise((resolve) => {
timeoutId = setTimeout(() => {
timedOut = true;
resolve();
}, timeoutMs);
});
try {
while (bytes < byteLimit && !timedOut) {
const outcome = await Promise.race([
reader.read().then((chunk) => ({ kind: 'chunk', chunk })),
expired.then(() => ({ kind: 'timeout' })),
]);
if (outcome.kind === 'timeout') {
timedOut = true;
break;
}
const { done, value } = outcome.chunk;
if (done) break;
if (!value) continue;
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const room = byteLimit - bytes;
if (chunk.byteLength > room) {
chunks.push(chunk.subarray(0, room));
bytes += room;
truncated = true;
break;
}
chunks.push(chunk);
bytes += chunk.byteLength;
}
} finally {
const cancel = Promise.resolve(reader.cancel()).catch(() => {});
await Promise.race([cancel, expired]);
clearTimeout(timeoutId);
}
return {
text: Buffer.concat(chunks.map((part) => Buffer.from(part))).toString(
'utf8',
),
bytes,
truncated,
timedOut,
};
}

export function describeUnexpectedResponse({
operation,
status,
body,
truncated = false,
timedOut = false,
bodyBytes,
headers,
processState,
}) {
const meta = selectedResponseMetadata(headers);
const state = processState ?? {};
const safeBody = redactAuthenticationJwts(body);
const flags = `${truncated ? ' truncated' : ''}${timedOut ? ' timed-out' : ''}`;
const lines = [
'=== unexpected response ===',
`operation: ${operation}`,
`status: ${status}`,
];
for (const name of RESPONSE_METADATA_NAMES) {
if (meta[name]) lines.push(`${name}: ${meta[name]}`);
}
lines.push(
`body-bytes: ${bodyBytes ?? Buffer.byteLength(String(body ?? ''))}${flags}`,
);
lines.push('body:');
lines.push(safeBody);
lines.push(
`process: pid=${state.pid ?? null} exit=${state.exitCode ?? null} signal=${state.signal ?? null}`,
);
lines.push('=== end unexpected response ===');
return `${lines.join('\n')}\n`;
}

function stdioOpen(server) {
return [server?.stdout, server?.stderr].some(
(stream) => stream && !stream.destroyed && !stream.readableEnded,
);
}

function waitForClose(server, timeoutMs) {
if (!server) return Promise.resolve();
if (
!stdioOpen(server) &&
(server.exitCode !== null || server.signalCode !== null)
) {
return Promise.resolve();
}
return new Promise((accept) => {
let settled = false;
const done = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
accept();
};
const timer = setTimeout(done, timeoutMs);
server.once('close', done);
if (
!stdioOpen(server) &&
(server.exitCode !== null || server.signalCode !== null)
) {
done();
}
});
}

function groupAlive(server) {
if (!server?.pid) return false;
if (process.platform === 'win32') {
return server.exitCode === null && server.signalCode === null;
}
try {
process.kill(-server.pid, 0);
return true;
} catch (error) {
if (error.code === 'ESRCH') return false;
throw error;
}
}

async function signalGroup(server, posixSignal) {
if (!server?.pid) return;
if (process.platform === 'win32') {
await new Promise((accept) => {
const stop = spawn('taskkill', ['/pid', String(server.pid), '/T', '/F'], {
windowsHide: true,
stdio: 'ignore',
});
stop.on('exit', accept);
stop.on('error', accept);
});
return;
}
try {
process.kill(-server.pid, posixSignal);
} catch (error) {
if (error.code !== 'ESRCH') {
console.error('Unable to stop test worker:', error);
process.exitCode = 1;
}
}
}

export async function finishProductionServer(
server,
log,
{ timeoutMs = 5_000 } = {},
) {
const closed = waitForClose(server, timeoutMs);
await signalGroup(server, 'SIGTERM');
await closed;
if (groupAlive(server)) {
const last = waitForClose(server, timeoutMs);
await signalGroup(server, 'SIGKILL');
await last;
}
if (log && !log.writableEnded) {
await new Promise((accept, reject) => {
log.once('finish', accept);
log.once('error', reject);
log.end();
});
}
}
106 changes: 55 additions & 51 deletions scripts/ci/run-production.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import { resolve } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { exportJWK, generateKeyPair, SignJWT } from 'jose';
import { releaseConfig } from '../release/config.mjs';
import {
describeUnexpectedResponse,
finishProductionServer,
processSnapshot,
readBoundedBody,
} from './production-diagnostics.mjs';

// Only the test entry supplies a local verification key. The compiled app,
// authentication handler, assets and SQL migrations are the release versions.
Expand Down Expand Up @@ -123,6 +129,25 @@ let startupError;
server.on('error', (error) => {
startupError = error;
});
async function expectStatus(response, expected, operation) {
if (response.status === expected) return response;
const body = await readBoundedBody(response);
const detail = describeUnexpectedResponse({
operation,
status: response.status,
body: body.text,
truncated: body.truncated,
timedOut: body.timedOut,
bodyBytes: body.bytes,
headers: response.headers,
processState: processSnapshot(server),
});
await new Promise((accept, reject) => {
log.write(detail, (error) => (error ? reject(error) : accept()));
Comment thread
sdcarlson marked this conversation as resolved.
Dismissed
});
assert.equal(response.status, expected, detail);
return response;
}

try {
const deadline = Date.now() + 120_000;
Expand Down Expand Up @@ -163,34 +188,40 @@ try {
},
...(body ? { body: JSON.stringify(body) } : {}),
});
assert.equal(
(
await call(null, undefined, {
'oai-authenticated-user-id': 'cloudflare:owner-a',
})
).status,
await expectStatus(
await call(null, undefined, {
'oai-authenticated-user-id': 'cloudflare:owner-a',
}),
401,
'spoofed identity header without Access token',
);
const htmlResponse = await fetch(base, {
headers: { 'Cf-Access-Jwt-Assertion': first },
});
assert.equal(htmlResponse.status, 200, await htmlResponse.clone().text());
await expectStatus(htmlResponse, 200, 'authenticated document');
const html = await htmlResponse.text();
assert.match(html, /Make your next move/);
const scriptPath = html.match(/<script[^>]+src="([^"]+\.js[^"]*)"/)?.[1];
assert.ok(scriptPath, 'Compiled page must load its client JavaScript');
assert.equal(
(await fetch(new URL(scriptPath, base))).status,
await expectStatus(
await fetch(new URL(scriptPath, base)),
401,
'Assets must pass through authentication',
'unauthenticated asset',
);
const script = await expectStatus(
await fetch(new URL(scriptPath, base), {
headers: { 'Cf-Access-Jwt-Assertion': first },
}),
200,
'authenticated asset',
);
const script = await fetch(new URL(scriptPath, base), {
headers: { 'Cf-Access-Jwt-Assertion': first },
});
assert.equal(script.status, 200);
assert.match(script.headers.get('content-type'), /javascript/);
assert.deepEqual((await (await call(first)).json()).jobs, []);
assert.equal((await call(first, { action: 'bootstrap' })).status, 200);
await expectStatus(
await call(first, { action: 'bootstrap' }),
200,
'owner-a bootstrap',
);
const firstWorkspace = await (await call(first)).json();
assert.equal(firstWorkspace.jobs.length, 3);
assert.deepEqual(
Expand All @@ -209,23 +240,18 @@ try {
draft: 'A fictional draft verified through the built application.',
blocker: '',
};
assert.equal(
(await call(second, update)).status,
await expectStatus(
await call(second, update),
404,
'Second owner cannot mutate the first owner record',
'owner-b save of owner-a record',
);
assert.equal(
(await call(first, update, { origin: 'https://untrusted.example.com' }))
.status,
await expectStatus(
await call(first, update, { origin: 'https://untrusted.example.com' }),
403,
'owner-a save with untrusted origin',
);
assert.equal((await call(first, update)).status, 200);
const stale = await call(first, update);
assert.equal(
stale.status,
409,
`Stale writes must fail in the actual worker: ${await stale.text()}`,
);
await expectStatus(await call(first, update), 200, 'owner-a save');
await expectStatus(await call(first, update), 409, 'owner-a stale save');
const saved = await (
await call(first, undefined, {
'oai-authenticated-user-id': 'cloudflare:owner-b',
Expand All @@ -248,27 +274,5 @@ try {
'PASS: built app rendering/assets, signed identity, persistence, tenant isolation, forged headers, request origin, exact acceptance and stale-write integrity.',
);
} finally {
if (server.pid && server.exitCode === null) {
if (process.platform === 'win32') {
await new Promise((accept) => {
const stop = spawn(
'taskkill',
['/pid', String(server.pid), '/T', '/F'],
{ windowsHide: true, stdio: 'ignore' },
);
stop.on('exit', accept);
stop.on('error', accept);
});
} else {
try {
process.kill(-server.pid, 'SIGTERM');
} catch (error) {
if (error.code !== 'ESRCH') {
console.error('Unable to stop test worker:', error);
process.exitCode = 1;
}
}
}
}
log.end();
await finishProductionServer(server, log);
}
Loading
Loading