Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ const SAM_START_TIMEOUT_MS = 90_000;
const SAM_START_ATTEMPTS = 2;
const SAM_OUTPUT_MAX_CHARS = 20_000;

const CONTAINER_REMOVAL_TIMEOUT_MS = 10_000;

/** Major Node for SAM `--invoke-image`; default is the oldest major the SDK supports and matches `pull-sam-image.sh`. */
const DEFAULT_NODE_VERSION_MAJOR = '20';

Expand Down Expand Up @@ -125,10 +127,7 @@ async function stopSam(samProcess: ChildProcess): Promise<void> {
if (samProcess.exitCode === null && samProcess.signalCode === null) {
samProcess.kill('SIGTERM');
await new Promise(resolve => {
const timer = setTimeout(() => {
samProcess.kill('SIGKILL');
resolve(void 0);
}, 5000);
const timer = setTimeout(() => samProcess.kill('SIGKILL'), 5000);
samProcess.once('exit', () => {
clearTimeout(timer);
resolve(void 0);
Expand All @@ -138,7 +137,7 @@ async function stopSam(samProcess: ChildProcess): Promise<void> {

// A SAM process killed mid-start leaves its runtime containers behind, which would keep the
// docker network in use and hold on to their ports.
removeNetworkContainers();
await removeNetworkContainers();
}

/** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */
Expand Down Expand Up @@ -171,13 +170,46 @@ function createDockerNetwork() {
}
}

function removeNetworkContainers() {
const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' })
function listNetworkContainers(): string[] {
return execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' })
.split('\n')
.filter(Boolean);
if (containerIds.length) {
execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' });
}

async function removeNetworkContainers(): Promise<void> {
const containerIds = listNetworkContainers();
if (!containerIds.length) {
return;
}

try {
execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'pipe' });
} catch (error) {
// `docker rm` fails for a container whose removal is already in progress, for example on a
// request of a SAM process that was killed. That removal still completes, so only a container
// that stays is a failure.
const stderr = (error as { stderr?: Buffer }).stderr?.toString() ?? '';
const remainingIds = await waitForNetworkContainersRemoved();
if (remainingIds.length) {
throw new Error(
`Failed to remove containers ${remainingIds.join(' ')} from docker network ${DOCKER_NETWORK_NAME}: ${stderr}`,
{ cause: error },
);
}
console.warn(`[testEnvironment fixture] docker rm failed, but the containers are gone: ${stderr}`);
}
}

async function waitForNetworkContainersRemoved(): Promise<string[]> {
const startTime = Date.now();
let remainingIds = listNetworkContainers();

while (remainingIds.length && Date.now() - startTime < CONTAINER_REMOVAL_TIMEOUT_MS) {
await new Promise(resolve => setTimeout(resolve, 500));
remainingIds = listNetworkContainers();
}

return remainingIds;
}

function removeDockerNetwork() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ const SAM_START_TIMEOUT_MS = 90_000;
const SAM_START_ATTEMPTS = 2;
const SAM_OUTPUT_MAX_CHARS = 20_000;

const CONTAINER_REMOVAL_TIMEOUT_MS = 10_000;

/** Major Node for SAM `--invoke-image`; default is the oldest major the SDK supports and matches `pull-sam-image.sh`. */
const DEFAULT_NODE_VERSION_MAJOR = '20';

Expand Down Expand Up @@ -125,10 +127,7 @@ async function stopSam(samProcess: ChildProcess): Promise<void> {
if (samProcess.exitCode === null && samProcess.signalCode === null) {
samProcess.kill('SIGTERM');
await new Promise(resolve => {
const timer = setTimeout(() => {
samProcess.kill('SIGKILL');
resolve(void 0);
}, 5000);
const timer = setTimeout(() => samProcess.kill('SIGKILL'), 5000);
samProcess.once('exit', () => {
clearTimeout(timer);
resolve(void 0);
Expand All @@ -138,7 +137,7 @@ async function stopSam(samProcess: ChildProcess): Promise<void> {

// A SAM process killed mid-start leaves its runtime containers behind, which would keep the
// docker network in use and hold on to their ports.
removeNetworkContainers();
await removeNetworkContainers();
}

/** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */
Expand Down Expand Up @@ -171,13 +170,46 @@ function createDockerNetwork() {
}
}

function removeNetworkContainers() {
const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' })
function listNetworkContainers(): string[] {
return execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' })
.split('\n')
.filter(Boolean);
if (containerIds.length) {
execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' });
}

async function removeNetworkContainers(): Promise<void> {
const containerIds = listNetworkContainers();
if (!containerIds.length) {
return;
}

try {
execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'pipe' });
} catch (error) {
// `docker rm` fails for a container whose removal is already in progress, for example on a
// request of a SAM process that was killed. That removal still completes, so only a container
// that stays is a failure.
const stderr = (error as { stderr?: Buffer }).stderr?.toString() ?? '';
const remainingIds = await waitForNetworkContainersRemoved();
if (remainingIds.length) {
throw new Error(
`Failed to remove containers ${remainingIds.join(' ')} from docker network ${DOCKER_NETWORK_NAME}: ${stderr}`,
{ cause: error },
);
}
console.warn(`[testEnvironment fixture] docker rm failed, but the containers are gone: ${stderr}`);
}
}

async function waitForNetworkContainersRemoved(): Promise<string[]> {
const startTime = Date.now();
let remainingIds = listNetworkContainers();

while (remainingIds.length && Date.now() - startTime < CONTAINER_REMOVAL_TIMEOUT_MS) {
await new Promise(resolve => setTimeout(resolve, 500));
remainingIds = listNetworkContainers();
}

return remainingIds;
}

function removeDockerNetwork() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"start": "docker compose up -d --wait && bun run dist/app.js",
"test": "playwright test",
"clean": "npx rimraf node_modules pnpm-lock.yaml dist",
"test:build": "pnpm install && bun run build.ts",
"test:build": "pnpm install && bun run build.ts && docker compose pull --quiet",
"test:assert": "pnpm test"
},
"dependencies": {
Expand Down
4 changes: 2 additions & 2 deletions dev-packages/e2e-tests/test-applications/deno/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
"start": "docker compose up -d --wait && deno run --allow-net --allow-env --allow-read --allow-sys --allow-write --preload=@sentry/deno/import src/app.ts",
"test": "playwright test",
"clean": "npx rimraf node_modules pnpm-lock.yaml",
"test:build": "pnpm install",
"test:build": "pnpm install && docker compose pull --quiet",
"test:assert": "pnpm test",
"test:build:static": "E2E_TEST_STATIC=true pnpm install",
"test:build:static": "E2E_TEST_STATIC=true pnpm install && docker compose pull --quiet",
"test:assert:static": "E2E_TEST_STATIC=true pnpm test"
},
"dependencies": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ export default async function globalSetup() {
stdio: 'inherit',
});
} catch (error) {
execSync('docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' });
// `ps` shows the exit code of a container and whether it restarts, which the logs do not.
execSync('docker compose ps -a; docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' });
throw error;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ export default async function globalSetup() {
stdio: 'inherit',
});
} catch (error) {
execSync('docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' });
// `ps` shows the exit code of a container and whether it restarts, which the logs do not.
execSync('docker compose ps -a; docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' });
throw error;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,20 @@ type DefineTests = () => void;

const SLOW_START_MS = 60_000;

// The pull time depends on the image registry, which this repo does not control. The slowest pull in CI took 237s.
const PULL_TIMEOUT_MS = 300_000;
const UP_TIMEOUT_MS = 120_000;
const UP_MAX_ATTEMPTS = 3;
const DOWN_TIMEOUT_MS = 60_000;
const LOGS_TIMEOUT_MS = 10_000;
const HOOK_MARGIN_MS = 5_000;

// `spawnSync` blocks the event loop, so a hook timeout cannot stop a phase. Each phase has its own
// enforced timeout, and the hook timeout is the worst case of all phases together: the hook can
// time out only when a phase did.
const START_TIMEOUT_MS =
DOWN_TIMEOUT_MS + PULL_TIMEOUT_MS + UP_MAX_ATTEMPTS * (UP_TIMEOUT_MS + LOGS_TIMEOUT_MS + DOWN_TIMEOUT_MS);

interface DockerOptions {
/**
* The working directory to run docker compose in
Expand Down Expand Up @@ -54,15 +68,14 @@ export function describeWithDockerCompose(
describe(name, options, () => {
let dockerDown: (() => void) | undefined;

// Bring the container up once for the whole group. The generous timeout covers image pulls and
// slow healthchecks on the first run.
// Bring the container up once for the whole group.
beforeAll(async () => {
dockerDown = await runDockerCompose(dockerOptions);
}, 120_000);
}, START_TIMEOUT_MS);

afterAll(() => {
dockerDown?.();
}, 60_000);
}, DOWN_TIMEOUT_MS + HOOK_MARGIN_MS);

callback();
});
Expand Down Expand Up @@ -93,13 +106,19 @@ async function runDockerCompose(options: DockerOptions): Promise<VoidFunction> {
spawnSync('docker', composeArgs('down', '--volumes'), {
cwd,
stdio: process.env.DEBUG ? 'inherit' : undefined,
timeout: DOWN_TIMEOUT_MS,
});
};

const output = (result: ReturnType<typeof spawnSync>): string =>
`${result.stderr?.toString() ?? ''}${result.stdout?.toString() ?? ''}`;
// Surface container logs to make healthcheck failures easier to diagnose in CI
const containerLogs = (): string => spawnSync('docker', composeArgs('logs'), { cwd }).stdout?.toString() ?? '';
const containerLogs = (): string =>
spawnSync('docker', composeArgs('logs'), { cwd, timeout: LOGS_TIMEOUT_MS }).stdout?.toString() ?? '';
const timedOut = (result: ReturnType<typeof spawnSync>): boolean =>
(result.error as { code?: string } | undefined)?.code === 'ETIMEDOUT';
const upOutcome = (result: ReturnType<typeof spawnSync>): string =>
timedOut(result) ? `timed out after ${UP_TIMEOUT_MS}ms` : `exit ${result.status}`;

// ensure we're starting fresh
close();
Expand All @@ -110,8 +129,13 @@ async function runDockerCompose(options: DockerOptions): Promise<VoidFunction> {
const pull = spawnSync('docker', composeArgs('pull', '--policy', 'missing', '--quiet'), {
cwd,
stdio: process.env.DEBUG ? 'inherit' : 'pipe',
timeout: PULL_TIMEOUT_MS,
});
const pullMs = Date.now() - pullStartedAt;
if (timedOut(pull)) {
// `up` would pull again inside its shorter timeout and hide which phase was slow.
throw new Error(`docker compose pull timed out after ${PULL_TIMEOUT_MS}ms in ${cwd}\n${output(pull)}`);
}
if (pull.status !== 0) {
// `up` pulls missing images too, so only report this
// eslint-disable-next-line no-console
Expand All @@ -122,21 +146,21 @@ async function runDockerCompose(options: DockerOptions): Promise<VoidFunction> {
spawnSync('docker', composeArgs('up', '-d', '--wait'), {
cwd,
stdio: process.env.DEBUG ? 'inherit' : 'pipe',
timeout: UP_TIMEOUT_MS,
});

// `docker compose up` occasionally fails on CI with transient daemon races
// (e.g. "failed to set up container networking: network <x>_default not
// found" right after the network was created). A clean teardown plus retry
// clears these, while genuine healthcheck failures stay red on every attempt.
const maxAttempts = 3;
const upStartedAt = Date.now();
let attemptStartedAt = upStartedAt;
let result = composeUp();
for (let attempt = 1; attempt < maxAttempts && result.status !== 0; attempt++) {
for (let attempt = 1; attempt < UP_MAX_ATTEMPTS && result.status !== 0; attempt++) {
// eslint-disable-next-line no-console
console.warn(
`[docker compose] up --wait attempt ${attempt} failed after ${Date.now() - attemptStartedAt}ms ` +
`(exit ${result.status}) in ${cwd}\n${output(result)}\n--- container logs ---\n${containerLogs()}`,
`(${upOutcome(result)}) in ${cwd}\n${output(result)}\n--- container logs ---\n${containerLogs()}`,
);
close();
attemptStartedAt = Date.now();
Expand All @@ -148,7 +172,7 @@ async function runDockerCompose(options: DockerOptions): Promise<VoidFunction> {
const logs = containerLogs();
close();
throw new Error(
`docker compose up --wait failed (exit ${result.status})\n${output(result)}\n--- container logs ---\n${logs}`,
`docker compose up --wait failed (${upOutcome(result)}) in ${cwd}\n${output(result)}\n--- container logs ---\n${logs}`,
);
}

Expand Down
Loading