From 78c9f1f8c6113a020292a757abeadea0a87dd011 Mon Sep 17 00:00:00 2001 From: s1gr1d <32902192+s1gr1d@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:15:40 +0100 Subject: [PATCH 1/3] test(node-integration): Give each docker compose phase its own timeout `spawnSync` blocks the event loop, so the 120 s hook timeout could not stop a slow image pull. It only failed the hook after the containers were already healthy (52 failures, pulls of up to 237 s). The pull, each `up` attempt, `down`, and `logs` now have an enforced timeout, and the hook timeout is the sum of them. A phase that times out names itself in the error. --- .../utils/runner/describeWithDockerCompose.ts | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/dev-packages/node-integration-tests/utils/runner/describeWithDockerCompose.ts b/dev-packages/node-integration-tests/utils/runner/describeWithDockerCompose.ts index 0ab245e48cb9..ed1ebd8a5a7d 100644 --- a/dev-packages/node-integration-tests/utils/runner/describeWithDockerCompose.ts +++ b/dev-packages/node-integration-tests/utils/runner/describeWithDockerCompose.ts @@ -8,6 +8,20 @@ type DefineTests = () => void; const SLOW_START_MS = 60_000; +// The pull time depends on the image registry, which this repo does not control. The slowest pull in CI took 237s. +const PULL_TIMEOUT_MS = 300_000; +const UP_TIMEOUT_MS = 120_000; +const UP_MAX_ATTEMPTS = 3; +const DOWN_TIMEOUT_MS = 60_000; +const LOGS_TIMEOUT_MS = 10_000; +const HOOK_MARGIN_MS = 5_000; + +// `spawnSync` blocks the event loop, so a hook timeout cannot stop a phase. Each phase has its own +// enforced timeout, and the hook timeout is the worst case of all phases together: the hook can +// time out only when a phase did. +const START_TIMEOUT_MS = + DOWN_TIMEOUT_MS + PULL_TIMEOUT_MS + UP_MAX_ATTEMPTS * (UP_TIMEOUT_MS + LOGS_TIMEOUT_MS + DOWN_TIMEOUT_MS); + interface DockerOptions { /** * The working directory to run docker compose in @@ -54,15 +68,14 @@ export function describeWithDockerCompose( describe(name, options, () => { let dockerDown: (() => void) | undefined; - // Bring the container up once for the whole group. The generous timeout covers image pulls and - // slow healthchecks on the first run. + // Bring the container up once for the whole group. beforeAll(async () => { dockerDown = await runDockerCompose(dockerOptions); - }, 120_000); + }, START_TIMEOUT_MS); afterAll(() => { dockerDown?.(); - }, 60_000); + }, DOWN_TIMEOUT_MS + HOOK_MARGIN_MS); callback(); }); @@ -93,13 +106,19 @@ async function runDockerCompose(options: DockerOptions): Promise { spawnSync('docker', composeArgs('down', '--volumes'), { cwd, stdio: process.env.DEBUG ? 'inherit' : undefined, + timeout: DOWN_TIMEOUT_MS, }); }; const output = (result: ReturnType): string => `${result.stderr?.toString() ?? ''}${result.stdout?.toString() ?? ''}`; // Surface container logs to make healthcheck failures easier to diagnose in CI - const containerLogs = (): string => spawnSync('docker', composeArgs('logs'), { cwd }).stdout?.toString() ?? ''; + const containerLogs = (): string => + spawnSync('docker', composeArgs('logs'), { cwd, timeout: LOGS_TIMEOUT_MS }).stdout?.toString() ?? ''; + const timedOut = (result: ReturnType): boolean => + (result.error as { code?: string } | undefined)?.code === 'ETIMEDOUT'; + const upOutcome = (result: ReturnType): string => + timedOut(result) ? `timed out after ${UP_TIMEOUT_MS}ms` : `exit ${result.status}`; // ensure we're starting fresh close(); @@ -110,8 +129,13 @@ async function runDockerCompose(options: DockerOptions): Promise { const pull = spawnSync('docker', composeArgs('pull', '--policy', 'missing', '--quiet'), { cwd, stdio: process.env.DEBUG ? 'inherit' : 'pipe', + timeout: PULL_TIMEOUT_MS, }); const pullMs = Date.now() - pullStartedAt; + if (timedOut(pull)) { + // `up` would pull again inside its shorter timeout and hide which phase was slow. + throw new Error(`docker compose pull timed out after ${PULL_TIMEOUT_MS}ms in ${cwd}\n${output(pull)}`); + } if (pull.status !== 0) { // `up` pulls missing images too, so only report this // eslint-disable-next-line no-console @@ -122,21 +146,21 @@ async function runDockerCompose(options: DockerOptions): Promise { spawnSync('docker', composeArgs('up', '-d', '--wait'), { cwd, stdio: process.env.DEBUG ? 'inherit' : 'pipe', + timeout: UP_TIMEOUT_MS, }); // `docker compose up` occasionally fails on CI with transient daemon races // (e.g. "failed to set up container networking: network _default not // found" right after the network was created). A clean teardown plus retry // clears these, while genuine healthcheck failures stay red on every attempt. - const maxAttempts = 3; const upStartedAt = Date.now(); let attemptStartedAt = upStartedAt; let result = composeUp(); - for (let attempt = 1; attempt < maxAttempts && result.status !== 0; attempt++) { + for (let attempt = 1; attempt < UP_MAX_ATTEMPTS && result.status !== 0; attempt++) { // eslint-disable-next-line no-console console.warn( `[docker compose] up --wait attempt ${attempt} failed after ${Date.now() - attemptStartedAt}ms ` + - `(exit ${result.status}) in ${cwd}\n${output(result)}\n--- container logs ---\n${containerLogs()}`, + `(${upOutcome(result)}) in ${cwd}\n${output(result)}\n--- container logs ---\n${containerLogs()}`, ); close(); attemptStartedAt = Date.now(); @@ -148,7 +172,7 @@ async function runDockerCompose(options: DockerOptions): Promise { const logs = containerLogs(); close(); throw new Error( - `docker compose up --wait failed (exit ${result.status})\n${output(result)}\n--- container logs ---\n${logs}`, + `docker compose up --wait failed (${upOutcome(result)}) in ${cwd}\n${output(result)}\n--- container logs ---\n${logs}`, ); } From 4c4468b999b6811c7da01b2a17a9f1e4630fe490 Mon Sep 17 00:00:00 2001 From: s1gr1d <32902192+s1gr1d@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:15:40 +0100 Subject: [PATCH 2/3] test(e2e): Wait for SAM to exit before removing its containers `stopSam` sent SIGKILL after 5 s and went on without waiting for the exit. SAM was still removing its warm containers, so `docker rm -f` raced that removal and failed the teardown after all tests had passed. The fixture now waits for the exit, and a failed `docker rm` is an error only if a container stays. --- .../tests/lambda-fixtures.ts | 50 +++++++++++++++---- .../aws-serverless/tests/lambda-fixtures.ts | 50 +++++++++++++++---- 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts index 1a041f9cd638..9338bb42c013 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts +++ b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts @@ -14,6 +14,8 @@ const SAM_START_TIMEOUT_MS = 90_000; const SAM_START_ATTEMPTS = 2; const SAM_OUTPUT_MAX_CHARS = 20_000; +const CONTAINER_REMOVAL_TIMEOUT_MS = 10_000; + /** Major Node for SAM `--invoke-image`; default is the oldest major the SDK supports and matches `pull-sam-image.sh`. */ const DEFAULT_NODE_VERSION_MAJOR = '20'; @@ -125,10 +127,7 @@ async function stopSam(samProcess: ChildProcess): Promise { if (samProcess.exitCode === null && samProcess.signalCode === null) { samProcess.kill('SIGTERM'); await new Promise(resolve => { - const timer = setTimeout(() => { - samProcess.kill('SIGKILL'); - resolve(void 0); - }, 5000); + const timer = setTimeout(() => samProcess.kill('SIGKILL'), 5000); samProcess.once('exit', () => { clearTimeout(timer); resolve(void 0); @@ -138,7 +137,7 @@ async function stopSam(samProcess: ChildProcess): Promise { // A SAM process killed mid-start leaves its runtime containers behind, which would keep the // docker network in use and hold on to their ports. - removeNetworkContainers(); + await removeNetworkContainers(); } /** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */ @@ -171,13 +170,46 @@ function createDockerNetwork() { } } -function removeNetworkContainers() { - const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) +function listNetworkContainers(): string[] { + return execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) .split('\n') .filter(Boolean); - if (containerIds.length) { - execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' }); +} + +async function removeNetworkContainers(): Promise { + const containerIds = listNetworkContainers(); + if (!containerIds.length) { + return; } + + try { + execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'pipe' }); + } catch (error) { + // `docker rm` fails for a container whose removal is already in progress, for example on a + // request of a SAM process that was killed. That removal still completes, so only a container + // that stays is a failure. + const stderr = (error as { stderr?: Buffer }).stderr?.toString() ?? ''; + const remainingIds = await waitForNetworkContainersRemoved(); + if (remainingIds.length) { + throw new Error( + `Failed to remove containers ${remainingIds.join(' ')} from docker network ${DOCKER_NETWORK_NAME}: ${stderr}`, + { cause: error }, + ); + } + console.warn(`[testEnvironment fixture] docker rm failed, but the containers are gone: ${stderr}`); + } +} + +async function waitForNetworkContainersRemoved(): Promise { + const startTime = Date.now(); + let remainingIds = listNetworkContainers(); + + while (remainingIds.length && Date.now() - startTime < CONTAINER_REMOVAL_TIMEOUT_MS) { + await new Promise(resolve => setTimeout(resolve, 500)); + remainingIds = listNetworkContainers(); + } + + return remainingIds; } function removeDockerNetwork() { diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts b/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts index 1a041f9cd638..9338bb42c013 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts +++ b/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts @@ -14,6 +14,8 @@ const SAM_START_TIMEOUT_MS = 90_000; const SAM_START_ATTEMPTS = 2; const SAM_OUTPUT_MAX_CHARS = 20_000; +const CONTAINER_REMOVAL_TIMEOUT_MS = 10_000; + /** Major Node for SAM `--invoke-image`; default is the oldest major the SDK supports and matches `pull-sam-image.sh`. */ const DEFAULT_NODE_VERSION_MAJOR = '20'; @@ -125,10 +127,7 @@ async function stopSam(samProcess: ChildProcess): Promise { if (samProcess.exitCode === null && samProcess.signalCode === null) { samProcess.kill('SIGTERM'); await new Promise(resolve => { - const timer = setTimeout(() => { - samProcess.kill('SIGKILL'); - resolve(void 0); - }, 5000); + const timer = setTimeout(() => samProcess.kill('SIGKILL'), 5000); samProcess.once('exit', () => { clearTimeout(timer); resolve(void 0); @@ -138,7 +137,7 @@ async function stopSam(samProcess: ChildProcess): Promise { // A SAM process killed mid-start leaves its runtime containers behind, which would keep the // docker network in use and hold on to their ports. - removeNetworkContainers(); + await removeNetworkContainers(); } /** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */ @@ -171,13 +170,46 @@ function createDockerNetwork() { } } -function removeNetworkContainers() { - const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) +function listNetworkContainers(): string[] { + return execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) .split('\n') .filter(Boolean); - if (containerIds.length) { - execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' }); +} + +async function removeNetworkContainers(): Promise { + const containerIds = listNetworkContainers(); + if (!containerIds.length) { + return; } + + try { + execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'pipe' }); + } catch (error) { + // `docker rm` fails for a container whose removal is already in progress, for example on a + // request of a SAM process that was killed. That removal still completes, so only a container + // that stays is a failure. + const stderr = (error as { stderr?: Buffer }).stderr?.toString() ?? ''; + const remainingIds = await waitForNetworkContainersRemoved(); + if (remainingIds.length) { + throw new Error( + `Failed to remove containers ${remainingIds.join(' ')} from docker network ${DOCKER_NETWORK_NAME}: ${stderr}`, + { cause: error }, + ); + } + console.warn(`[testEnvironment fixture] docker rm failed, but the containers are gone: ${stderr}`); + } +} + +async function waitForNetworkContainersRemoved(): Promise { + const startTime = Date.now(); + let remainingIds = listNetworkContainers(); + + while (remainingIds.length && Date.now() - startTime < CONTAINER_REMOVAL_TIMEOUT_MS) { + await new Promise(resolve => setTimeout(resolve, 500)); + remainingIds = listNetworkContainers(); + } + + return remainingIds; } function removeDockerNetwork() { From efd06c1ce52f1c16d9f5d53b428b2d1352dc8123 Mon Sep 17 00:00:00 2001 From: s1gr1d <32902192+s1gr1d@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:15:40 +0100 Subject: [PATCH 3/3] test(e2e): Pull compose images in the build step of deno and bun-mysql `docker compose up` runs in the `start` script, inside the 60 s web server timeout of Playwright. A slow image pull used up that time. The build step now pulls the images. The global setup of nuxt-4 and nuxt-5 also prints `docker compose ps -a` on a failed start, because the logs do not show the exit code of a container that restarts. --- .../e2e-tests/test-applications/bun-mysql/package.json | 2 +- dev-packages/e2e-tests/test-applications/deno/package.json | 4 ++-- .../e2e-tests/test-applications/nuxt-4/global-setup.mjs | 3 ++- .../e2e-tests/test-applications/nuxt-5/global-setup.mjs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/bun-mysql/package.json b/dev-packages/e2e-tests/test-applications/bun-mysql/package.json index 5527205df01a..e64844549375 100644 --- a/dev-packages/e2e-tests/test-applications/bun-mysql/package.json +++ b/dev-packages/e2e-tests/test-applications/bun-mysql/package.json @@ -7,7 +7,7 @@ "start": "docker compose up -d --wait && bun run dist/app.js", "test": "playwright test", "clean": "npx rimraf node_modules pnpm-lock.yaml dist", - "test:build": "pnpm install && bun run build.ts", + "test:build": "pnpm install && bun run build.ts && docker compose pull --quiet", "test:assert": "pnpm test" }, "dependencies": { diff --git a/dev-packages/e2e-tests/test-applications/deno/package.json b/dev-packages/e2e-tests/test-applications/deno/package.json index df4fb4e3e705..8a9315279005 100644 --- a/dev-packages/e2e-tests/test-applications/deno/package.json +++ b/dev-packages/e2e-tests/test-applications/deno/package.json @@ -6,9 +6,9 @@ "start": "docker compose up -d --wait && deno run --allow-net --allow-env --allow-read --allow-sys --allow-write --preload=@sentry/deno/import src/app.ts", "test": "playwright test", "clean": "npx rimraf node_modules pnpm-lock.yaml", - "test:build": "pnpm install", + "test:build": "pnpm install && docker compose pull --quiet", "test:assert": "pnpm test", - "test:build:static": "E2E_TEST_STATIC=true pnpm install", + "test:build:static": "E2E_TEST_STATIC=true pnpm install && docker compose pull --quiet", "test:assert:static": "E2E_TEST_STATIC=true pnpm test" }, "dependencies": { diff --git a/dev-packages/e2e-tests/test-applications/nuxt-4/global-setup.mjs b/dev-packages/e2e-tests/test-applications/nuxt-4/global-setup.mjs index 0eabcb30c905..f2b2d17fcfa7 100644 --- a/dev-packages/e2e-tests/test-applications/nuxt-4/global-setup.mjs +++ b/dev-packages/e2e-tests/test-applications/nuxt-4/global-setup.mjs @@ -13,7 +13,8 @@ export default async function globalSetup() { stdio: 'inherit', }); } catch (error) { - execSync('docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' }); + // `ps` shows the exit code of a container and whether it restarts, which the logs do not. + execSync('docker compose ps -a; docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' }); throw error; } } diff --git a/dev-packages/e2e-tests/test-applications/nuxt-5/global-setup.mjs b/dev-packages/e2e-tests/test-applications/nuxt-5/global-setup.mjs index 826a78375d74..85801b93a5d6 100644 --- a/dev-packages/e2e-tests/test-applications/nuxt-5/global-setup.mjs +++ b/dev-packages/e2e-tests/test-applications/nuxt-5/global-setup.mjs @@ -13,7 +13,8 @@ export default async function globalSetup() { stdio: 'inherit', }); } catch (error) { - execSync('docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' }); + // `ps` shows the exit code of a container and whether it restarts, which the logs do not. + execSync('docker compose ps -a; docker compose logs --tail 50', { cwd: __dirname, stdio: 'inherit' }); throw error; } }