From fc94f673248fd58f6883b375260a40a63c696ffc Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Tue, 18 Aug 2026 16:20:39 -0500 Subject: [PATCH 1/5] [e2e] Re-enable concurrent execution of the e2e suite Serial execution has been the dominant wall-clock cost per matrix entry since concurrency was disabled before conf (78048e0f4b): ~128 tests at ~22 of 24 minutes on the Vercel lanes, and lately the slowest lane cannot finish under its 30-minute job timeout on a slow runner day at all. #2083 measured the concurrent suite at ~3x job wall-clock (4-5x on the vitest phase) and identified what broke; its blockers are now fixed: world-local writeExclusive is atomic (write-then-link), abort-fetch tests are hermetic (#3618), the fibonacci tree fits the scheduler (#3619), and source-map assertions are positive-only (#3620). What this change adds is concurrency-safe per-test attribution. The harness tracked runs and test names in module globals reset by a beforeEach - under concurrency every test clobbered every other's state, so a failing test dumped an unrelated sibling's diagnostics. vitest's getCurrentTest() cannot substitute: it is a plain module variable, wrong after any await. Instead an auto fixture - the one place that receives the test's own context unambiguously - binds a per-test state (name, tracked runs, the test's own skip) via AsyncLocalStorage around each test body, and trackRun / recordInfraEvent / requireFixture read it ambiently with no call-site changes. The conformance gates skip through the bound state's skip, so a mid-body requireFixture skips the right test. Sequential suites (dev, agent, region) keep setupRunTracking's module-global fallback. Full suite passes 137/137 concurrently against a local dev server in under 2 minutes. A test that genuinely cannot share a deployment can opt out with test.sequential. Builds on VaguelySerious's investigation in #2083. Signed-off-by: Alex Langenfeld --- packages/core/e2e/e2e.test.ts | 61 ++++++++++++--- packages/core/e2e/utils.test.ts | 32 ++++++++ packages/core/e2e/utils.ts | 128 ++++++++++++++++++++++++-------- 3 files changed, 177 insertions(+), 44 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 52867c4e4b..7acd53d9ee 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -16,10 +16,9 @@ import { afterAll, assert, beforeAll, - beforeEach, describe, expect, - test, + test as vitestTest, } from 'vitest'; import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs'; import type { Run } from '../src/runtime'; @@ -46,7 +45,11 @@ import { isJsApp, isLocalDeployment, requireFixture, - setupRunTracking, + announceTestStart, + createPerTestState, + dumpTrackedRunDiagnostics, + requireSupported, + runInTestState, setupWorld, startTracked, trackRun, @@ -144,6 +147,43 @@ const e2e = (fn: string) => { * Every test not marked here is in scope for cross-language conformance, and is * gated only by `e2e-conformance.json`. No-op for the JS workbench apps. */ +/** + * Every test in this suite runs through this auto fixture, which owns the + * per-test harness plumbing the sequential suites do in a `beforeEach` + * (announce heartbeat, conformance gate, failure diagnostics): + * + * - The suite runs concurrently, and vitest's `getCurrentTest()` is a plain + * module variable that is wrong after any `await`, so nothing per-test can + * live in module globals. The fixture is the one place that receives the + * test's own context unambiguously; it binds a per-test state (name, + * tracked runs, the test's own `skip`) via AsyncLocalStorage around the + * test body, and `trackRun`/`recordInfraEvent`/`requireFixture` read it + * ambiently — no call-site changes. + * - Failure diagnostics dump from the state the fixture bound, so a failing + * test reports its own runs, not a concurrent sibling's. + */ +const test = vitestTest.extend<{ e2eTracking: unknown }>({ + e2eTracking: [ + // biome-ignore lint/correctness/noEmptyPattern: vitest fixture signature + async ({ task, skip, onTestFailed }, use) => { + const state = createPerTestState(task.name, skip); + announceTestStart(task.name); + onTestFailed( + (result) => + dumpTrackedRunDiagnostics(state, result.errors?.[0]?.message), + 30_000 // Allow 30s for diagnostics fetching (default hookTimeout is 10s) + ); + await runInTestState(state, async () => { + // Second conformance gate — inside the bound state so the skip + // targets this test. + requireSupported(task.name); + await use(state); + }); + }, + { auto: true }, + ], +}); + const testJsOnly = isJsApp() ? test : test.skip; const describeJsOnly = isJsApp() ? describe : describe.skip; @@ -321,9 +361,13 @@ async function startWorkflowViaHttp( return run; } -// NOTE: Temporarily disabling concurrent tests to avoid flakiness. -// TODO: Re-enable concurrent tests after conf when we have more time to investigate. -describe('e2e', () => { +// Concurrent: ~128 serial tests were the dominant wall-clock cost per matrix +// entry (~22 of 24 minutes on the Vercel lanes). The known blockers are +// fixed: per-test attribution is concurrency-safe (see the e2eTracking +// fixture), abort-fetch tests are hermetic, the fibonacci tree fits the +// scheduler, and source-map assertions are positive-only. A test that +// genuinely cannot share a deployment can opt out with `test.sequential`. +describe.concurrent('e2e', () => { // Configure the World for the test runner process so that start() and // run.returnValue can communicate with the same backend as the workbench app. // Also warm the target before the first test starts a run: a fresh Vercel @@ -346,11 +390,6 @@ describe('e2e', () => { ); }, 150_000); - // Enable automatic run diagnostics on test failure - beforeEach((ctx) => { - setupRunTracking(ctx.task.name); - }); - // Write E2E metadata and diagnostics files afterAll(() => { writeE2EMetadata(); diff --git a/packages/core/e2e/utils.test.ts b/packages/core/e2e/utils.test.ts index b985ae5c8a..9315f0b14c 100644 --- a/packages/core/e2e/utils.test.ts +++ b/packages/core/e2e/utils.test.ts @@ -1,7 +1,11 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { + createPerTestState, + getCollectedRunIds, getRecordedInfraEvents, hasStepSourceMaps, + runInTestState, + trackRun, waitForRunPickup, warmDeployment, } from './utils'; @@ -214,3 +218,31 @@ describe('warmDeployment', () => { ).toBe(startProbe.mock.calls.length); }); }); + +describe('per-test state isolation', () => { + test('interleaved contexts attribute runs to their own test', async () => { + const before = getCollectedRunIds().length; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + const fakeRun = (id: string) => ({ runId: id }) as never; + + // Two "tests" interleaving on the event loop, as under + // describe.concurrent: each tracks a run after yielding, so a + // module-global current-test-name would attribute both to whichever + // context touched it last. + await Promise.all([ + runInTestState(createPerTestState('test-a'), async () => { + await sleep(20); + trackRun(fakeRun('wrun_a')); + }), + runInTestState(createPerTestState('test-b'), async () => { + await sleep(10); + trackRun(fakeRun('wrun_b')); + }), + ]); + + const entries = getCollectedRunIds().slice(before); + expect( + Object.fromEntries(entries.map((e) => [e.runId, e.testName])) + ).toEqual({ wrun_a: 'test-a', wrun_b: 'test-b' }); + }); +}); diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 7c0779ab0a..b9b5eddb3b 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path, { dirname } from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; +import { AsyncLocalStorage } from 'node:async_hooks'; import { getCurrentTest } from '@vitest/runner'; import { createWorkflowUrl } from '@workflow/utils'; import { createWorld as createVercelTestWorld } from '@workflow/world-vercel'; @@ -268,7 +269,7 @@ export function hasFixture(fixtureName: string): boolean { */ export function requireFixture(fixtureName: string): void { if (hasFixture(fixtureName)) return; - getCurrentTest()?.context.skip( + currentSkip()?.( `"${fixtureName}" is not listed in ${CONFORMANCE_CONFIG_FILENAME}` ); } @@ -290,7 +291,7 @@ export function requireSupported(testName: string): void { seenTestNames.add(testName); const reason = getConformanceConfig()?.unsupported?.[testName]; if (!reason) return; - getCurrentTest()?.context.skip( + currentSkip()?.( `${CONFORMANCE_CONFIG_FILENAME} declares this unsupported: ${reason}` ); } @@ -758,8 +759,56 @@ interface TrackedRun { workflowFn?: string; } -// Per-test tracked runs — reset between tests via setupRunTracking() -let trackedRuns: TrackedRun[] = []; +/** + * Per-test harness state: the name used to attribute runs and infra events, + * and the runs whose diagnostics dump if the test fails. + * + * Concurrent suites bind one of these per test via {@link runInTestState} + * (AsyncLocalStorage), so tests interleaving on the event loop cannot + * clobber each other's attribution — `getCurrentTest()` is a plain module + * variable in vitest and is wrong after any `await` under concurrency. + * Sequential suites (dev.test.ts, e2e-agent.test.ts, e2e-region.test.ts) + * keep the classic path: {@link setupRunTracking} resets a module-level + * fallback that is safe when only one test runs at a time. + */ +interface PerTestState { + testName: string; + trackedRuns: TrackedRun[]; + /** + * The test's own `ctx.skip`, captured where the context is unambiguous + * (the fixture), so conformance gates called mid-test-body can skip the + * right test — `getCurrentTest()?.context.skip` would target whichever + * test most recently started. + */ + skip?: (note?: string) => void; +} + +const testStateStorage = new AsyncLocalStorage(); +let fallbackTestState: PerTestState = { + testName: 'unknown', + trackedRuns: [], +}; +const currentTestState = (): PerTestState => + testStateStorage.getStore() ?? fallbackTestState; + +export function createPerTestState( + testName: string, + skip?: (note?: string) => void +): PerTestState { + return { testName, trackedRuns: [], skip }; +} + +/** ALS-bound skip when available, vitest's global otherwise. */ +const currentSkip = (): ((note?: string) => void) | undefined => + testStateStorage.getStore()?.skip ?? getCurrentTest()?.context.skip; + +/** Run `fn` with `state` bound as the ambient per-test state. */ +export function runInTestState( + state: PerTestState, + fn: () => Promise +): Promise { + return testStateStorage.run(state, fn); +} // Global list of run IDs collected for metadata (observability links) const globalCollectedRunIds: { @@ -790,8 +839,9 @@ export function trackRun( workflowFn?: string; } ): Run { - const testName = options?.testName ?? currentTestName; - trackedRuns.push({ + const state = currentTestState(); + const testName = options?.testName ?? state.testName; + state.trackedRuns.push({ run, workflowFile: options?.workflowFile, workflowFn: options?.workflowFn, @@ -865,7 +915,7 @@ export function recordInfraEvent( ) { infraEvents.push({ ...event, - testName: event.testName ?? currentTestName, + testName: event.testName ?? currentTestState().testName, timestamp: new Date().toISOString(), }); } @@ -1223,43 +1273,55 @@ function emitGitHubAnnotation( * beforeEach((ctx) => { setupRunTracking(ctx.task.name); }); */ export function setupRunTracking(testName: string) { - currentTestName = testName; - trackedRuns = []; + fallbackTestState = createPerTestState(testName); // Second conformance gate. Sited here because every test in the suite calls // setupRunTracking from `beforeEach`, which makes this the one place that // sees a test's name without the test having to declare anything. requireSupported(testName); - // Heartbeat: announce the test the moment it starts, written straight to - // stdout to bypass vitest's per-file console buffering. Without this, a - // test that stalls (e.g. polling a run that never progresses) produces no - // output until its timeout, making CI look like a silent hang — the - // reporter only prints a test's result line once it completes. Emitting the - // name on start makes the stalling test immediately identifiable. - process.stdout.write(`\n[e2e] ▶ start: ${testName}\n`); + announceTestStart(testName); + const state = fallbackTestState; onTestFailed( - async (result) => { - const errorMessage = result.errors?.[0]?.message || 'Test failed'; - - for (const tracked of trackedRuns) { - try { - const diagnostics = await getRunDiagnostics(tracked); - console.error(diagnostics); - emitGitHubAnnotation(testName, tracked, errorMessage); - } catch { - console.error( - `[diagnostics] Failed to fetch diagnostics for run ${tracked.run.runId}` - ); - } - } - }, + (result) => dumpTrackedRunDiagnostics(state, result.errors?.[0]?.message), 30_000 // Allow 30s for diagnostics fetching (default hookTimeout is 10s) ); } -// Current test name for auto-tracking -let currentTestName = 'unknown'; +/** + * Heartbeat: announce the test the moment it starts, written straight to + * stdout to bypass vitest's per-file console buffering. Without this, a + * test that stalls (e.g. polling a run that never progresses) produces no + * output until its timeout, making CI look like a silent hang — the + * reporter only prints a test's result line once it completes. Emitting the + * name on start makes the stalling test immediately identifiable. + */ +export function announceTestStart(testName: string) { + process.stdout.write(`\n[e2e] ▶ start: ${testName}\n`); +} + +/** + * Dump diagnostics for every run tracked by `state`. Shared by the + * sequential path (setupRunTracking's onTestFailed) and the concurrent + * fixture, which passes the state it bound for its own test — the one + * thing vitest's globals cannot provide under concurrency. + */ +export async function dumpTrackedRunDiagnostics( + state: PerTestState, + errorMessage = 'Test failed' +) { + for (const tracked of state.trackedRuns) { + try { + const diagnostics = await getRunDiagnostics(tracked); + console.error(diagnostics); + emitGitHubAnnotation(state.testName, tracked, errorMessage); + } catch { + console.error( + `[diagnostics] Failed to fetch diagnostics for run ${tracked.run.runId}` + ); + } + } +} /** * Write diagnostics sidecar file with per-test run info for the aggregation script. From f437a2121852be31075d0a72b56e1b261be100ad Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Tue, 18 Aug 2026 17:12:37 -0500 Subject: [PATCH 2/5] [e2e] Bind per-test state via a task collector, not a fixture CI on the first concurrent run caught the fixture design failing in the one place skip-correctness is load-bearing: vitest resolves fixtures in a separate async context, so an AsyncLocalStorage store bound around use() never reaches the test body. Every ambient read silently fell to the module fallback - conformance-gated tests on the Python lane ran instead of skipping and hard-failed on fixtures missing from the manifest ('declared in e2e-conformance.json but not in the deployed manifest'). The suite's test is now built with createTaskCollector, mirroring vitest's own argument normalization, and the wrapper binds the state around the handler call itself - a direct call stack, so propagation is guaranteed rather than assumed. Validated by a collector-pattern ALS experiment (per-test store after awaits, mid-body skip targets self, each/skip/todo chains), a conformance-file simulation reproducing the Python failure shape (fixture-missing tests now skip with the right note), and a 137/137 concurrent local run. Also raises RACE_WINNER_MAX_DURATION_MS 5s -> 8s: the winner takes 1s, the loser 10s, and the bound only has to sit clearly below the loser - concurrent-suite queue latency pushed observed winner durations to ~6.5s on loaded local-dev lanes (vite, first concurrent CI run), so 5s flaked without catching anything 8s misses. Signed-off-by: Alex Langenfeld --- packages/core/e2e/e2e.test.ts | 112 +++++++++++++++++++++++----------- packages/core/e2e/utils.ts | 2 +- 2 files changed, 76 insertions(+), 38 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 7acd53d9ee..1ec5970ee5 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -18,8 +18,10 @@ import { beforeAll, describe, expect, - test as vitestTest, + type TestContext, + type test as vitestTest, } from 'vitest'; +import { createTaskCollector, getCurrentSuite } from 'vitest/suite'; import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs'; import type { Run } from '../src/runtime'; import { @@ -31,11 +33,14 @@ import { resumeHook, } from '../src/runtime'; import { + announceTestStart, assertUnsupportedTestsExist, cliCancel, cliHealthJson, cliInspectJson, cliInspectJsonUntil, + createPerTestState, + dumpTrackedRunDiagnostics, fetchManifest, getCollectedRunIds, getWorkflowMetadata, @@ -45,9 +50,6 @@ import { isJsApp, isLocalDeployment, requireFixture, - announceTestStart, - createPerTestState, - dumpTrackedRunDiagnostics, requireSupported, runInTestState, setupWorld, @@ -64,7 +66,12 @@ if (!deploymentUrl) { } const DISTRIBUTED_CLOCK_TOLERANCE_MS = 1_000; -const RACE_WINNER_MAX_DURATION_MS = 5_000; +// The race winner takes 1s; the loser would take 10s. The bound only has to +// sit clearly below the loser to catch badly delayed or sequential +// completion — under the concurrent suite, queue latency pushed the winner's +// observed duration to ~6.5s on loaded local-dev lanes, so 5s was tight +// enough to flake without being any better at catching the regression. +const RACE_WINNER_MAX_DURATION_MS = 8_000; const EVENT_POLL_PAGE_SIZE = 100; function expectElapsedAtLeast( @@ -148,41 +155,72 @@ const e2e = (fn: string) => { * gated only by `e2e-conformance.json`. No-op for the JS workbench apps. */ /** - * Every test in this suite runs through this auto fixture, which owns the + * Every test in this suite runs through this handler wrapper, which owns the * per-test harness plumbing the sequential suites do in a `beforeEach` - * (announce heartbeat, conformance gate, failure diagnostics): + * (announce heartbeat, conformance gates, failure diagnostics). * - * - The suite runs concurrently, and vitest's `getCurrentTest()` is a plain - * module variable that is wrong after any `await`, so nothing per-test can - * live in module globals. The fixture is the one place that receives the - * test's own context unambiguously; it binds a per-test state (name, - * tracked runs, the test's own `skip`) via AsyncLocalStorage around the - * test body, and `trackRun`/`recordInfraEvent`/`requireFixture` read it - * ambiently — no call-site changes. - * - Failure diagnostics dump from the state the fixture bound, so a failing - * test reports its own runs, not a concurrent sibling's. + * The suite runs concurrently, and vitest's `getCurrentTest()` is a plain + * module variable that is wrong after any `await`, so nothing per-test can + * live in module globals. The wrapper binds a per-test state (name, tracked + * runs, the test's own `skip`) via AsyncLocalStorage *around the handler + * call itself* — a direct call stack, so the store provably reaches the test + * body — and `trackRun`/`recordInfraEvent`/`requireFixture` read it + * ambiently with no call-site changes. (A `test.extend` auto fixture cannot + * do this: vitest resolves fixtures in a separate async context, so a store + * bound around `use()` never reaches the test body.) Failure diagnostics + * dump from the bound state, so a failing test reports its own runs, not a + * concurrent sibling's. */ -const test = vitestTest.extend<{ e2eTracking: unknown }>({ - e2eTracking: [ - // biome-ignore lint/correctness/noEmptyPattern: vitest fixture signature - async ({ task, skip, onTestFailed }, use) => { - const state = createPerTestState(task.name, skip); - announceTestStart(task.name); - onTestFailed( - (result) => - dumpTrackedRunDiagnostics(state, result.errors?.[0]?.message), - 30_000 // Allow 30s for diagnostics fetching (default hookTimeout is 10s) - ); - await runInTestState(state, async () => { - // Second conformance gate — inside the bound state so the skip - // targets this test. - requireSupported(task.name); - await use(state); - }); - }, - { auto: true }, - ], -}); +const wrapE2EHandler = + (handler: (ctx: TestContext) => unknown) => (ctx: TestContext) => { + const state = createPerTestState(ctx.task.name, ctx.skip); + announceTestStart(ctx.task.name); + ctx.onTestFailed( + (result) => dumpTrackedRunDiagnostics(state, result.errors?.[0]?.message), + 30_000 // Allow 30s for diagnostics fetching (default hookTimeout is 10s) + ); + return runInTestState(state, async () => { + // Second conformance gate — inside the bound state so the skip + // targets this test. + requireSupported(ctx.task.name); + return handler(ctx); + }); + }; + +/** + * Drop-in `test` whose collector mirrors vitest's own argument handling + * (options-second, legacy options/timeout-third, `.each`'s generated + * callbacks) and wraps every handler with {@link wrapE2EHandler}. Built on + * `createTaskCollector`, so the whole chainable surface (`.skip`, `.only`, + * `.each`, `.runIf`, `.sequential`, …) keeps working. + */ +const test = createTaskCollector(function ( + this: Record, + name: string, + optionsOrFn?: unknown, + optionsOrTest?: unknown +) { + let options: Record = {}; + let handler: (ctx: TestContext) => unknown = () => {}; + if (typeof optionsOrTest === 'object' && optionsOrTest !== null) { + options = optionsOrTest as Record; + handler = optionsOrFn as typeof handler; + } else if (typeof optionsOrTest === 'number') { + options = { timeout: optionsOrTest }; + handler = optionsOrFn as typeof handler; + } else if (typeof optionsOrFn === 'object' && optionsOrFn !== null) { + options = optionsOrFn as Record; + handler = optionsOrTest as typeof handler; + } else if (typeof optionsOrFn === 'function') { + handler = optionsOrFn as typeof handler; + } + + getCurrentSuite().task(name, { + ...this, + ...options, + handler: wrapE2EHandler(handler), + }); +}) as typeof vitestTest; const testJsOnly = isJsApp() ? test : test.skip; const describeJsOnly = isJsApp() ? describe : describe.skip; diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index b9b5eddb3b..5dbdb9c080 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -1,9 +1,9 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import { spawn } from 'node:child_process'; import fs from 'node:fs'; import path, { dirname } from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; -import { AsyncLocalStorage } from 'node:async_hooks'; import { getCurrentTest } from '@vitest/runner'; import { createWorkflowUrl } from '@workflow/utils'; import { createWorld as createVercelTestWorld } from '@workflow/world-vercel'; From ef36a7e9dcf973c746a1a05d47b968410b083887 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Wed, 19 Aug 2026 11:02:41 -0500 Subject: [PATCH 3/5] [e2e] Delegate to the suite collector so describe.concurrent applies The previous commit's collector called getCurrentSuite().task() directly, which skips the suite collector's Object.assign({}, suiteOptions, options) - the merge that carries describe.concurrent onto each test. The suite therefore still ran sequentially, and CI proved it: every Vercel Prod lane matched the serial baseline minute-for-minute (nextjs-turbopack quickjs 25m vs 25m), so a fully green run said nothing about concurrency. The same slowdown showed up locally as a full-suite run taking 6m15s where the earlier concurrent one took 1m58s, which I had misread as server slowness. The collector now wraps the handler and hands the call to getCurrentSuite().test.fn - exactly what vitest's own top-level test does - so suite-option inheritance is vitest's code path again, while the wrapper still owns the handler call stack that makes the AsyncLocalStorage binding reach the test body. Verified with an experiment asserting both properties at once: four 300ms tests plus a 600ms observer finish in 602ms (1800ms if sequential), max-in-flight > 1, and each test still reads its own store after awaits. Signed-off-by: Alex Langenfeld --- packages/core/e2e/e2e.test.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 1ec5970ee5..292b3f4b07 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -188,11 +188,19 @@ const wrapE2EHandler = }; /** - * Drop-in `test` whose collector mirrors vitest's own argument handling - * (options-second, legacy options/timeout-third, `.each`'s generated - * callbacks) and wraps every handler with {@link wrapE2EHandler}. Built on - * `createTaskCollector`, so the whole chainable surface (`.skip`, `.only`, - * `.each`, `.runIf`, `.sequential`, …) keeps working. + * Drop-in `test` that wraps every handler with {@link wrapE2EHandler} and + * then hands the call to the enclosing suite's own collector — exactly what + * vitest's top-level `test` does (`getCurrentSuite().test.fn.call(this, …)`). + * + * Delegating rather than calling `getCurrentSuite().task()` directly is + * load-bearing: the suite collector is where suite options are merged into + * each test (`Object.assign({}, suiteOptions, options)`), which is how + * `describe.concurrent` reaches its tests. Calling `task()` directly skips + * that merge, and the suite silently runs sequentially — caught in CI as + * lanes matching the serial baseline minute-for-minute. + * + * Built on `createTaskCollector`, so the whole chainable surface (`.skip`, + * `.only`, `.each`, `.runIf`, `.sequential`, …) keeps working. */ const test = createTaskCollector(function ( this: Record, @@ -200,26 +208,22 @@ const test = createTaskCollector(function ( optionsOrFn?: unknown, optionsOrTest?: unknown ) { - let options: Record = {}; + let options: unknown = {}; let handler: (ctx: TestContext) => unknown = () => {}; if (typeof optionsOrTest === 'object' && optionsOrTest !== null) { - options = optionsOrTest as Record; + options = optionsOrTest; handler = optionsOrFn as typeof handler; } else if (typeof optionsOrTest === 'number') { options = { timeout: optionsOrTest }; handler = optionsOrFn as typeof handler; } else if (typeof optionsOrFn === 'object' && optionsOrFn !== null) { - options = optionsOrFn as Record; + options = optionsOrFn; handler = optionsOrTest as typeof handler; } else if (typeof optionsOrFn === 'function') { handler = optionsOrFn as typeof handler; } - getCurrentSuite().task(name, { - ...this, - ...options, - handler: wrapE2EHandler(handler), - }); + getCurrentSuite().test.fn.call(this, name, options, wrapE2EHandler(handler)); }) as typeof vitestTest; const testJsOnly = isJsApp() ? test : test.skip; From ecde0f16e9922a924aeb656b06cf452854c6f8b6 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Wed, 19 Aug 2026 11:24:58 -0500 Subject: [PATCH 4/5] [e2e] Report per-lane load so concurrency tuning has numbers Under concurrency the interesting question is not pass/fail but where per-test latency went: the first genuinely-concurrent CI run halved lane wall-clock (25m -> 13m) while inflating individual tests 5-7x (9s -> 50-68s), pushing some past budgets written for an unloaded suite. Two candidate causes - deployment queueing, or runner CPU contention from the child every CLI assertion spawns - call for different fixes and were indistinguishable from the logs. Each lane now ends with a load summary: test count and peak concurrent tests, summed and median test wall time, CLI child count with peak concurrency and wall time as a share of test time, and the ten slowest tests. CLI accounting hangs off awaitCommand, the single point every CLI assertion spawns through. maxConcurrency also becomes tunable via WORKFLOW_E2E_MAX_CONCURRENCY (default unchanged at vitest's 5), since the right value is a property of the runner and deployment rather than of the tests. Signed-off-by: Alex Langenfeld --- packages/core/e2e/e2e.test.ts | 3 ++ packages/core/e2e/utils.ts | 88 ++++++++++++++++++++++++++++++++++- vitest.config.ts | 9 ++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 292b3f4b07..11567f9183 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -49,11 +49,14 @@ import { hasWorkflowSourceMaps, isJsApp, isLocalDeployment, + noteTestSettled, + noteTestStarted, requireFixture, requireSupported, runInTestState, setupWorld, startTracked, + summarizeLoad, trackRun, warmDeployment, writeDiagnosticsSidecar, diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 5dbdb9c080..689b5bf208 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -413,6 +413,76 @@ function getCliArgs(): string { return `--backend vercel --verbose`; } +// --------------------------------------------------------------------------- +// Load observability +// +// The concurrent suite's cost is not obvious from pass/fail: tests can pass +// while per-test latency inflates several-fold, and the inflation can come +// from the deployment (queueing) or from the runner (every CLI assertion +// spawns a full `node` child, and a CI runner has few cores). The counters +// here make each lane report which one it was — peak test and CLI +// concurrency, CLI child count, and CLI wall time as a share of total test +// wall time — so a tuning decision has numbers behind it instead of a guess. +// --------------------------------------------------------------------------- + +const loadStats = { + cliCalls: 0, + cliMs: 0, + cliInFlight: 0, + cliPeakInFlight: 0, + testsInFlight: 0, + testsPeakInFlight: 0, + testMs: 0, + durations: [] as { name: string; ms: number }[], +}; + +/** Called by the suite's handler wrapper when a test body starts. */ +export function noteTestStarted() { + loadStats.testsInFlight++; + loadStats.testsPeakInFlight = Math.max( + loadStats.testsPeakInFlight, + loadStats.testsInFlight + ); +} + +/** Called by the suite's handler wrapper when a test body settles. */ +export function noteTestSettled(name: string, ms: number) { + loadStats.testsInFlight--; + loadStats.testMs += ms; + loadStats.durations.push({ name, ms }); +} + +/** + * One-line-per-fact load summary for the job log. Logged from `afterAll`. + */ +export function summarizeLoad(): string { + const { durations } = loadStats; + if (durations.length === 0) return ''; + const slowest = [...durations].sort((a, b) => b.ms - a.ms).slice(0, 10); + const sum = durations.reduce((acc, d) => acc + d.ms, 0); + const cliShare = + loadStats.testMs > 0 + ? Math.round((loadStats.cliMs / loadStats.testMs) * 100) + : 0; + const lines = [ + '', + '━━━ e2e load summary ━━━', + `tests: ${durations.length} · peak concurrent: ${loadStats.testsPeakInFlight}`, + `test wall time (summed): ${Math.round(sum / 1000)}s · median ${Math.round( + [...durations].sort((a, b) => a.ms - b.ms)[ + Math.floor(durations.length / 2) + ].ms + )}ms`, + `cli children: ${loadStats.cliCalls} · peak concurrent: ${loadStats.cliPeakInFlight} · ` + + `wall time ${Math.round(loadStats.cliMs / 1000)}s (${cliShare}% of summed test time)`, + 'slowest tests:', + ...slowest.map((d) => ` ${Math.round(d.ms / 1000)}s ${d.name}`), + '━━━━━━━━━━━━━━━━━━━━━━', + '', + ]; + return lines.join('\n'); +} + const awaitCommand = async ( command: string, args: string[], @@ -423,6 +493,18 @@ const awaitCommand = async ( console.log(`[Debug]: Executing ${command} ${args.join(' ')}`); console.log(`[Debug]: in CWD: ${cwd}`); + loadStats.cliCalls++; + loadStats.cliInFlight++; + loadStats.cliPeakInFlight = Math.max( + loadStats.cliPeakInFlight, + loadStats.cliInFlight + ); + const cliStartedAt = Date.now(); + const noteCliSettled = () => { + loadStats.cliInFlight--; + loadStats.cliMs += Date.now() - cliStartedAt; + }; + return await new Promise<{ stdout: string; stderr: string }>( (resolve, reject) => { const child = spawn(command, args, { @@ -459,8 +541,12 @@ const awaitCommand = async ( }); } - child.on('error', (err) => reject(err)); + child.on('error', (err) => { + noteCliSettled(); + reject(err); + }); child.on('close', (code, signal) => { + noteCliSettled(); if (code !== 0) { const exitReason = signal ? `killed by signal ${signal}` diff --git a/vitest.config.ts b/vitest.config.ts index ad59d47870..78cac94104 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,6 +12,15 @@ export default defineConfig({ // the signal (event-log-race-repro, benchmarks) pin `retry: 0` locally. // Local runs keep retry at 0 so races reproduce while debugging. retry: process.env.CI ? 1 : 0, + // How many concurrent tests vitest runs from a `describe.concurrent` + // suite (vitest's own default is 5). Only the e2e conformance suite is + // concurrent, so this is effectively its dial. Tunable because the right + // value is a property of the runner and the deployment rather than of + // the tests: every CLI assertion spawns a `node` child, and a CI runner + // has few cores, so too high a value inflates per-test latency until + // tests exceed budgets written for an unloaded suite. Each lane logs + // what it observed (see `summarizeLoad` in the e2e utils). + maxConcurrency: Number(process.env.WORKFLOW_E2E_MAX_CONCURRENCY ?? 5), // Positional file arguments are regex filters, not paths, so // `vitest run packages/core/e2e/x.test.ts` also matches // `.claude/worktrees//packages/core/e2e/x.test.ts` when agent From c35ea2c39ee58d5de5ad5f5d3f7d8764adcc8dd0 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Wed, 19 Aug 2026 11:44:47 -0500 Subject: [PATCH 5/5] [e2e] Actually call the load-observability hooks The previous commit added the counters and the summary but left the call sites out of the handler wrapper and afterAll, so nothing was collected and no summary printed. Caught by review. Signed-off-by: Alex Langenfeld --- packages/core/e2e/e2e.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 11567f9183..43a74e608f 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -186,7 +186,16 @@ const wrapE2EHandler = // Second conformance gate — inside the bound state so the skip // targets this test. requireSupported(ctx.task.name); - return handler(ctx); + // Timed for the per-lane load summary (see summarizeLoad): under + // concurrency the interesting number is not pass/fail but how far + // per-test latency moved and whether CLI children dominate it. + const startedAt = Date.now(); + noteTestStarted(); + try { + return await handler(ctx); + } finally { + noteTestSettled(ctx.task.name, Date.now() - startedAt); + } }); }; @@ -437,6 +446,9 @@ describe.concurrent('e2e', () => { // Write E2E metadata and diagnostics files afterAll(() => { + // First, so the numbers reach the log even if a later assertion in this + // hook throws. + process.stdout.write(summarizeLoad()); writeE2EMetadata(); writeDiagnosticsSidecar(); writeInfraSidecar();