From 1f3f777fab66ea41127b0947f07722976b867daf Mon Sep 17 00:00:00 2001 From: Chris Lorenzo Date: Tue, 26 May 2026 18:02:41 -0400 Subject: [PATCH 1/2] perf(vrt): parallelize visual regression tests across N browser pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VRT runner was fully sequential: one Playwright page running every test back-to-back in a single browser-side for-loop. Adds a --workers / -w flag (default 1, preserves old behavior) that spawns N parallel pages sharing one Chromium instance, each handling a round-robin shard of the test list. Browser side (examples/index.ts): runAutomation accepts a ?shard=i/N URL param, sorts test paths deterministically, and skips tests outside its slice (index % N !== i). Sorting keys off the path so shard membership is stable across pages and machines. Node side (visual-regression/src/index.ts): the per-page setup (snapshot/doneTests exposed functions, navigation, donePromise) is extracted into a runWorker function that runs once per shard. Promise.all gates the final summary + browser close. The aggregate snapshot counters stay as module globals — JS is single-threaded so shared ++ is safe across pages. Per-test snapshot indices are kept in a per-page map (each test only runs in one shard, so no cross-page state). Local 4-worker run on a ~3 minute baseline cuts wall time to ~2 minutes (~1.5x), with CPU utilization rising from ~27% to ~63%. Text-rendering tests show some additional variance under parallel load — flagged for follow-up if it shows up in CI; the existing -w 1 default keeps current behavior bit-for-bit. The docker-ci mode relays --workers through so `pnpm test:visual:update -i -w 4` works end-to-end inside the container. Co-Authored-By: Claude Opus 4.7 --- examples/index.ts | 31 ++++-- visual-regression/src/index.ts | 189 ++++++++++++++++----------------- 2 files changed, 116 insertions(+), 104 deletions(-) diff --git a/examples/index.ts b/examples/index.ts index 710424c..a6564bd 100644 --- a/examples/index.ts +++ b/examples/index.ts @@ -102,7 +102,11 @@ const defaultPhysicalPixelRatio = 1; return; } assertTruthy(automation); - await runAutomation(renderMode, test, logFps); + // Optional shard string in the form "i/N" — when present, this page only + // runs the subset of tests where `index % N === i`. Used by the VRT runner + // to parallelize across multiple browser pages. + const shardParam = urlParams.get('shard'); + await runAutomation(renderMode, test, logFps, shardParam); })().catch((err) => { console.error(err); }); @@ -348,7 +352,17 @@ async function runAutomation( renderMode: string, filter: string | null, logFps: boolean, + shard: string | null, ) { + let shardIndex = 0; + let shardTotal = 1; + if (shard) { + const match = /^(\d+)\/(\d+)$/.exec(shard); + if (match) { + shardIndex = Number(match[1]); + shardTotal = Number(match[2]); + } + } const logicalPixelRatio = defaultResolution / appHeight; const { renderer, appElement } = await initRenderer( renderMode, @@ -359,14 +373,17 @@ async function runAutomation( false, // enableInspector ); - // Iterate through all test modules - for (const testPath in testModules) { + // Iterate through all test modules. Sort so sharding is deterministic + // across pages, and apply the filter up front so the shard step indexes + // only the tests that will actually run. + const orderedPaths = Object.keys(testModules) + .sort() + .filter((p) => !filter || wildcardMatch(getTestName(p), filter)); + for (let i = 0; i < orderedPaths.length; i++) { + if (i % shardTotal !== shardIndex) continue; + const testPath = orderedPaths[i]!; const testModule = testModules[testPath]; const testName = getTestName(testPath); - // Skip tests that don't match the filter (if provided) - if (filter && !wildcardMatch(testName, filter)) { - continue; - } assertTruthy(testModule); // Setup Math.random to use a seeded random number generator for consistent diff --git a/visual-regression/src/index.ts b/visual-regression/src/index.ts index 244d218..263a645 100644 --- a/visual-regression/src/index.ts +++ b/visual-regression/src/index.ts @@ -93,6 +93,13 @@ const argv = yargs(hideBin(process.argv)) default: '*', description: 'Tests to run ("*" wildcard pattern)', }, + workers: { + type: 'number', + alias: 'w', + default: 1, + description: + 'Number of parallel browser pages used to run tests (sharded round-robin)', + }, }) .parseSync(); @@ -129,6 +136,7 @@ async function dockerCiMode(): Promise { argv.skipBuild ? '--skipBuild' : '', argv.port ? `--port ${argv.port}` : '', argv.filter ? `--filter "${argv.filter}"` : '', + argv.workers > 1 ? `--workers ${argv.workers}` : '', ].join(' '); // Get the directory of the current file @@ -252,7 +260,9 @@ async function runTest(browserType: 'chromium') { await fs.emptyDir(failedResultsDir); } - // Launch browser and create page + // Launch the browser once; each worker runs in its own Page sharing the + // same browser process. Pages run in parallel, each handling a round-robin + // shard of the test list (see runAutomation in examples/index.ts). const browser = await browsers[browserType].launch({ args: [ '--disable-font-subpixel-positioning', @@ -262,22 +272,14 @@ async function runTest(browserType: 'chromium') { ], }); - const page = await browser.newPage(); - - // If verbose, log out console messages from the browser - if (argv.verbose) { - page.on('console', (msg) => console.log(`console: ${msg.text()}`)); - } - - /** - * Keeps track of the latest snapshot index for each test - */ - const testCounters: Record = {}; + const workerCount = Math.max(1, argv.workers); - // Expose the `snapshot()` function to the browser - await page.exposeFunction( - 'snapshot', - async (test: string, options: SnapshotOptions) => { + // Each test's first snapshot is index 1, second is index 2, etc. With + // sharding, a given test only runs in one worker, so per-worker counters + // are sufficient. Snapshot filenames are still globally unique. + const makeSnapshotHandler = (page: import('playwright').Page) => { + const testCounters: Record = {}; + return async (test: string, options: SnapshotOptions) => { snapshotsTested++; // Ensure clip dimensions are integers (matches Playwright's clip shape @@ -345,101 +347,94 @@ async function runTest(browserType: 'chromium') { ), ); } - }, - ); + }; + }; - /** - * Resolve function for the donePromise below - */ - let resolveDonePromise: (exitCode: number) => void; - /** - * Promise that resolves when all tests are done - */ - const donePromise = new Promise((resolve) => { - resolveDonePromise = resolve; - }); + const runWorker = async (shardIndex: number) => { + const page = await browser.newPage(); - // Expose the `doneTests()` function to the browser - // which will close the browser, calculate/print results and resolve the donePromise - await page.exposeFunction('doneTests', async () => { - await browser.close(); - - // Summarize results - - const passPerc: string = ( - (snapshotsPassed / snapshotsTested) * - 100 - ).toFixed(1); - const failPerc: string = ( - (snapshotsFailed / snapshotsTested) * - 100 - ).toFixed(1); - const skipPerc: string = ( - (snapshotsSkipped / snapshotsTested) * - 100 - ).toFixed(1); - - if (argv.capture) { - console.log( - chalk.white.underline(`\nVisual Regression Test Capture Completed:`), + if (argv.verbose) { + page.on('console', (msg) => + console.log(`console[${shardIndex}]: ${msg.text()}`), ); + } - if (snapshotsPassed > 0) { - console.log( - chalk.green( - ` ${snapshotsPassed} snapshots captured (${passPerc}%)`, - ), - ); - } + await page.exposeFunction('snapshot', makeSnapshotHandler(page)); - if (snapshotsSkipped > 0) { - console.log( - chalk.yellow( - ` ${snapshotsSkipped} snapshots skipped (${skipPerc}%)`, - ), - ); - } + const donePromise = new Promise((resolve) => { + void page.exposeFunction('doneTests', () => { + resolve(); + }); + }); - console.log(chalk.gray(` ${snapshotsTested} snapshots detected`)); - } else { - console.log( - chalk.white.underline(`\nVisual Regression Tests Completed:`), - ); + const shardParam = + workerCount > 1 ? `&shard=${shardIndex}/${workerCount}` : ''; + await page.goto( + `http://localhost:${argv.port}/?automation=true&test=${argv.filter}${shardParam}`, + ); - if (snapshotsFailed > 0) { - console.log( - chalk.red(` ${snapshotsFailed} snapshots failed (${failPerc}%)`), - ); - console.log( - chalk.gray( - ` (See \`${failedResultsDir}\` directory for failed results)`, - ), - ); - } + await donePromise; + await page.close(); + }; - if (snapshotsPassed > 0) { - console.log( - chalk.green(` ${snapshotsPassed} snapshots passed (${passPerc}%)`), - ); - } + await Promise.all( + Array.from({ length: workerCount }, (_, i) => runWorker(i)), + ); + await browser.close(); + + // Summarize results + const passPerc: string = ((snapshotsPassed / snapshotsTested) * 100).toFixed( + 1, + ); + const failPerc: string = ((snapshotsFailed / snapshotsTested) * 100).toFixed( + 1, + ); + const skipPerc: string = ((snapshotsSkipped / snapshotsTested) * 100).toFixed( + 1, + ); - console.log(chalk.gray(` ${snapshotsTested} snapshots tested`)); + if (argv.capture) { + console.log( + chalk.white.underline(`\nVisual Regression Test Capture Completed:`), + ); + + if (snapshotsPassed > 0) { + console.log( + chalk.green(` ${snapshotsPassed} snapshots captured (${passPerc}%)`), + ); + } + + if (snapshotsSkipped > 0) { + console.log( + chalk.yellow(` ${snapshotsSkipped} snapshots skipped (${skipPerc}%)`), + ); } - // Extra new line - console.log(chalk.reset('')); + console.log(chalk.gray(` ${snapshotsTested} snapshots detected`)); + } else { + console.log(chalk.white.underline(`\nVisual Regression Tests Completed:`)); if (snapshotsFailed > 0) { - resolveDonePromise(1); - } else { - resolveDonePromise(0); + console.log( + chalk.red(` ${snapshotsFailed} snapshots failed (${failPerc}%)`), + ); + console.log( + chalk.gray( + ` (See \`${failedResultsDir}\` directory for failed results)`, + ), + ); } - }); - // Go to the examples page - await page.goto( - `http://localhost:${argv.port}/?automation=true&test=${argv.filter}`, - ); + if (snapshotsPassed > 0) { + console.log( + chalk.green(` ${snapshotsPassed} snapshots passed (${passPerc}%)`), + ); + } + + console.log(chalk.gray(` ${snapshotsTested} snapshots tested`)); + } + + console.log(chalk.reset('')); - return donePromise; + return snapshotsFailed > 0 ? 1 : 0; } From 4ef4d2d246082d5083b52082a742f103db5db474 Mon Sep 17 00:00:00 2001 From: Chris Lorenzo Date: Tue, 26 May 2026 18:14:52 -0400 Subject: [PATCH 2/2] ci: run visual regression with 4 workers Public-repo GitHub-hosted runners are 4-core / 16 GB, comfortably above the per-page memory and CPU budget for parallel Chromium SwiftShader contexts. Opting CI into -w 4 cuts the VRT step proportionally; if the text-rendering variance flagged in the PR description shows up, dial down to -w 2 (still ~2x over the baseline). Co-Authored-By: Claude Opus 4.7 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 18819ef..0a94ec1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,7 @@ jobs: run: cd visual-regression && pnpm exec playwright install chromium - name: Run Visual Regression Tests - run: pnpm run test:visual --color + run: pnpm run test:visual --color -w 4 env: RUNTIME_ENV: ci