diff --git a/.github/screenshots/deploy-after.png b/.github/screenshots/deploy-after.png new file mode 100644 index 0000000..45825cd Binary files /dev/null and b/.github/screenshots/deploy-after.png differ diff --git a/.github/screenshots/deploy-before.png b/.github/screenshots/deploy-before.png new file mode 100644 index 0000000..45825cd Binary files /dev/null and b/.github/screenshots/deploy-before.png differ diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts index 0d7cc10..3105d3d 100644 --- a/website/src/benchmark-results.ts +++ b/website/src/benchmark-results.ts @@ -424,26 +424,33 @@ export function getBenchmarkOverviewData(runs: Array): BenchmarkOv } } +async function readBenchmarkRun(benchmarkId: string, candidate: OutputCandidate): Promise { + const output = await readBenchmarkOutput(candidate.filepath) + if (output === null) { + return null + } + if (output.id !== benchmarkId) { + throw new Error(`Benchmark ID "${output.id}" does not match "${benchmarkId}" in ${candidate.filepath}`) + } + return { + id: candidate.date, + name: candidate.date, + directory: path.dirname(candidate.filepath), + date: new Date(`${candidate.date}T00:00:00.000Z`), + output, + } +} + export async function listBenchmarkRuns(benchmarkId: string): Promise> { const candidates = (await getDatedCandidates(benchmarkId)).toSorted((a, b) => { return b.date.localeCompare(a.date) }) const runs: Array = [] for (const candidate of candidates) { - const output = await readBenchmarkOutput(candidate.filepath) - if (output === null) { - continue - } - if (output.id !== benchmarkId) { - throw new Error(`Benchmark ID "${output.id}" does not match "${benchmarkId}" in ${candidate.filepath}`) + const run = await readBenchmarkRun(benchmarkId, candidate) + if (run) { + runs.push(run) } - runs.push({ - id: candidate.date, - name: candidate.date, - directory: path.dirname(candidate.filepath), - date: new Date(`${candidate.date}T00:00:00.000Z`), - output, - }) } return runs } @@ -452,12 +459,10 @@ export async function getBenchmarkRun(benchmarkId: string, date: string): Promis if (!isRunDate(date)) { return null } - const runs = await listBenchmarkRuns(benchmarkId) - return ( - runs.find(run => { - return run.name === date - }) ?? null - ) + const candidate = (await getDatedCandidates(benchmarkId)).find(candidate => { + return candidate.date === date + }) + return candidate ? readBenchmarkRun(benchmarkId, candidate) : null } function createScenarioResults( diff --git a/website/src/result-files.test.ts b/website/src/result-files.test.ts index 0cdc8a7..ddda3be 100644 --- a/website/src/result-files.test.ts +++ b/website/src/result-files.test.ts @@ -1,4 +1,5 @@ import fs from 'node:fs/promises' +import os from 'node:os' import path from 'node:path' import type {BenchmarkOutput, ExperimentOutput} from '@primer/agent-eval' import {afterEach, expect, onTestFinished, test, vi} from 'vitest' @@ -8,12 +9,11 @@ import {createExperimentRunDetails, getWalkthroughAssets} from './run-details' afterEach(() => { vi.restoreAllMocks() + vi.unstubAllEnvs() }) async function createDirectory(): Promise { - const root = path.resolve('.agents/tmp') - await fs.mkdir(root, {recursive: true}) - const directory = await fs.mkdtemp(path.join(root, 'website-results-')) + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'website-results-')) onTestFinished(async () => { await fs.rm(directory, {recursive: true, force: true}) }) @@ -46,6 +46,101 @@ async function writeBundle( return filepath } +test('getBenchmarkRun reads only the requested date, not broken historical bundles', async () => { + const directory = await createDirectory() + const output = createBenchmarkOutput() + const results = path.join(directory, 'results/benchmarks', output.id) + await writeBundle(path.join(results, '2026-09-15'), output) + const broken = await writeBundle(path.join(results, '2026-09-14'), output) + await fs.rm(path.join(path.dirname(broken), 'artifacts'), {recursive: true}) + vi.spyOn(process, 'cwd').mockReturnValue(path.join(directory, 'website')) + vi.resetModules() + const {getBenchmarkRun} = await import('./benchmark-results') + + expect(await getBenchmarkRun(output.id, '2026-09-15')).toMatchObject({name: '2026-09-15', output}) + expect(await getBenchmarkRun(output.id, '2026-09-16')).toBeNull() + expect(await getBenchmarkRun(output.id, '2026-02-30')).toBeNull() + await expect(getBenchmarkRun(output.id, '2026-09-14')).rejects.toThrow('ENOENT') +}) + +test.each(['benchmark', 'experiment'] as const)( + 'reuses concurrent and repeated production %s bundle reads', + async kind => { + const directory = await createDirectory() + const output = kind === 'benchmark' ? createBenchmarkOutput() : createExperimentOutput() + const filepath = await writeBundle(directory, output) + vi.stubEnv('NODE_ENV', 'production') + vi.resetModules() + const readers = await import('./result-files') + const read = kind === 'benchmark' ? readers.readBenchmarkOutput : readers.readExperimentOutput + const readFile = vi.spyOn(fs, 'readFile') + + expect(await Promise.all([read(filepath), read(filepath), read(filepath)])).toEqual([output, output, output]) + expect(await read(filepath)).toEqual(output) + expect(readFile).toHaveBeenCalledTimes(2) + }, +) + +test.each(['benchmark', 'experiment'] as const)( + 'bounds the production %s bundle cache and keeps recently used runs', + async kind => { + const directory = await createDirectory() + const output = kind === 'benchmark' ? createBenchmarkOutput() : createExperimentOutput() + const first = await writeBundle(path.join(directory, 'first'), output) + const second = await writeBundle(path.join(directory, 'second'), output) + const third = await writeBundle(path.join(directory, 'third'), output) + vi.stubEnv('NODE_ENV', 'production') + vi.resetModules() + const readers = await import('./result-files') + const read = kind === 'benchmark' ? readers.readBenchmarkOutput : readers.readExperimentOutput + await read(first) + await read(second) + await read(first) + await read(third) + const readFile = vi.spyOn(fs, 'readFile') + + expect(await read(first)).toEqual(output) + expect(readFile).not.toHaveBeenCalled() + expect(await read(second)).toEqual(output) + expect(readFile).toHaveBeenCalledTimes(2) + }, +) + +test.each(['benchmark', 'experiment'] as const)( + 'retries failed and incompatible production %s bundle reads', + async kind => { + const directory = await createDirectory() + const output = kind === 'benchmark' ? createBenchmarkOutput() : createExperimentOutput() + const filepath = path.join(directory, 'output.json') + vi.stubEnv('NODE_ENV', 'production') + vi.resetModules() + const readers = await import('./result-files') + const read = kind === 'benchmark' ? readers.readBenchmarkOutput : readers.readExperimentOutput + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await expect(read(filepath)).rejects.toThrow('ENOENT') + await fs.writeFile(filepath, '{}') + expect(await read(filepath)).toBeNull() + await writeBundle(directory, output) + expect(await read(filepath)).toEqual(output) + }, +) + +test.each(['benchmark', 'experiment'] as const)('rereads changed %s bundles outside production', async kind => { + const directory = await createDirectory() + const output = kind === 'benchmark' ? createBenchmarkOutput() : createExperimentOutput() + const filepath = await writeBundle(directory, output) + vi.stubEnv('NODE_ENV', 'development') + vi.resetModules() + const readers = await import('./result-files') + const read = kind === 'benchmark' ? readers.readBenchmarkOutput : readers.readExperimentOutput + expect(await read(filepath)).toEqual(output) + output.trials.clear() + await writeBundle(directory, output) + + expect(await read(filepath)).toEqual(output) +}) + test.each(['benchmark', 'experiment'] as const)( 'reads a portable %s bundle repeatedly without deleting artifacts', async kind => { diff --git a/website/src/result-files.ts b/website/src/result-files.ts index 1b6861d..6f4f664 100644 --- a/website/src/result-files.ts +++ b/website/src/result-files.ts @@ -17,6 +17,45 @@ type ResultSchema = { safeParse: (json: unknown) => {success: true; data: T} | {success: false; error: {message: string}} } +function cacheResultReader(read: (filepath: string) => Promise) { + // Static exports reuse immutable bundles across thousands of asset routes. + const bundles = new Map>() + const maxCachedBundles = 2 + + return async (filepath: string): Promise => { + if (process.env.NODE_ENV !== 'production') { + return read(filepath) + } + const key = path.resolve(filepath) + const cached = bundles.get(key) + if (cached) { + bundles.delete(key) + bundles.set(key, cached) + return cached + } + const bundle = read(key) + bundles.set(key, bundle) + if (bundles.size > maxCachedBundles) { + const oldestKey = bundles.keys().next().value + if (oldestKey !== undefined) { + bundles.delete(oldestKey) + } + } + try { + const output = await bundle + if (output === null && bundles.get(key) === bundle) { + bundles.delete(key) + } + return output + } catch (error) { + if (bundles.get(key) === bundle) { + bundles.delete(key) + } + throw error + } + } +} + async function readResultFile(filepath: string, schema: ResultSchema): Promise { const contents = await fs.readFile(filepath, 'utf8') let json: unknown @@ -65,7 +104,7 @@ async function readTrials( return results } -async function readBenchmarkOutput(filepath: string): Promise { +async function loadBenchmarkOutput(filepath: string): Promise { const file = await readResultFile(filepath, BenchmarkOutputFileSchema) if (file === null) { return null @@ -87,7 +126,7 @@ async function readBenchmarkOutput(filepath: string): Promise { +async function loadExperimentOutput(filepath: string): Promise { const file = await readResultFile(filepath, ExperimentOutputFileSchema) if (file === null) { return null @@ -104,4 +143,7 @@ async function readExperimentOutput(filepath: string): Promise