Automate E2E timing matrix repair

docs/troubleshooting/repair-harness.md

@@ -89,19 +89,59 @@ or omit `--agent=codex` when you want to ensure no agent can run.
 
 ## E2E timing diagnostics
 
-The timing diagnostic reads an existing Playwright JSON report. It does not
-change Playwright workers, retries, timeouts, or test source:
+The timing diagnostic accepts either Playwright's JSON report or the `index.html`
+file from the downloaded GitHub Actions HTML report artifact. Playwright embeds
+`report.json` inside that HTML file, so there is no need to find or create a
+separate JSON file:
 
 ```bash
 npm run repair:ci -- e2e-timing \
-  --report playwright-report.json
+  --report .plans/index.html
 ```
 
+Replace `.plans/index.html` with the path where the artifact was downloaded.
+The command extracts only the embedded test results and writes normalized
+output under `.repair-harness/runs/<run-id>/`, including
+`e2e-timing-summary.md`, `e2e-timing-evidence.json`, and
+`e2e-timing-events.jsonl`.
+
+It does not change Playwright workers, retries, timeouts, or test source.
+
 Individual tests at or below 7 seconds are normal, tests above 7 seconds and
 through 10 seconds are reported as warnings, and tests above 10 seconds fail
 the diagnostic. The run directory contains redacted timing evidence grouped
 by project and test file.
 
+### Fully automated GitHub matrix repair
+
+To have the harness resolve the PR's failing Actions run, download every matrix
+artifact, merge the shard reports, ask Codex for a bounded timing repair, and
+independently verify it:
+
+```bash
+PHASERFORGE_CODEX_COMMAND=codex \\
+npm run repair:ci -- e2e-timing-repair \\
+  --pr 123 --agent=codex
+```
+
+Use `--run <run-id>` instead of `--pr` when the Actions run is known. The
+download is automatic; no artifact or `index.html` download is required. `--pr`
+also works when the E2E check passed, because timing failures are distinct from
+test assertion failures. The command stops after verification by default. Add
+`--publish` only when the
+verified change should be committed, pushed on an `agent/*` branch, and opened
+as a draft PR:
+
+```bash
+PHASERFORGE_CODEX_COMMAND=codex \\
+npm run repair:ci -- e2e-timing-repair \\
+  --pr 123 --agent=codex --publish
+```
+
+The command does not repair warnings (7–10 seconds); only tests over the
+10-second hard ceiling are eligible for an agent repair. It never changes
+workflow files, test retries, workers, or timeouts.
+
 ## Measure completed runs
 
 Aggregate redacted outcomes from local runs with:

scripts/repair-harness/cli.ts

@@ -12,6 +12,7 @@ import { aggregateRepairOutcomes, formatRepairMetrics, readRepairOutcomes, write
 import { loadHostedConfig } from './hosted/config';
 import { runHostedBrowser, runHostedIsolationCommand, runHostedMutationCommand, runHostedProbe } from './hosted/run';
 import { runE2ETiming } from './e2eTimingRun';
+import { runAutomatedTimingRepair } from './timingRepair';
 import { runHostedOAuthPreflight } from './hosted/oauth';
 import { assertHostedScope, assertRepairCannotUseHostedScope } from './scope';
 
@@ -35,14 +36,32 @@ async function main(): Promise<void> {
 if (args[0] === 'e2e-timing') {
   try {
     const reportPath = value('--report');
-    if (!reportPath) throw new Error('Expected --report <playwright-report.json> for e2e-timing.');
+    if (!reportPath) throw new Error('Expected --report <playwright-report.json|index.html> for e2e-timing.');
     const result = runE2ETiming({ repo: value('--repo') ?? repositoryRoot, reportPath: path.resolve(reportPath), runId: value('--run-id') });
     console.log(`E2E timing ${result.analysis.status} in ${result.runDirectory}`);
     if (result.analysis.status === 'failed' || result.analysis.status === 'invalid') process.exitCode = 1;
   } catch (error) {
     console.error(error instanceof Error ? error.message : String(error));
     process.exitCode = 1;
   }
+} else if (args[0] === 'e2e-timing-repair') {
+  try {
+    if (!value('--pr') && !value('--run')) throw new Error('Expected --pr <number> or --run <run-id> for e2e-timing-repair.');
+    if (value('--agent') !== 'codex') throw new Error('Automated timing repair requires explicit --agent=codex.');
+    const result = await runAutomatedTimingRepair({
+      repo: value('--repo') ?? repositoryRoot,
+      pr: value('--pr'),
+      run: value('--run'),
+      agent: 'codex',
+      publish: has('--publish'),
+    });
+    console.log(`E2E timing repair ${result.status} in ${result.runDirectory}`);
+    if (result.pullRequestUrl) console.log(`Pull request: ${result.pullRequestUrl}`);
+    if (result.status === 'failed') process.exitCode = 1;
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  }
 } else if (args[0] === 'hosted-probe') {
   try {
     const dryRun = prepareHosted('hosted-probe');

scripts/repair-harness/e2eTiming.ts

@@ -69,16 +69,51 @@ export interface E2ETimingAnalysis {
 }
 
 export function parsePlaywrightJsonReport(value: unknown): PlaywrightJsonReport {
-  if (!value || typeof value !== 'object' || !Array.isArray((value as { suites?: unknown }).suites)) {
+  if (!value || typeof value !== 'object') {
     throw new Error('Playwright JSON report must contain a suites array.');
   }
-  return value as PlaywrightJsonReport;
+  const report = value as { suites?: unknown; files?: unknown };
+  if (Array.isArray(report.suites)) return value as PlaywrightJsonReport;
+  if (Array.isArray(report.files)) return normalizePlaywrightFilesReport(value as PlaywrightFilesReport);
+  throw new Error('Playwright JSON report must contain a suites array.');
+}
+
+interface PlaywrightFilesReport { files: PlaywrightFile[]; [key: string]: unknown; }
+interface PlaywrightFile { fileName?: string; tests?: PlaywrightFileTest[]; }
+interface PlaywrightFileTest { title?: string; projectName?: string; duration?: number; outcome?: string; results?: PlaywrightJsonResult[]; }
+
+function normalizePlaywrightFilesReport(report: PlaywrightFilesReport): PlaywrightJsonReport {
+  return {
+    suites: report.files.map((file) => ({
+      file: safeText(file.fileName, '[unknown-file]'),
+      specs: (file.tests ?? []).map((test) => ({
+        title: test.title,
+        tests: [{
+          projectName: test.projectName,
+          results: [{ ...(test.results?.[0] ?? {}), duration: test.duration, status: test.outcome }],
+        }],
+      })),
+    })),
+  };
 }
 
 export function analyzeE2ETiming(value: unknown): E2ETimingAnalysis {
   const report = parsePlaywrightJsonReport(value);
   const entries: E2ETimingEntry[] = [];
   for (const suite of report.suites) collectSuite(suite, undefined, entries);
+  return summarizeE2ETiming(entries);
+}
+
+export function analyzeE2ETimingReports(values: unknown[]): E2ETimingAnalysis {
+  const entries: E2ETimingEntry[] = [];
+  for (const value of values) {
+    const report = parsePlaywrightJsonReport(value);
+    for (const suite of report.suites) collectSuite(suite, undefined, entries);
+  }
+  return summarizeE2ETiming(entries);
+}
+
+function summarizeE2ETiming(entries: E2ETimingEntry[]): E2ETimingAnalysis {
   const counts: Record<E2ETimingCategory, number> = { normal: 0, warning: 0, slow: 0, 'missing-duration': 0, 'invalid-duration': 0 };
   for (const entry of entries) counts[entry.category] += 1;
   const valid = entries.filter((entry): entry is E2ETimingEntry & { durationMs: number } => typeof entry.durationMs === 'number');

scripts/repair-harness/e2eTimingRun.ts

@@ -1,20 +1,89 @@
-import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
+import { inflateRawSync } from 'node:zlib';
 import path from 'node:path';
-import { analyzeE2ETiming, type E2ETimingAnalysis } from './e2eTiming';
+import { analyzeE2ETiming, analyzeE2ETimingReports, type E2ETimingAnalysis } from './e2eTiming';
 
 export interface E2ETimingRun { runId: string; runDirectory: string; analysis: E2ETimingAnalysis; }
 
 export function runE2ETiming(options: { repo: string; reportPath: string; runId?: string }): E2ETimingRun {
   const runId = options.runId ?? `e2e-timing-${Date.now()}`;
   const runDirectory = path.resolve(options.repo, '.repair-harness', 'runs', runId);
   mkdirSync(runDirectory, { recursive: true });
-  const analysis = analyzeE2ETiming(JSON.parse(readFileSync(options.reportPath, 'utf8')));
+  const reportPaths = discoverReportPaths(options.reportPath);
+  const analysis = reportPaths.length === 1
+    ? analyzeE2ETiming(readPlaywrightReport(reportPaths[0]))
+    : analyzeE2ETimingReports(reportPaths.map(readPlaywrightReport));
   writeFileSync(path.join(runDirectory, 'e2e-timing-evidence.json'), `${JSON.stringify({ version: 1, kind: 'e2e-timing-diagnostic', reportPath: path.relative(options.repo, options.reportPath), ...analysis }, null, 2)}\n`);
   writeFileSync(path.join(runDirectory, 'e2e-timing-events.jsonl'), analysis.entries.map((entry) => `${JSON.stringify({ event: 'e2e-test-timing', title: entry.title, project: entry.project, file: entry.file, retry: entry.retry, durationMs: entry.durationMs, outcome: entry.outcome, category: entry.category })}\n`).join(''));
   writeFileSync(path.join(runDirectory, 'e2e-timing-summary.md'), renderSummary(analysis));
   return { runId, runDirectory, analysis };
 }
 
+function discoverReportPaths(reportPath: string): string[] {
+  if (!statSync(reportPath).isDirectory()) return [reportPath];
+  const paths: string[] = [];
+  const visit = (directory: string): void => {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const fullPath = path.join(directory, entry.name);
+      if (entry.isDirectory()) visit(fullPath);
+      else if (entry.name === 'index.html' && fullPath.includes(`${path.sep}playwright-report${path.sep}`)) paths.push(fullPath);
+      else if (entry.name.endsWith('.json') && entry.name === 'report.json') paths.push(fullPath);
+    }
+  };
+  visit(reportPath);
+  if (!paths.length) throw new Error(`No Playwright index.html or report.json files found under ${reportPath}.`);
+  return paths.sort();
+}
+
+function readPlaywrightReport(reportPath: string): unknown {
+  const contents = readFileSync(reportPath);
+  if (path.extname(reportPath).toLowerCase() !== '.html') return JSON.parse(contents.toString('utf8'));
+
+  const html = contents.toString('utf8');
+  const match = html.match(/data:application\/zip;base64,([^<]+)/);
+  if (!match) throw new Error('Playwright HTML report does not contain an embedded results archive.');
+  const reportJson = readZipEntry(Buffer.from(match[1], 'base64'), 'report.json');
+  if (!reportJson) throw new Error('Playwright HTML report does not contain report.json.');
+  return JSON.parse(reportJson.toString('utf8'));
+}
+
+function readZipEntry(archive: Buffer, wantedName: string): Buffer | undefined {
+  const endOffset = findSignatureFromEnd(archive, 0x06054b50);
+  if (endOffset < 0) throw new Error('Embedded Playwright results archive is not a valid ZIP.');
+  const entryCount = archive.readUInt16LE(endOffset + 10);
+  const centralOffset = archive.readUInt32LE(endOffset + 16);
+  let offset = centralOffset;
+  for (let index = 0; index < entryCount; index += 1) {
+    if (archive.readUInt32LE(offset) !== 0x02014b50) throw new Error('Embedded Playwright results archive has an invalid directory.');
+    const compression = archive.readUInt16LE(offset + 10);
+    const compressedSize = archive.readUInt32LE(offset + 20);
+    const nameLength = archive.readUInt16LE(offset + 28);
+    const extraLength = archive.readUInt16LE(offset + 30);
+    const commentLength = archive.readUInt16LE(offset + 32);
+    const localOffset = archive.readUInt32LE(offset + 42);
+    const name = archive.toString('utf8', offset + 46, offset + 46 + nameLength);
+    if (name === wantedName) {
+      if (archive.readUInt32LE(localOffset) !== 0x04034b50) throw new Error(`ZIP entry ${wantedName} has an invalid local header.`);
+      const localNameLength = archive.readUInt16LE(localOffset + 26);
+      const localExtraLength = archive.readUInt16LE(localOffset + 28);
+      const dataStart = localOffset + 30 + localNameLength + localExtraLength;
+      const data = archive.subarray(dataStart, dataStart + compressedSize);
+      if (compression === 0) return data;
+      if (compression === 8) return inflateRawSync(data);
+      throw new Error(`ZIP entry ${wantedName} uses unsupported compression.`);
+    }
+    offset += 46 + nameLength + extraLength + commentLength;
+  }
+  return undefined;
+}
+
+function findSignatureFromEnd(buffer: Buffer, signature: number): number {
+  for (let offset = buffer.length - 22; offset >= 0; offset -= 1) {
+    if (buffer.readUInt32LE(offset) === signature) return offset;
+  }
+  return -1;
+}
+
 function renderSummary(analysis: E2ETimingAnalysis): string {
   const lines = [
     '# E2E timing diagnostic',

scripts/repair-harness/github.ts

@@ -1,4 +1,5 @@
 import { spawnSync } from 'node:child_process';
+import { mkdirSync } from 'node:fs';
 
 import { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields } from './ghHelpers';
 
@@ -68,13 +69,19 @@ export function resolveRun(runId: string, repo: string, jobName?: string): Resol
   return { metadata, job, log: logResult.stdout || logResult.stderr || '', artifacts };
 }
 
-export function resolveRunFromPr(pr: string, repo: string): { runId: string; check: JsonRecord } {
-  const check = fetchChecks(pr, repo).find(isFailingCheck);
+export function resolveRunFromPr(pr: string, repo: string, options: { allowPassingE2E?: boolean } = {}): { runId: string; check: JsonRecord } {
+  const checks = fetchChecks(pr, repo);
+  const check = checks.find(isFailingCheck) ?? (options.allowPassingE2E ? checks.find((item) => /e2e|playwright/i.test(String(item.name ?? ''))) : undefined);
   if (!check) throw new Error(`PR #${pr}: no failing GitHub Actions checks detected.`);
   const url = String(check.detailsUrl ?? check.link ?? '');
   const runId = extractRunIdFromUrl(url);
   if (!runId) throw new Error(`Unsupported external check: ${String(check.name ?? 'Unnamed check')}`);
   return { runId, check };
 }
 
+export function downloadRunArtifacts(runId: string, destination: string, repo: string): void {
+  mkdirSync(destination, { recursive: true });
+  runGh(['run', 'download', runId, '--dir', destination], { cwd: repo });
+}
+
 export { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields };

scripts/repair-harness/timingRepair.ts

@@ -0,0 +1,90 @@
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+import { downloadRunArtifacts, resolveRun, resolveRunFromPr } from './github';
+import { runE2ETiming } from './e2eTimingRun';
+import { runBoundedRepair, type RepairResult } from './repair';
+import { appendEvent, writeState } from './state';
+import type { EvidenceEnvelope } from './types';
+
+export interface AutomatedTimingOptions {
+  repo: string;
+  pr?: string;
+  run?: string;
+  agent: 'codex';
+  publish: boolean;
+}
+
+export interface AutomatedTimingResult {
+  sourceRunId: string;
+  runDirectory: string;
+  status: 'passed' | 'warning' | 'failed' | 'repaired' | 'published';
+  analysis: ReturnType<typeof runE2ETiming>['analysis'];
+  repair?: RepairResult;
+  pullRequestUrl?: string;
+}
+
+export async function runAutomatedTimingRepair(options: AutomatedTimingOptions): Promise<AutomatedTimingResult> {
+  if (options.publish && execFileSync('git', ['status', '--porcelain'], { cwd: options.repo, encoding: 'utf8' }).trim()) {
+    throw new Error('Refusing --publish with pre-existing working-tree changes; start from a clean checkout.');
+  }
+  const source = options.run ? { runId: options.run } : resolveRunFromPr(options.pr!, options.repo, { allowPassingE2E: true });
+  const resolved = resolveRun(source.runId, options.repo);
+  const runDirectory = path.resolve(options.repo, '.repair-harness', 'runs', `timing-repair-${source.runId}-${Date.now()}`);
+  const artifactDirectory = path.join(runDirectory, 'github-artifacts');
+  mkdirSync(runDirectory, { recursive: true });
+  downloadRunArtifacts(source.runId, artifactDirectory, options.repo);
+  const timing = runE2ETiming({ repo: options.repo, reportPath: artifactDirectory, runId: path.basename(runDirectory) });
+  const analysis = timing.analysis;
+  if (analysis.status === 'warning' || analysis.status === 'passed') return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: analysis.status, analysis };
+
+  const slow = analysis.entries.filter((entry) => entry.category === 'slow');
+  const first = slow[0];
+  const boundedSlow = slow.slice(0, 20);
+  const job = String(resolved.job.name ?? resolved.metadata.workflowName ?? 'GitHub Actions E2E matrix');
+  const command = commandForJob(job);
+  const evidence: EvidenceEnvelope = {
+    workflow: String(resolved.metadata.workflowName ?? resolved.metadata.name ?? 'GitHub Actions'),
+    job,
+    runId: source.runId,
+    commit: String(resolved.metadata.headSha ?? ''),
+    scope: 'e2e-timing-repair',
+    reproduction: { command },
+    failure: {
+      class: 'timeout',
+      testFile: first.file,
+      testTitle: first.title,
+      message: `${slow.length} tests exceeded the hard ceiling.\n` + boundedSlow.map((entry) => `${entry.title} — ${entry.project} — ${entry.file} — ${entry.durationMs}ms`).join('\n'),
+      stackExcerpt: `Playwright timing diagnostic exceeded the 10,000ms hard ceiling. Showing ${boundedSlow.length} of ${slow.length} slow tests.\n` + boundedSlow.map((entry) => `${entry.file}: ${entry.title} (${entry.durationMs}ms)`).join('\n'),
+    },
+    artifacts: { tracePaths: [], screenshotPaths: [] },
+    redactionsApplied: ['GitHub artifact contents were reduced to timing fields'],
+  };
+  writeFileSync(path.join(timing.runDirectory, 'evidence.json'), `${JSON.stringify(evidence, null, 2)}\n`);
+  writeState(timing.runDirectory, { runId: path.basename(timing.runDirectory), phase: 'timing', status: 'active', budgets: { diagnosisCalls: 0, implementationAttempts: 0 }, scope: 'e2e-timing-repair', updatedAt: new Date().toISOString() });
+  appendEvent(timing.runDirectory, { event: 'github-matrix-downloaded', sourceRunId: source.runId, slowTests: slow.length });
+  if (!options.agent) throw new Error('Automated timing repair requires --agent=codex.');
+  const repair = await runBoundedRepair({ repo: options.repo, runDirectory: timing.runDirectory, evidence });
+  if (repair.status !== 'verified') return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: 'failed', analysis, repair };
+  if (!options.publish) return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: 'repaired', analysis, repair };
+  const pullRequestUrl = publishRepair(options.repo, source.runId);
+  return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: 'published', analysis, repair, pullRequestUrl };
+}
+
+function commandForJob(job: string): string {
+  const shard = job.match(/shard\s+(\d+)\s*\/\s*(\d+)/i);
+  if (/full matrix/i.test(job)) return 'npm run test:e2e -- --project=firefox --project=webkit --project=msedge --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1');
+  return 'npm run test:e2e -- --project=chromium --grep "@smoke|@critical" --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1');
+}
+
+function publishRepair(repo: string, sourceRunId: string): string {
+  const status = execFileSync('git', ['status', '--porcelain'], { cwd: repo, encoding: 'utf8' });
+  if (!status.trim()) throw new Error('Verified timing repair produced no working-tree changes to publish.');
+  const branch = `agent/e2e-timing-${sourceRunId}`;
+  execFileSync('git', ['switch', '-c', branch], { cwd: repo, stdio: 'inherit' });
+  execFileSync('git', ['add', '-A'], { cwd: repo, stdio: 'inherit' });
+  execFileSync('git', ['commit', '-m', 'Fix slow E2E test'], { cwd: repo, stdio: 'inherit' });
+  execFileSync('git', ['push', '--set-upstream', 'origin', branch], { cwd: repo, stdio: 'inherit' });
+  return execFileSync('gh', ['pr', 'create', '--draft', '--title', 'Fix slow E2E test', '--body', `Automated repair from GitHub Actions run ${sourceRunId}.\n\nTiming evidence and independent verification are recorded in the repair run.`], { cwd: repo, encoding: 'utf8' }).trim();
+}