Harness repair

docs/troubleshooting/repair-harness.md

@@ -16,6 +16,28 @@ change GitHub secrets, or modify hosted databases.
 
 ## Collect a failed PR job
 
+Pass `--clean` when starting a new harness run to remove existing `.log`
+files anywhere under `.repair-harness/runs` before the run begins. Other run
+artifacts, including evidence and state files, are preserved:
+
+```bash
+npm run repair:ci:collect -- --pr 123 --clean
+```
+
+The option is also supported by the timing and hosted run commands. Do not use
+it when resuming a run whose logs you still need.
+
+To remove every file under `.repair-harness` and the complete
+`.repair-harness/runs` directory tree, including evidence, state, metrics, and
+logs, use the stronger `--clean-all` option when starting a new run:
+
+```bash
+npm run repair:ci:collect -- --pr 123 --clean-all
+```
+
+If both flags are supplied, `--clean-all` takes precedence. Do not use either
+cleanup option when resuming a run.
+
 For a pull request, collect the failing GitHub Actions check:
 
 ```bash

scripts/repair-harness/cli.ts

@@ -6,7 +6,7 @@ import { collectEvidence } from './collect';
 import { expectedEvidenceFingerprint, reproduce } from './reproduce';
 import { verify } from './verify';
 import { runBoundedRepair } from './repair';
-import { appendEvent, readState, resolveResumeDirectory, writeState } from './state';
+import { appendEvent, cleanAllHarnessFiles, cleanRunLogs, readState, resolveResumeDirectory, writeState } from './state';
 import type { EvidenceEnvelope } from './types';
 import { aggregateRepairOutcomes, formatRepairMetrics, readRepairOutcomes, writeRepairMetrics } from './metrics';
 import { loadHostedConfig } from './hosted/config';
@@ -26,6 +26,10 @@ const value = (name: string): string | undefined => {
 };
 
 const has = (name: string): boolean => args.includes(name);
+const cleanIfRequested = (repo: string): void => {
+  if (has('--clean-all')) cleanAllHarnessFiles(repo);
+  else if (has('--clean')) cleanRunLogs(repo);
+};
 const prepareHosted = (command: string): boolean => {
   assertHostedScope(command, value('--scope'));
   if (value('--agent')) throw new Error('Hosted validation cannot invoke an agent; use --no-agent.');
@@ -37,7 +41,9 @@ if (args[0] === 'e2e-timing') {
   try {
     const reportPath = value('--report');
     if (!reportPath) throw new Error('Expected --report <playwright-report.json|index.html> for e2e-timing.');
-    const result = runE2ETiming({ repo: value('--repo') ?? repositoryRoot, reportPath: path.resolve(reportPath), runId: value('--run-id') });
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
+    const result = runE2ETiming({ repo, reportPath: path.resolve(reportPath), runId: value('--run-id') });
     console.log(`E2E timing ${result.analysis.status} in ${result.runDirectory}`);
     if (result.analysis.status === 'failed' || result.analysis.status === 'invalid') process.exitCode = 1;
   } catch (error) {
@@ -47,8 +53,10 @@ if (args[0] === 'e2e-timing') {
 } else if (args[0] === 'e2e-timing-repair') {
   try {
     if (!value('--pr') && !value('--run')) throw new Error('Expected --pr <number> or --run <run-id> for e2e-timing-repair.');
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
     const result = await runAutomatedTimingRepair({
-      repo: value('--repo') ?? repositoryRoot,
+      repo,
       pr: value('--pr'),
       run: value('--run'),
       agent: 'codex',
@@ -69,7 +77,9 @@ if (args[0] === 'e2e-timing') {
     if (!configPath) throw new Error('Expected --config <path> for hosted-probe.');
     const config = loadHostedConfig(path.resolve(configPath));
     if (dryRun) { console.log('Hosted probe dry-run: configuration validated; no network request will run.'); return; }
-    const result = await runHostedProbe({ repo: value('--repo') ?? repositoryRoot, config, runId: value('--run-id') });
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
+    const result = await runHostedProbe({ repo, config, runId: value('--run-id') });
     console.log(`Hosted probe ${result.status} in ${result.runDirectory}`);
     if (result.status !== 'passed') process.exitCode = 1;
   } catch (error) {
@@ -83,7 +93,9 @@ if (args[0] === 'e2e-timing') {
     if (!configPath) throw new Error('Expected --config <path> for hosted-browser.');
     const config = loadHostedConfig(path.resolve(configPath));
     if (dryRun) { console.log('Hosted browser dry-run: configuration validated; no browser or network request will run.'); return; }
-    const result = await runHostedBrowser({ repo: value('--repo') ?? repositoryRoot, config, runId: value('--run-id') });
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
+    const result = await runHostedBrowser({ repo, config, runId: value('--run-id') });
     console.log(`Hosted browser ${result.status} in ${result.runDirectory}`);
     if (result.status !== 'passed') process.exitCode = 1;
   } catch (error) {
@@ -100,8 +112,10 @@ if (args[0] === 'e2e-timing') {
     if (!email || !password) throw new Error('Hosted mutation requires --email and --password; credentials are used in memory only.');
     const config = loadHostedConfig(path.resolve(configPath));
     if (dryRun) { console.log('Hosted mutation dry-run: configuration and credentials presence validated; no browser or mutation will run.'); return; }
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
     const result = await runHostedMutationCommand({
-      repo: value('--repo') ?? repositoryRoot,
+      repo,
       config,
       runId: value('--run-id'),
       account: { email, password, inviteToken: value('--invite-token') },
@@ -126,8 +140,10 @@ if (args[0] === 'e2e-timing') {
     if (!devEmail || !devPassword || !stableEmail || !stablePassword) throw new Error('Hosted isolation requires separate --dev-email/--dev-password and --stable-email/--stable-password credentials; credentials are used in memory only.');
     const config = loadHostedConfig(path.resolve(configPath));
     if (dryRun) { console.log('Hosted isolation dry-run: configuration and credential presence validated; no browser or mutation will run.'); return; }
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
     const result = await runHostedIsolationCommand({
-      repo: value('--repo') ?? repositoryRoot,
+      repo,
       config,
       runId: value('--run-id'),
       accounts: { dev: { email: devEmail, password: devPassword }, stable: { email: stableEmail, password: stablePassword } },
@@ -166,7 +182,9 @@ if (args[0] === 'e2e-timing') {
   }
 } else if (args[0] === 'collect') {
   try {
-    const result = collectEvidence({ repo: value('--repo') ?? repositoryRoot, pr: value('--pr'), run: value('--run'), job: value('--job'), outputRoot: value('--output-root') });
+    const repo = value('--repo') ?? repositoryRoot;
+    cleanIfRequested(repo);
+    const result = collectEvidence({ repo, pr: value('--pr'), run: value('--run'), job: value('--job'), outputRoot: value('--output-root') });
     console.log(`Collected ${result.envelope.failure.class} evidence in ${result.runDirectory}`);
   } catch (error) {
     console.error(error instanceof Error ? error.message : String(error));

scripts/repair-harness/packet.ts

@@ -51,7 +51,9 @@ export function createDiagnosisPacket(options: PacketOptions): string {
 export function createImplementationPacket(options: PacketOptions): string {
   if (!options.diagnosis) throw new Error('Implementation packets require a diagnosis.');
   const files = options.targetedFiles ?? options.diagnosis.files;
-  const diff = (options.diff ?? '').slice(0, options.maxDiffChars ?? 24_000);
+  // Keep the packet within the implementation input budget after adding
+  // repository guidance and timing evidence.
+  const diff = (options.diff ?? '').slice(0, options.maxDiffChars ?? 16_000);
   return redactSecrets([
     '# PhaserForge CI repair implementation',
     'Implement only the approved diagnosis. Do not commit, push, merge, deploy, edit workflows, weaken tests, or change secrets/configuration.',

scripts/repair-harness/repair.ts

@@ -73,7 +73,9 @@ export async function runBoundedRepair(options: RepairOptions): Promise<RepairRe
   const implementationBudgetViolation = tokenBudgetViolation('implementation', implementationResult);
   if (implementationBudgetViolation) { const reason = implementationBudgetViolation; outcome(options, 'stopped', 1, reason); return stop(options.runDirectory, state, reason); }
   if (implementationResult.exitCode !== 0) { const reason = 'Implementation command failed.'; outcome(options, 'failed', 1, reason); return { status: 'failed', diagnosis, reason }; }
-  const verification = await (options.verifyPatch ?? (() => verify({ evidence: options.evidence, cwd: options.repo })));
+  const verification = options.verifyPatch
+    ? await options.verifyPatch()
+    : await verify({ evidence: options.evidence, cwd: options.repo });
   if (!verification) {
     const reason = 'Verification adapter returned no result.';
     mkdirSync(path.join(options.runDirectory, 'verification'), { recursive: true });

scripts/repair-harness/state.ts

@@ -1,11 +1,44 @@
-import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 
 import { redactSecrets } from './artifacts';
 
 export interface RepairState { runId: string; phase: string; status: string; budgets: Record<string, number>; scope?: string; updatedAt: string; }
 export interface RepairEvent { event: string; at: string; [key: string]: unknown; }
 
+export function cleanRunLogs(repo: string): void {
+  const runsRoot = path.resolve(repo, '.repair-harness', 'runs');
+  if (!existsSync(runsRoot)) return;
+
+  const visit = (directory: string): void => {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const entryPath = path.join(directory, entry.name);
+      if (entry.isDirectory()) visit(entryPath);
+      else if (entry.isFile() && entry.name.endsWith('.log')) unlinkSync(entryPath);
+    }
+  };
+
+  visit(runsRoot);
+}
+
+export function cleanAllHarnessFiles(repo: string): void {
+  const harnessRoot = path.resolve(repo, '.repair-harness');
+  if (!existsSync(harnessRoot)) return;
+
+  const runsRoot = path.join(harnessRoot, 'runs');
+  if (existsSync(runsRoot)) rmSync(runsRoot, { recursive: true, force: true });
+
+  const visit = (directory: string): void => {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const entryPath = path.join(directory, entry.name);
+      if (entry.isDirectory()) visit(entryPath);
+      else if (entry.isFile()) unlinkSync(entryPath);
+    }
+  };
+
+  visit(harnessRoot);
+}
+
 export function createRun(repo: string, runId = `${Date.now()}`): { directory: string; state: RepairState } {
   const directory = path.resolve(repo, '.repair-harness', 'runs', runId);
   mkdirSync(directory, { recursive: true });

scripts/repair-harness/timingRepair.ts

@@ -43,7 +43,6 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions):
 
   const slow = analysis.entries.filter((entry) => entry.category === 'slow');
   const first = slow[0];
-  const boundedSlow = slow.slice(0, 20);
   const job = String(resolved.job.name ?? resolved.metadata.workflowName ?? 'GitHub Actions E2E matrix');
   const command = commandForJob(job);
   const evidence: EvidenceEnvelope = {
@@ -57,8 +56,8 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions):
       class: 'timeout',
       testFile: first.file,
       testTitle: first.title,
-      message: `${slow.length} tests exceeded the hard ceiling.\n` + boundedSlow.map((entry) => `${entry.title} — ${entry.project} — ${entry.file} — ${entry.durationMs}ms`).join('\n'),
-      stackExcerpt: `Playwright timing diagnostic exceeded the 10,000ms hard ceiling. Showing ${boundedSlow.length} of ${slow.length} slow tests.\n` + boundedSlow.map((entry) => `${entry.file}: ${entry.title} (${entry.durationMs}ms)`).join('\n'),
+      message: formatSlowTestEvidence(slow),
+      stackExcerpt: formatSlowTestEvidence(slow),
     },
     artifacts: { tracePaths: [], screenshotPaths: [] },
     redactionsApplied: ['GitHub artifact contents were reduced to timing fields'],
@@ -74,9 +73,27 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions):
   return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: 'published', analysis, repair, pullRequestUrl };
 }
 
-function commandForJob(job: string): string {
+export function formatSlowTestEvidence(slow: Array<{ title: string; project: string; file: string; durationMs?: number }>): string {
+  const groups = new Map<string, { project: string; file: string; count: number; fastestMs: number; slowestMs: number }>();
+  for (const entry of slow) {
+    const key = `${entry.project}\u0000${entry.file}`;
+    const durationMs = entry.durationMs ?? 0;
+    const group = groups.get(key) ?? { project: entry.project, file: entry.file, count: 0, fastestMs: durationMs, slowestMs: durationMs };
+    group.count += 1;
+    group.fastestMs = Math.min(group.fastestMs, durationMs);
+    group.slowestMs = Math.max(group.slowestMs, durationMs);
+    groups.set(key, group);
+  }
+  const inventory = [...groups.values()]
+    .sort((left, right) => right.slowestMs - left.slowestMs)
+    .map((group) => `- ${group.project} — ${group.file} — ${group.count} slow (${group.fastestMs}-${group.slowestMs}ms)`)
+    .join('\n');
+  return `${slow.length} tests exceeded the hard ceiling across ${groups.size} project/file groups. Full normalized test-level inventory is in e2e-timing-evidence.json.\n${inventory}`;
+}
+
+export function commandForJob(job: string): string {
   const shard = job.match(/shard\s+(\d+)\s*\/\s*(\d+)/i);
-  if (/full matrix/i.test(job)) return 'npm run test:e2e -- --project=firefox --project=webkit --project=msedge --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1');
+  if (/full matrix/i.test(job)) return 'PW_PROJECTS=firefox,webkit,msedge npm run test:e2e -- --project=firefox --project=webkit --project=msedge --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1');
   return 'npm run test:e2e -- --project=chromium --grep "@smoke|@critical" --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1');
 }
 

scripts/repair-harness/verify.ts

@@ -1,4 +1,5 @@
 import type { EvidenceEnvelope } from './types';
+import { analyzeE2ETiming, type E2ETimingAnalysis } from './e2eTiming';
 import { expectedEvidenceFingerprint, focusedReproductionCommand, reproduce, type ReproductionResult } from './reproduce';
 
 export interface VerificationOptions {
@@ -14,23 +15,48 @@ export interface VerificationResult {
   focused?: ReproductionResult;
   requiredCommand: string;
   required?: ReproductionResult;
+  timing?: Pick<E2ETimingAnalysis, 'status' | 'counts' | 'slowest'>;
   reason?: string;
 }
 
 export async function verify(options: VerificationOptions): Promise<VerificationResult> {
   const run = options.run ?? ((command, cwd, timeoutMs) => reproduce({ command, cwd, timeoutMs }));
-  const focusedCommand = options.evidence.failure.testFile ? focusedReproductionCommand(options.evidence) : undefined;
+  const candidateFocusedCommand = options.evidence.failure.testFile ? focusedReproductionCommand(options.evidence) : undefined;
+  const focusedCommand = candidateFocusedCommand === options.evidence.reproduction.command ? undefined : candidateFocusedCommand;
   let focused: ReproductionResult | undefined;
   if (focusedCommand) {
     focused = await run(focusedCommand, options.cwd, options.timeoutMs);
     if (focused.status !== 'passed') return { verified: false, focusedCommand, focused, requiredCommand: options.evidence.reproduction.command, reason: 'Focused verification failed.' };
   }
-  const requiredCommand = options.evidence.reproduction.command;
+  const isTimingRepair = options.evidence.scope === 'e2e-timing-repair';
+  const requiredCommand = isTimingRepair ? `${options.evidence.reproduction.command} --reporter=json` : options.evidence.reproduction.command;
   const required = await run(requiredCommand, options.cwd, options.timeoutMs);
+  if (isTimingRepair && required.status === 'passed') {
+    try {
+      const analysis = analyzeE2ETiming(parseJsonReporterOutput(required.stdout));
+      const timing = { status: analysis.status, counts: analysis.counts, slowest: analysis.slowest };
+      if (analysis.status === 'failed' || analysis.status === 'invalid') {
+        return { verified: false, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), timing, reason: 'Timing verification still exceeds the hard ceiling or has invalid durations.' };
+      }
+      return { verified: true, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), timing };
+    } catch (error) {
+      return { verified: false, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), reason: `Timing verification did not produce a readable Playwright JSON report: ${error instanceof Error ? error.message : String(error)}` };
+    }
+  }
   return { verified: required.status === 'passed', focusedCommand, focused, requiredCommand, required,
     reason: required.status === 'passed' ? undefined : 'Required verification failed.' };
 }
 
+function parseJsonReporterOutput(stdout: string): unknown {
+  const start = stdout.indexOf('{');
+  if (start < 0) throw new Error('JSON object not found in test output.');
+  return JSON.parse(stdout.slice(start));
+}
+
+function redactTimingReport(result: ReproductionResult): ReproductionResult {
+  return { ...result, stdout: '[Playwright JSON report omitted; normalized timing summary recorded instead.]', stderr: '' };
+}
+
 export function reproductionMatchesEvidence(result: ReproductionResult, evidence: EvidenceEnvelope): boolean {
   return result.status === 'failed' && result.evidenceFingerprint === expectedEvidenceFingerprint(evidence);
 }

tests/e2e/pattern-demo-persistence.spec.ts

@@ -522,7 +522,7 @@ async function expectSnapshot(page: Page, expected: PersistenceSnapshot): Promis
 async function reopenAndAssert(page: Page, expected: PersistenceSnapshot, label: string): Promise<{ page: Page; errors: ErrorCollector }> {
   const context = page.context();
   await expectPersistedSnapshot(page, expected);
-  await page.close({ runBeforeUnload: true });
+  await page.close();
   const reopenedPage = await context.newPage();
   const errors = trackErrors(reopenedPage);
   await bootStudio(reopenedPage, { forceNavigate: true });