diff --git a/docs/troubleshooting/repair-harness.md b/docs/troubleshooting/repair-harness.md index a030ecf8..27e68d0d 100644 --- a/docs/troubleshooting/repair-harness.md +++ b/docs/troubleshooting/repair-harness.md @@ -16,6 +16,28 @@ change GitHub secrets, or modify hosted databases. ## Collect a failed PR job +Pass `--clean` when starting a new harness run to remove existing `.log` +files anywhere under `.repair-harness/runs` before the run begins. Other run +artifacts, including evidence and state files, are preserved: + +```bash +npm run repair:ci:collect -- --pr 123 --clean +``` + +The option is also supported by the timing and hosted run commands. Do not use +it when resuming a run whose logs you still need. + +To remove every file under `.repair-harness` and the complete +`.repair-harness/runs` directory tree, including evidence, state, metrics, and +logs, use the stronger `--clean-all` option when starting a new run: + +```bash +npm run repair:ci:collect -- --pr 123 --clean-all +``` + +If both flags are supplied, `--clean-all` takes precedence. Do not use either +cleanup option when resuming a run. + For a pull request, collect the failing GitHub Actions check: ```bash diff --git a/scripts/repair-harness/cli.ts b/scripts/repair-harness/cli.ts index 94224888..a403e220 100644 --- a/scripts/repair-harness/cli.ts +++ b/scripts/repair-harness/cli.ts @@ -6,7 +6,7 @@ import { collectEvidence } from './collect'; import { expectedEvidenceFingerprint, reproduce } from './reproduce'; import { verify } from './verify'; import { runBoundedRepair } from './repair'; -import { appendEvent, readState, resolveResumeDirectory, writeState } from './state'; +import { appendEvent, cleanAllHarnessFiles, cleanRunLogs, readState, resolveResumeDirectory, writeState } from './state'; import type { EvidenceEnvelope } from './types'; import { aggregateRepairOutcomes, formatRepairMetrics, readRepairOutcomes, writeRepairMetrics } from './metrics'; import { loadHostedConfig } from './hosted/config'; @@ -26,6 +26,10 @@ const value = (name: string): string | undefined => { }; const has = (name: string): boolean => args.includes(name); +const cleanIfRequested = (repo: string): void => { + if (has('--clean-all')) cleanAllHarnessFiles(repo); + else if (has('--clean')) cleanRunLogs(repo); +}; const prepareHosted = (command: string): boolean => { assertHostedScope(command, value('--scope')); if (value('--agent')) throw new Error('Hosted validation cannot invoke an agent; use --no-agent.'); @@ -37,7 +41,9 @@ if (args[0] === 'e2e-timing') { try { const reportPath = value('--report'); if (!reportPath) throw new Error('Expected --report for e2e-timing.'); - const result = runE2ETiming({ repo: value('--repo') ?? repositoryRoot, reportPath: path.resolve(reportPath), runId: value('--run-id') }); + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); + const result = runE2ETiming({ repo, reportPath: path.resolve(reportPath), runId: value('--run-id') }); console.log(`E2E timing ${result.analysis.status} in ${result.runDirectory}`); if (result.analysis.status === 'failed' || result.analysis.status === 'invalid') process.exitCode = 1; } catch (error) { @@ -47,8 +53,10 @@ if (args[0] === 'e2e-timing') { } else if (args[0] === 'e2e-timing-repair') { try { if (!value('--pr') && !value('--run')) throw new Error('Expected --pr or --run for e2e-timing-repair.'); + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); const result = await runAutomatedTimingRepair({ - repo: value('--repo') ?? repositoryRoot, + repo, pr: value('--pr'), run: value('--run'), agent: 'codex', @@ -69,7 +77,9 @@ if (args[0] === 'e2e-timing') { if (!configPath) throw new Error('Expected --config for hosted-probe.'); const config = loadHostedConfig(path.resolve(configPath)); if (dryRun) { console.log('Hosted probe dry-run: configuration validated; no network request will run.'); return; } - const result = await runHostedProbe({ repo: value('--repo') ?? repositoryRoot, config, runId: value('--run-id') }); + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); + const result = await runHostedProbe({ repo, config, runId: value('--run-id') }); console.log(`Hosted probe ${result.status} in ${result.runDirectory}`); if (result.status !== 'passed') process.exitCode = 1; } catch (error) { @@ -83,7 +93,9 @@ if (args[0] === 'e2e-timing') { if (!configPath) throw new Error('Expected --config for hosted-browser.'); const config = loadHostedConfig(path.resolve(configPath)); if (dryRun) { console.log('Hosted browser dry-run: configuration validated; no browser or network request will run.'); return; } - const result = await runHostedBrowser({ repo: value('--repo') ?? repositoryRoot, config, runId: value('--run-id') }); + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); + const result = await runHostedBrowser({ repo, config, runId: value('--run-id') }); console.log(`Hosted browser ${result.status} in ${result.runDirectory}`); if (result.status !== 'passed') process.exitCode = 1; } catch (error) { @@ -100,8 +112,10 @@ if (args[0] === 'e2e-timing') { if (!email || !password) throw new Error('Hosted mutation requires --email and --password; credentials are used in memory only.'); const config = loadHostedConfig(path.resolve(configPath)); if (dryRun) { console.log('Hosted mutation dry-run: configuration and credentials presence validated; no browser or mutation will run.'); return; } + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); const result = await runHostedMutationCommand({ - repo: value('--repo') ?? repositoryRoot, + repo, config, runId: value('--run-id'), account: { email, password, inviteToken: value('--invite-token') }, @@ -126,8 +140,10 @@ if (args[0] === 'e2e-timing') { if (!devEmail || !devPassword || !stableEmail || !stablePassword) throw new Error('Hosted isolation requires separate --dev-email/--dev-password and --stable-email/--stable-password credentials; credentials are used in memory only.'); const config = loadHostedConfig(path.resolve(configPath)); if (dryRun) { console.log('Hosted isolation dry-run: configuration and credential presence validated; no browser or mutation will run.'); return; } + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); const result = await runHostedIsolationCommand({ - repo: value('--repo') ?? repositoryRoot, + repo, config, runId: value('--run-id'), accounts: { dev: { email: devEmail, password: devPassword }, stable: { email: stableEmail, password: stablePassword } }, @@ -166,7 +182,9 @@ if (args[0] === 'e2e-timing') { } } else if (args[0] === 'collect') { try { - const result = collectEvidence({ repo: value('--repo') ?? repositoryRoot, pr: value('--pr'), run: value('--run'), job: value('--job'), outputRoot: value('--output-root') }); + const repo = value('--repo') ?? repositoryRoot; + cleanIfRequested(repo); + const result = collectEvidence({ repo, pr: value('--pr'), run: value('--run'), job: value('--job'), outputRoot: value('--output-root') }); console.log(`Collected ${result.envelope.failure.class} evidence in ${result.runDirectory}`); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/scripts/repair-harness/packet.ts b/scripts/repair-harness/packet.ts index d1f28a8a..c7947a74 100644 --- a/scripts/repair-harness/packet.ts +++ b/scripts/repair-harness/packet.ts @@ -51,7 +51,9 @@ export function createDiagnosisPacket(options: PacketOptions): string { export function createImplementationPacket(options: PacketOptions): string { if (!options.diagnosis) throw new Error('Implementation packets require a diagnosis.'); const files = options.targetedFiles ?? options.diagnosis.files; - const diff = (options.diff ?? '').slice(0, options.maxDiffChars ?? 24_000); + // Keep the packet within the implementation input budget after adding + // repository guidance and timing evidence. + const diff = (options.diff ?? '').slice(0, options.maxDiffChars ?? 16_000); return redactSecrets([ '# PhaserForge CI repair implementation', 'Implement only the approved diagnosis. Do not commit, push, merge, deploy, edit workflows, weaken tests, or change secrets/configuration.', diff --git a/scripts/repair-harness/repair.ts b/scripts/repair-harness/repair.ts index 6b337d1b..6467187e 100644 --- a/scripts/repair-harness/repair.ts +++ b/scripts/repair-harness/repair.ts @@ -73,7 +73,9 @@ export async function runBoundedRepair(options: RepairOptions): Promise verify({ evidence: options.evidence, cwd: options.repo }))); + const verification = options.verifyPatch + ? await options.verifyPatch() + : await verify({ evidence: options.evidence, cwd: options.repo }); if (!verification) { const reason = 'Verification adapter returned no result.'; mkdirSync(path.join(options.runDirectory, 'verification'), { recursive: true }); diff --git a/scripts/repair-harness/state.ts b/scripts/repair-harness/state.ts index 57902bb9..63fd8c89 100644 --- a/scripts/repair-harness/state.ts +++ b/scripts/repair-harness/state.ts @@ -1,4 +1,4 @@ -import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { redactSecrets } from './artifacts'; @@ -6,6 +6,39 @@ import { redactSecrets } from './artifacts'; export interface RepairState { runId: string; phase: string; status: string; budgets: Record; scope?: string; updatedAt: string; } export interface RepairEvent { event: string; at: string; [key: string]: unknown; } +export function cleanRunLogs(repo: string): void { + const runsRoot = path.resolve(repo, '.repair-harness', 'runs'); + if (!existsSync(runsRoot)) return; + + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) visit(entryPath); + else if (entry.isFile() && entry.name.endsWith('.log')) unlinkSync(entryPath); + } + }; + + visit(runsRoot); +} + +export function cleanAllHarnessFiles(repo: string): void { + const harnessRoot = path.resolve(repo, '.repair-harness'); + if (!existsSync(harnessRoot)) return; + + const runsRoot = path.join(harnessRoot, 'runs'); + if (existsSync(runsRoot)) rmSync(runsRoot, { recursive: true, force: true }); + + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) visit(entryPath); + else if (entry.isFile()) unlinkSync(entryPath); + } + }; + + visit(harnessRoot); +} + export function createRun(repo: string, runId = `${Date.now()}`): { directory: string; state: RepairState } { const directory = path.resolve(repo, '.repair-harness', 'runs', runId); mkdirSync(directory, { recursive: true }); diff --git a/scripts/repair-harness/timingRepair.ts b/scripts/repair-harness/timingRepair.ts index 206a367a..83d44a20 100644 --- a/scripts/repair-harness/timingRepair.ts +++ b/scripts/repair-harness/timingRepair.ts @@ -43,7 +43,6 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions): const slow = analysis.entries.filter((entry) => entry.category === 'slow'); const first = slow[0]; - const boundedSlow = slow.slice(0, 20); const job = String(resolved.job.name ?? resolved.metadata.workflowName ?? 'GitHub Actions E2E matrix'); const command = commandForJob(job); const evidence: EvidenceEnvelope = { @@ -57,8 +56,8 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions): class: 'timeout', testFile: first.file, testTitle: first.title, - message: `${slow.length} tests exceeded the hard ceiling.\n` + boundedSlow.map((entry) => `${entry.title} — ${entry.project} — ${entry.file} — ${entry.durationMs}ms`).join('\n'), - stackExcerpt: `Playwright timing diagnostic exceeded the 10,000ms hard ceiling. Showing ${boundedSlow.length} of ${slow.length} slow tests.\n` + boundedSlow.map((entry) => `${entry.file}: ${entry.title} (${entry.durationMs}ms)`).join('\n'), + message: formatSlowTestEvidence(slow), + stackExcerpt: formatSlowTestEvidence(slow), }, artifacts: { tracePaths: [], screenshotPaths: [] }, redactionsApplied: ['GitHub artifact contents were reduced to timing fields'], @@ -74,9 +73,27 @@ export async function runAutomatedTimingRepair(options: AutomatedTimingOptions): return { sourceRunId: source.runId, runDirectory: timing.runDirectory, status: 'published', analysis, repair, pullRequestUrl }; } -function commandForJob(job: string): string { +export function formatSlowTestEvidence(slow: Array<{ title: string; project: string; file: string; durationMs?: number }>): string { + const groups = new Map(); + for (const entry of slow) { + const key = `${entry.project}\u0000${entry.file}`; + const durationMs = entry.durationMs ?? 0; + const group = groups.get(key) ?? { project: entry.project, file: entry.file, count: 0, fastestMs: durationMs, slowestMs: durationMs }; + group.count += 1; + group.fastestMs = Math.min(group.fastestMs, durationMs); + group.slowestMs = Math.max(group.slowestMs, durationMs); + groups.set(key, group); + } + const inventory = [...groups.values()] + .sort((left, right) => right.slowestMs - left.slowestMs) + .map((group) => `- ${group.project} — ${group.file} — ${group.count} slow (${group.fastestMs}-${group.slowestMs}ms)`) + .join('\n'); + return `${slow.length} tests exceeded the hard ceiling across ${groups.size} project/file groups. Full normalized test-level inventory is in e2e-timing-evidence.json.\n${inventory}`; +} + +export function commandForJob(job: string): string { const shard = job.match(/shard\s+(\d+)\s*\/\s*(\d+)/i); - if (/full matrix/i.test(job)) return 'npm run test:e2e -- --project=firefox --project=webkit --project=msedge --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1'); + if (/full matrix/i.test(job)) return 'PW_PROJECTS=firefox,webkit,msedge npm run test:e2e -- --project=firefox --project=webkit --project=msedge --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1'); return 'npm run test:e2e -- --project=chromium --grep "@smoke|@critical" --shard={shard}/{shards} --fail-on-flaky-tests'.replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '1'); } diff --git a/scripts/repair-harness/verify.ts b/scripts/repair-harness/verify.ts index 2bd0ed33..2aa40315 100644 --- a/scripts/repair-harness/verify.ts +++ b/scripts/repair-harness/verify.ts @@ -1,4 +1,5 @@ import type { EvidenceEnvelope } from './types'; +import { analyzeE2ETiming, type E2ETimingAnalysis } from './e2eTiming'; import { expectedEvidenceFingerprint, focusedReproductionCommand, reproduce, type ReproductionResult } from './reproduce'; export interface VerificationOptions { @@ -14,23 +15,48 @@ export interface VerificationResult { focused?: ReproductionResult; requiredCommand: string; required?: ReproductionResult; + timing?: Pick; reason?: string; } export async function verify(options: VerificationOptions): Promise { const run = options.run ?? ((command, cwd, timeoutMs) => reproduce({ command, cwd, timeoutMs })); - const focusedCommand = options.evidence.failure.testFile ? focusedReproductionCommand(options.evidence) : undefined; + const candidateFocusedCommand = options.evidence.failure.testFile ? focusedReproductionCommand(options.evidence) : undefined; + const focusedCommand = candidateFocusedCommand === options.evidence.reproduction.command ? undefined : candidateFocusedCommand; let focused: ReproductionResult | undefined; if (focusedCommand) { focused = await run(focusedCommand, options.cwd, options.timeoutMs); if (focused.status !== 'passed') return { verified: false, focusedCommand, focused, requiredCommand: options.evidence.reproduction.command, reason: 'Focused verification failed.' }; } - const requiredCommand = options.evidence.reproduction.command; + const isTimingRepair = options.evidence.scope === 'e2e-timing-repair'; + const requiredCommand = isTimingRepair ? `${options.evidence.reproduction.command} --reporter=json` : options.evidence.reproduction.command; const required = await run(requiredCommand, options.cwd, options.timeoutMs); + if (isTimingRepair && required.status === 'passed') { + try { + const analysis = analyzeE2ETiming(parseJsonReporterOutput(required.stdout)); + const timing = { status: analysis.status, counts: analysis.counts, slowest: analysis.slowest }; + if (analysis.status === 'failed' || analysis.status === 'invalid') { + return { verified: false, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), timing, reason: 'Timing verification still exceeds the hard ceiling or has invalid durations.' }; + } + return { verified: true, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), timing }; + } catch (error) { + return { verified: false, focusedCommand, focused, requiredCommand, required: redactTimingReport(required), reason: `Timing verification did not produce a readable Playwright JSON report: ${error instanceof Error ? error.message : String(error)}` }; + } + } return { verified: required.status === 'passed', focusedCommand, focused, requiredCommand, required, reason: required.status === 'passed' ? undefined : 'Required verification failed.' }; } +function parseJsonReporterOutput(stdout: string): unknown { + const start = stdout.indexOf('{'); + if (start < 0) throw new Error('JSON object not found in test output.'); + return JSON.parse(stdout.slice(start)); +} + +function redactTimingReport(result: ReproductionResult): ReproductionResult { + return { ...result, stdout: '[Playwright JSON report omitted; normalized timing summary recorded instead.]', stderr: '' }; +} + export function reproductionMatchesEvidence(result: ReproductionResult, evidence: EvidenceEnvelope): boolean { return result.status === 'failed' && result.evidenceFingerprint === expectedEvidenceFingerprint(evidence); } diff --git a/tests/e2e/pattern-demo-persistence.spec.ts b/tests/e2e/pattern-demo-persistence.spec.ts index 64f16453..233417a7 100644 --- a/tests/e2e/pattern-demo-persistence.spec.ts +++ b/tests/e2e/pattern-demo-persistence.spec.ts @@ -522,7 +522,7 @@ async function expectSnapshot(page: Page, expected: PersistenceSnapshot): Promis async function reopenAndAssert(page: Page, expected: PersistenceSnapshot, label: string): Promise<{ page: Page; errors: ErrorCollector }> { const context = page.context(); await expectPersistedSnapshot(page, expected); - await page.close({ runBeforeUnload: true }); + await page.close(); const reopenedPage = await context.newPage(); const errors = trackErrors(reopenedPage); await bootStudio(reopenedPage, { forceNavigate: true }); diff --git a/tests/scripts/repair-harness-e2e-timing.test.ts b/tests/scripts/repair-harness-e2e-timing.test.ts index 430004e4..81f549db 100644 --- a/tests/scripts/repair-harness-e2e-timing.test.ts +++ b/tests/scripts/repair-harness-e2e-timing.test.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { analyzeE2ETiming, parsePlaywrightJsonReport, type PlaywrightJsonReport } from '../../scripts/repair-harness/e2eTiming'; import { runE2ETiming } from '../../scripts/repair-harness/e2eTimingRun'; +import { commandForJob, formatSlowTestEvidence } from '../../scripts/repair-harness/timingRepair'; const report: PlaywrightJsonReport = { suites: [{ @@ -27,6 +28,31 @@ describe('repair harness E2E timing diagnostics', () => { expect(result.slowest).toMatchObject({ title: 'slow test', durationMs: 10_001 }); }); + it('fails a hard-ceiling duration even when Playwright reports the test as passed', () => { + const result = analyzeE2ETiming({ + suites: [{ + file: 'tests/e2e/editor.spec.ts', + specs: [{ title: 'slow but passing test', tests: [{ projectName: 'webkit', results: [{ retry: 0, duration: 10_001, status: 'passed' }] }] }], + }], + }); + + expect(result).toMatchObject({ status: 'failed', counts: { slow: 1 } }); + }); + + it('retains every slow test in repair evidence', () => { + const slow = Array.from({ length: 21 }, (_, index) => ({ title: `slow ${index + 1}`, project: 'webkit', file: `tests/e2e/slow-${index + 1}.spec.ts`, durationMs: 10_001 + index })); + + const evidence = formatSlowTestEvidence(slow); + + expect(evidence).toContain('21 tests exceeded the hard ceiling across 21 project/file groups.'); + expect(evidence).toContain('webkit — tests/e2e/slow-1.spec.ts — 1 slow (10001-10001ms)'); + expect(evidence).toContain('webkit — tests/e2e/slow-21.spec.ts — 1 slow (10021-10021ms)'); + }); + + it('enables every browser named by a full-matrix reproduction command', () => { + expect(commandForJob('E2E Full Matrix (shard 4/8)')).toContain('PW_PROJECTS=firefox,webkit,msedge'); + }); + it('groups summary data by project and file without retaining raw report fields', () => { const result = analyzeE2ETiming(report); expect(result.groups).toEqual([{ project: 'chromium', file: 'tests/e2e/editor.spec.ts', count: 2, slowCount: 0, warningCount: 1, fastestMs: 6_500, slowestMs: 8_000 }, { project: 'webkit', file: 'tests/e2e/editor.spec.ts', count: 1, slowCount: 1, warningCount: 0, fastestMs: 10_001, slowestMs: 10_001 }]); diff --git a/tests/scripts/repair-harness-phase2.test.ts b/tests/scripts/repair-harness-phase2.test.ts index 54bbb86d..90bad59f 100644 --- a/tests/scripts/repair-harness-phase2.test.ts +++ b/tests/scripts/repair-harness-phase2.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { readFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { focusedReproductionCommand, reproduce } from '../../scripts/repair-harness/reproduce'; import { evaluatePolicy, stagnationReasons } from '../../scripts/repair-harness/policy'; -import { appendEvent, createRun, readState, writeState } from '../../scripts/repair-harness/state'; +import { appendEvent, cleanAllHarnessFiles, cleanRunLogs, createRun, readState, writeState } from '../../scripts/repair-harness/state'; import { reproductionMatchesEvidence, verify } from '../../scripts/repair-harness/verify'; import type { EvidenceEnvelope } from '../../scripts/repair-harness/types'; @@ -46,9 +48,64 @@ describe('repair harness phase 2 reproduction', () => { expect(result.verified).toBe(true); expect(commands).toEqual([focusedReproductionCommand(evidence), evidence.reproduction.command]); }); + + it('requires a timing-clean Playwright JSON report for timing repairs', async () => { + const commands: string[] = []; + const timingEvidence: EvidenceEnvelope = { + ...evidence, + scope: 'e2e-timing-repair', + failure: { ...evidence.failure }, + }; + const result = await verify({ evidence: timingEvidence, cwd: process.cwd(), run: async (command) => { + commands.push(command); + return { + command, + cwd: process.cwd(), + stdout: `\n> test:e2e\n\n${JSON.stringify({ suites: [{ file: 'tests/e2e/editor.spec.ts', specs: [{ title: 'slow but passing test', tests: [{ projectName: 'webkit', results: [{ retry: 0, duration: 10_001, status: 'passed' }] }] }] }] })}`, + stderr: '', durationMs: 1, exitCode: 0, signal: null, timedOut: false, status: 'passed', evidenceFingerprint: 'ok', + }; + } }); + + expect(commands).toEqual([`${timingEvidence.reproduction.command} --reporter=json`]); + expect(result.verified).toBe(false); + expect(result.reason).toContain('hard ceiling'); + }); }); describe('repair harness phase 2 policy and state', () => { + it('cleans log files recursively without removing run evidence', () => { + const repo = mkdtempSync(path.join(os.tmpdir(), 'phaserforge-repair-clean-')); + const runsRoot = path.join(repo, '.repair-harness', 'runs'); + mkdirSync(path.join(runsRoot, 'old', 'reproduce'), { recursive: true }); + writeFileSync(path.join(runsRoot, 'old', 'reproduce', 'stdout.log'), 'old output'); + writeFileSync(path.join(runsRoot, 'old', 'reproduce', 'stderr.log'), 'old error'); + writeFileSync(path.join(runsRoot, 'old', 'evidence.json'), '{}'); + + cleanRunLogs(repo); + + expect(existsSync(path.join(runsRoot, 'old', 'reproduce', 'stdout.log'))).toBe(false); + expect(existsSync(path.join(runsRoot, 'old', 'reproduce', 'stderr.log'))).toBe(false); + expect(existsSync(path.join(runsRoot, 'old', 'evidence.json'))).toBe(true); + }); + + it('clean-all removes every harness file, including evidence and state', () => { + const repo = mkdtempSync(path.join(os.tmpdir(), 'phaserforge-repair-clean-all-')); + const harnessRoot = path.join(repo, '.repair-harness'); + mkdirSync(path.join(harnessRoot, 'runs', 'old', 'reproduce'), { recursive: true }); + writeFileSync(path.join(harnessRoot, 'runs', 'old', 'reproduce', 'stdout.log'), 'old output'); + writeFileSync(path.join(harnessRoot, 'runs', 'old', 'evidence.json'), '{}'); + writeFileSync(path.join(harnessRoot, 'runs', 'old', 'state.json'), '{}'); + writeFileSync(path.join(harnessRoot, 'metrics.json'), '{}'); + + cleanAllHarnessFiles(repo); + + expect(existsSync(path.join(harnessRoot, 'runs', 'old', 'reproduce', 'stdout.log'))).toBe(false); + expect(existsSync(path.join(harnessRoot, 'runs', 'old', 'evidence.json'))).toBe(false); + expect(existsSync(path.join(harnessRoot, 'runs', 'old', 'state.json'))).toBe(false); + expect(existsSync(path.join(harnessRoot, 'metrics.json'))).toBe(false); + expect(existsSync(path.join(harnessRoot, 'runs'))).toBe(false); + }); + it('denies unsafe test, workflow, config, and retry changes', () => { const result = evaluatePolicy([ 'M\t.github/workflows/e2e-pr.yml', diff --git a/tests/scripts/repair-harness-phase3.test.ts b/tests/scripts/repair-harness-phase3.test.ts index 7b6cf3e5..0351b4f4 100644 --- a/tests/scripts/repair-harness-phase3.test.ts +++ b/tests/scripts/repair-harness-phase3.test.ts @@ -29,6 +29,13 @@ describe('repair harness phase 3 packets and agent', () => { expect(implementationPacket).toContain('[REDACTED]'); }); + it('keeps implementation packets within the bounded input budget', () => { + const diagnosis = { failureClass: 'assertion' as const, likelyCause: 'state is not persisted', files: ['src/editor/store.ts'], symbols: ['saveProject'], reproductionCommand: evidence.reproduction.command, confidence: 0.8 }; + const implementationPacket = createImplementationPacket({ repo: process.cwd(), evidence, diagnosis, diff: 'x'.repeat(24_000) }); + + expect(Buffer.byteLength(implementationPacket)).toBeLessThanOrEqual(32_000); + }); + it('parses the strict diagnosis contract, including fenced JSON', () => { expect(parseDiagnosis('```json\n{"failureClass":"assertion","likelyCause":"bad state","files":["src/a.ts"],"symbols":["save"],"reproductionCommand":"npm test","confidence":0.7}\n```')).toMatchObject({ failureClass: 'assertion', confidence: 0.7 }); expect(() => parseDiagnosis('{"failureClass":"assertion"}')).toThrow('required contract'); @@ -62,6 +69,28 @@ describe('repair harness phase 3 bounds and verification gate', () => { expect(readState(directory).status).toBe('failed'); }); + it('runs the default verifier instead of serializing its callback', async () => { + const { directory } = createRun(process.cwd(), `phase3-default-verify-${Date.now()}`); + const verificationEvidence: EvidenceEnvelope = { + ...evidence, + scope: 'manual', + reproduction: { command: `${process.execPath} -e "process.exit(0)"` }, + failure: { ...evidence.failure, testFile: undefined, testTitle: undefined }, + }; + const diagnosis = JSON.stringify({ failureClass: 'assertion', likelyCause: 'bad state', files: ['src/editor/store.ts'], symbols: ['save'], reproductionCommand: verificationEvidence.reproduction.command, confidence: 0.9 }); + + const result = await runBoundedRepair({ + repo: process.cwd(), + runDirectory: directory, + evidence: verificationEvidence, + diff: '', + callAgent: async (kind) => agentResult(kind, kind === 'diagnosis' ? diagnosis : 'model says done'), + }); + + expect(result).toMatchObject({ status: 'verified', verification: { verified: true, requiredCommand: verificationEvidence.reproduction.command } }); + expect(readState(directory).status).toBe('verified'); + }); + it('stops when the diagnosis or implementation budget is exhausted', () => { const state: RepairState = { runId: 'x', phase: 'diagnosis', status: 'active', budgets: { diagnosisCalls: PHASE3_BUDGETS.diagnosisCalls }, updatedAt: new Date().toISOString() }; expect(budgetAllows(state, 'diagnosis', 0, 10)).toBe(false);