Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 168 additions & 0 deletions .github/workflows/runtime-adoption-validation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
name: Runtime adoption validation

on:
pull_request:
branches: [main]
paths:
- .github/workflows/runtime-adoption-validation.yml

permissions:
contents: read

env:
PROOFDIFF_FIX_SHA: 1f2e8b00e1c3923d4d678a933bec80ad605593ed

jobs:
zod-vitest:
name: Real repo / Zod / fixed targeted Vitest
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 24
- uses: pnpm/action-setup@v4
with:
version: 10.12.1
run_install: false
- name: Acquire exact ProofDiff fix commit
shell: bash
run: |
mkdir -p work/proofdiff-fixed
cd work/proofdiff-fixed
git init -q
git remote add origin https://github.com/hzw0813/proofdiff.git
git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA"
git checkout --quiet --detach "$PROOFDIFF_FIX_SHA"
test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA"
npm ci
npm run build
- name: Acquire pinned Zod
shell: bash
run: |
mkdir -p work/zod
cd work/zod
git init -q
git remote add origin https://github.com/colinhacks/zod.git
git fetch --quiet --depth=1 origin 2d90846af918af9602e088812d63a035d47cdbe4
git checkout --quiet --detach 2d90846af918af9602e088812d63a035d47cdbe4
test "$(git rev-parse HEAD)" = "2d90846af918af9602e088812d63a035d47cdbe4"
- name: Install pinned repository dependencies
working-directory: work/zod
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Create syntax-preserving source mutation
shell: bash
run: |
printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts
- name: Require real exact-target evidence from fixed observer
shell: bash
run: |
node --input-type=module <<'NODE'
import path from 'node:path';
import { pathToFileURL } from 'node:url';

const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href;
const { analyzeRepository } = await import(proofdiffModule);
const root = path.resolve('work/zod');
const changedPath = 'packages/zod/src/v3/types.ts';
const expectedTest = 'packages/zod/src/v3/tests/string.test.ts';
const targetId = 'js:test:test:targeted:vitest';

const staticReport = await analyzeRepository({ repo: root, runChecks: false });
const target = staticReport.discoveredChecks.find((check) => check.id === targetId);
if (!target) throw new Error(`Missing ${targetId}`);
if (!(target.targetFiles ?? []).includes(expectedTest)) {
throw new Error(`Expected ${expectedTest} in targeted files; got ${(target.targetFiles ?? []).join(', ')}`);
}
console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA);
console.log('Targeted check discovered:', target.id);
console.log('Target count:', target.targetFiles?.length ?? 0);

const report = await analyzeRepository({
repo: root,
runChecks: true,
selectedChecks: [targetId],
timeoutMs: 180_000,
maxOutputBytes: 512_000,
});
const result = report.checks.find((check) => check.id === targetId);
if (!result) throw new Error(`Missing executed ${targetId}`);
if (result.status !== 'passed') throw new Error(`Targeted check failed: ${result.status}`);

const expectedObservation = result.targetObservations?.find(
(item) => item.path === expectedTest || item.runnerPath === expectedTest,
);
if (!expectedObservation) throw new Error(`No exact observation for ${expectedTest}`);
console.log('Expected target observation:', JSON.stringify(expectedObservation));
if (expectedObservation.outcome !== 'passed') {
throw new Error(`Expected exact target to pass, got ${expectedObservation.outcome}`);
}

const observed = result.targetObservations?.filter((item) => item.outcome !== 'not-observed') ?? [];
console.log('Observed exact targets:', observed.length, '/', result.targetObservations?.length ?? 0);
if (observed.length === 0) throw new Error('Observer still produced no exact target observations');

const assessment = report.assessments.find((item) => item.file.path === changedPath);
if (!assessment) throw new Error(`Missing assessment for ${changedPath}`);
console.log('Changed source assessment:', assessment.status);
console.log('Strongest evidence:', assessment.evidenceBoundary?.strongestEvidence);
if (assessment.status !== 'verified') {
throw new Error(`Expected source to be verified, got ${assessment.status}`);
}
if (assessment.evidenceBoundary?.strongestEvidence !== 'related-test-file-passed') {
throw new Error(`Unexpected strongest evidence: ${assessment.evidenceBoundary?.strongestEvidence}`);
}
NODE

vitest-monorepo-negative:
name: Real repo / Vitest monorepo / orchestration stays opaque
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 24
- name: Acquire exact ProofDiff fix commit
shell: bash
run: |
mkdir -p work/proofdiff-fixed
cd work/proofdiff-fixed
git init -q
git remote add origin https://github.com/hzw0813/proofdiff.git
git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA"
git checkout --quiet --detach "$PROOFDIFF_FIX_SHA"
test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA"
npm ci
npm run build
- name: Acquire pinned Vitest monorepo without installing dependencies
shell: bash
run: |
mkdir -p work/vitest
cd work/vitest
git init -q
git remote add origin https://github.com/vitest-dev/vitest.git
git fetch --quiet --depth=1 --filter=blob:none origin 667c13954daff2d1bec8a866702da8934b9ce539
git checkout --quiet --detach 667c13954daff2d1bec8a866702da8934b9ce539
test "$(git rev-parse HEAD)" = "667c13954daff2d1bec8a866702da8934b9ce539"
printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> packages/utils/src/error.ts
- name: Require root workspace orchestration to remain opaque
shell: bash
run: |
node --input-type=module <<'NODE'
import path from 'node:path';
import { pathToFileURL } from 'node:url';

const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href;
const { analyzeRepository } = await import(proofdiffModule);
const root = path.resolve('work/vitest');
const report = await analyzeRepository({ repo: root, runChecks: false });
const rootTest = report.discoveredChecks.find((check) => check.id === 'js:test:test');
if (!rootTest) throw new Error('Expected root test check discovery');
const invented = report.discoveredChecks.filter((check) => check.id.startsWith('js:test:test:targeted:'));
console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA);
console.log('Root test command:', rootTest.command, ...(rootTest.args ?? []));
console.log('Invented exact-target checks:', invented.length);
if (invented.length !== 0) {
throw new Error(`Workspace orchestration unexpectedly became exact-target evidence: ${invented.map((item) => item.id).join(', ')}`);
}
NODE
Loading