diff --git a/.github/workflows/runtime-adoption-validation.yml b/.github/workflows/runtime-adoption-validation.yml new file mode 100644 index 0000000..e0151f8 --- /dev/null +++ b/.github/workflows/runtime-adoption-validation.yml @@ -0,0 +1,168 @@ +name: Runtime adoption validation + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/runtime-adoption-validation.yml + +permissions: + contents: read + +env: + PROOFDIFF_FIX_SHA: 1f2e8b00e1c3923d4d678a933bec80ad605593ed + +jobs: + zod-vitest: + name: Real repo / Zod / fixed targeted Vitest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - uses: pnpm/action-setup@v4 + with: + version: 10.12.1 + run_install: false + - name: Acquire exact ProofDiff fix commit + shell: bash + run: | + mkdir -p work/proofdiff-fixed + cd work/proofdiff-fixed + git init -q + git remote add origin https://github.com/hzw0813/proofdiff.git + git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA" + git checkout --quiet --detach "$PROOFDIFF_FIX_SHA" + test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA" + npm ci + npm run build + - name: Acquire pinned Zod + shell: bash + run: | + mkdir -p work/zod + cd work/zod + git init -q + git remote add origin https://github.com/colinhacks/zod.git + git fetch --quiet --depth=1 origin 2d90846af918af9602e088812d63a035d47cdbe4 + git checkout --quiet --detach 2d90846af918af9602e088812d63a035d47cdbe4 + test "$(git rev-parse HEAD)" = "2d90846af918af9602e088812d63a035d47cdbe4" + - name: Install pinned repository dependencies + working-directory: work/zod + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Create syntax-preserving source mutation + shell: bash + run: | + printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts + - name: Require real exact-target evidence from fixed observer + shell: bash + run: | + node --input-type=module <<'NODE' + import path from 'node:path'; + import { pathToFileURL } from 'node:url'; + + const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href; + const { analyzeRepository } = await import(proofdiffModule); + const root = path.resolve('work/zod'); + const changedPath = 'packages/zod/src/v3/types.ts'; + const expectedTest = 'packages/zod/src/v3/tests/string.test.ts'; + const targetId = 'js:test:test:targeted:vitest'; + + const staticReport = await analyzeRepository({ repo: root, runChecks: false }); + const target = staticReport.discoveredChecks.find((check) => check.id === targetId); + if (!target) throw new Error(`Missing ${targetId}`); + if (!(target.targetFiles ?? []).includes(expectedTest)) { + throw new Error(`Expected ${expectedTest} in targeted files; got ${(target.targetFiles ?? []).join(', ')}`); + } + console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA); + console.log('Targeted check discovered:', target.id); + console.log('Target count:', target.targetFiles?.length ?? 0); + + const report = await analyzeRepository({ + repo: root, + runChecks: true, + selectedChecks: [targetId], + timeoutMs: 180_000, + maxOutputBytes: 512_000, + }); + const result = report.checks.find((check) => check.id === targetId); + if (!result) throw new Error(`Missing executed ${targetId}`); + if (result.status !== 'passed') throw new Error(`Targeted check failed: ${result.status}`); + + const expectedObservation = result.targetObservations?.find( + (item) => item.path === expectedTest || item.runnerPath === expectedTest, + ); + if (!expectedObservation) throw new Error(`No exact observation for ${expectedTest}`); + console.log('Expected target observation:', JSON.stringify(expectedObservation)); + if (expectedObservation.outcome !== 'passed') { + throw new Error(`Expected exact target to pass, got ${expectedObservation.outcome}`); + } + + const observed = result.targetObservations?.filter((item) => item.outcome !== 'not-observed') ?? []; + console.log('Observed exact targets:', observed.length, '/', result.targetObservations?.length ?? 0); + if (observed.length === 0) throw new Error('Observer still produced no exact target observations'); + + const assessment = report.assessments.find((item) => item.file.path === changedPath); + if (!assessment) throw new Error(`Missing assessment for ${changedPath}`); + console.log('Changed source assessment:', assessment.status); + console.log('Strongest evidence:', assessment.evidenceBoundary?.strongestEvidence); + if (assessment.status !== 'verified') { + throw new Error(`Expected source to be verified, got ${assessment.status}`); + } + if (assessment.evidenceBoundary?.strongestEvidence !== 'related-test-file-passed') { + throw new Error(`Unexpected strongest evidence: ${assessment.evidenceBoundary?.strongestEvidence}`); + } + NODE + + vitest-monorepo-negative: + name: Real repo / Vitest monorepo / orchestration stays opaque + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Acquire exact ProofDiff fix commit + shell: bash + run: | + mkdir -p work/proofdiff-fixed + cd work/proofdiff-fixed + git init -q + git remote add origin https://github.com/hzw0813/proofdiff.git + git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA" + git checkout --quiet --detach "$PROOFDIFF_FIX_SHA" + test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA" + npm ci + npm run build + - name: Acquire pinned Vitest monorepo without installing dependencies + shell: bash + run: | + mkdir -p work/vitest + cd work/vitest + git init -q + git remote add origin https://github.com/vitest-dev/vitest.git + git fetch --quiet --depth=1 --filter=blob:none origin 667c13954daff2d1bec8a866702da8934b9ce539 + git checkout --quiet --detach 667c13954daff2d1bec8a866702da8934b9ce539 + test "$(git rev-parse HEAD)" = "667c13954daff2d1bec8a866702da8934b9ce539" + printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> packages/utils/src/error.ts + - name: Require root workspace orchestration to remain opaque + shell: bash + run: | + node --input-type=module <<'NODE' + import path from 'node:path'; + import { pathToFileURL } from 'node:url'; + + const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href; + const { analyzeRepository } = await import(proofdiffModule); + const root = path.resolve('work/vitest'); + const report = await analyzeRepository({ repo: root, runChecks: false }); + const rootTest = report.discoveredChecks.find((check) => check.id === 'js:test:test'); + if (!rootTest) throw new Error('Expected root test check discovery'); + const invented = report.discoveredChecks.filter((check) => check.id.startsWith('js:test:test:targeted:')); + console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA); + console.log('Root test command:', rootTest.command, ...(rootTest.args ?? [])); + console.log('Invented exact-target checks:', invented.length); + if (invented.length !== 0) { + throw new Error(`Workspace orchestration unexpectedly became exact-target evidence: ${invented.map((item) => item.id).join(', ')}`); + } + NODE