From c6c689798dedd3c2d0a40d7b9b26d2f8493adb7c Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 14 Aug 2026 21:31:05 +0800 Subject: [PATCH 1/4] Add temporary runtime adoption validation --- .../workflows/runtime-adoption-validation.yml | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 .github/workflows/runtime-adoption-validation.yml diff --git a/.github/workflows/runtime-adoption-validation.yml b/.github/workflows/runtime-adoption-validation.yml new file mode 100644 index 0000000..fda3d0f --- /dev/null +++ b/.github/workflows/runtime-adoption-validation.yml @@ -0,0 +1,132 @@ +name: Runtime adoption validation + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/runtime-adoption-validation.yml + +permissions: + contents: read + +jobs: + zod-vitest: + name: Real repo / Zod / targeted Vitest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npm run build + - uses: pnpm/action-setup@v4 + with: + version: 10.12.1 + run_install: false + - name: Acquire pinned Zod + shell: bash + run: | + mkdir -p work/zod + cd work/zod + git init -q + git remote add origin https://github.com/colinhacks/zod.git + git fetch --quiet --depth=1 origin 2d90846af918af9602e088812d63a035d47cdbe4 + git checkout --quiet --detach 2d90846af918af9602e088812d63a035d47cdbe4 + test "$(git rev-parse HEAD)" = "2d90846af918af9602e088812d63a035d47cdbe4" + - name: Install pinned repository dependencies + working-directory: work/zod + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Create syntax-preserving source mutation + shell: bash + run: printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts + - name: Run only ProofDiff exact-target Vitest evidence + shell: bash + run: | + node --input-type=module <<'NODE' + import path from 'node:path'; + import { analyzeRepository } from './dist/analyze.js'; + + const root = path.resolve('work/zod'); + const changedPath = 'packages/zod/src/v3/types.ts'; + const expectedTest = 'packages/zod/src/v3/tests/string.test.ts'; + const targetId = 'js:test:test:targeted:vitest'; + + const staticReport = await analyzeRepository({ repo: root, runChecks: false }); + const target = staticReport.discoveredChecks.find((check) => check.id === targetId); + if (!target) throw new Error(`Missing ${targetId}`); + if (!(target.targetFiles ?? []).includes(expectedTest)) { + throw new Error(`Expected ${expectedTest} in targeted files; got ${(target.targetFiles ?? []).join(', ')}`); + } + console.log('Targeted check discovered:', target.id); + console.log('Target count:', target.targetFiles?.length ?? 0); + console.log('Includes expected test:', expectedTest); + + const report = await analyzeRepository({ + repo: root, + runChecks: true, + selectedChecks: [targetId], + timeoutMs: 180_000, + maxOutputBytes: 512_000, + }); + const result = report.checks.find((check) => check.id === targetId); + if (!result) throw new Error(`Missing executed ${targetId}`); + console.log('Targeted check status:', result.status); + const expectedObservation = result.targetObservations?.find((item) => item.path === expectedTest || item.runnerPath === expectedTest); + if (!expectedObservation) throw new Error(`No exact observation for ${expectedTest}`); + console.log('Expected target observation:', JSON.stringify(expectedObservation)); + if (expectedObservation.outcome !== 'passed') { + throw new Error(`Expected exact target to pass, got ${expectedObservation.outcome}`); + } + const assessment = report.assessments.find((item) => item.file.path === changedPath); + if (!assessment) throw new Error(`Missing assessment for ${changedPath}`); + console.log('Changed source assessment:', assessment.status); + console.log('Strongest evidence:', assessment.evidenceBoundary?.strongestEvidence); + if (assessment.status !== 'verified') { + throw new Error(`Expected real-repo source assessment to be verified, got ${assessment.status}`); + } + if (assessment.evidenceBoundary?.strongestEvidence !== 'related-test-file-passed') { + throw new Error(`Unexpected strongest evidence: ${assessment.evidenceBoundary?.strongestEvidence}`); + } + NODE + + vitest-monorepo-negative: + name: Real repo / Vitest monorepo / orchestration stays opaque + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npm run build + - name: Acquire pinned Vitest monorepo without installing dependencies + shell: bash + run: | + mkdir -p work/vitest + cd work/vitest + git init -q + git remote add origin https://github.com/vitest-dev/vitest.git + git fetch --quiet --depth=1 --filter=blob:none origin 667c13954daff2d1bec8a866702da8934b9ce539 + git checkout --quiet --detach 667c13954daff2d1bec8a866702da8934b9ce539 + test "$(git rev-parse HEAD)" = "667c13954daff2d1bec8a866702da8934b9ce539" + printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> packages/utils/src/error.ts + - name: Verify root workspace orchestration is not invented as exact Vitest evidence + shell: bash + run: | + node --input-type=module <<'NODE' + import path from 'node:path'; + import { analyzeRepository } from './dist/analyze.js'; + + const root = path.resolve('work/vitest'); + const report = await analyzeRepository({ repo: root, runChecks: false }); + const rootTest = report.discoveredChecks.find((check) => check.id === 'js:test:test'); + if (!rootTest) throw new Error('Expected root test check discovery'); + const invented = report.discoveredChecks.filter((check) => check.id.startsWith('js:test:test:targeted:')); + console.log('Root test origin:', rootTest.origin); + console.log('Root test command:', rootTest.command, ...(rootTest.args ?? [])); + console.log('Invented exact-target checks:', invented.length); + if (invented.length !== 0) throw new Error(`Workspace orchestration unexpectedly became exact-target evidence: ${invented.map((item) => item.id).join(', ')}`); + NODE From c1f7be84fbfba9e0fd16c144ded4aefa0e75f04d Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 14 Aug 2026 21:32:21 +0800 Subject: [PATCH 2/4] Fix temporary adoption workflow YAML --- .github/workflows/runtime-adoption-validation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runtime-adoption-validation.yml b/.github/workflows/runtime-adoption-validation.yml index fda3d0f..f1feddd 100644 --- a/.github/workflows/runtime-adoption-validation.yml +++ b/.github/workflows/runtime-adoption-validation.yml @@ -40,7 +40,8 @@ jobs: run: pnpm install --frozen-lockfile --ignore-scripts - name: Create syntax-preserving source mutation shell: bash - run: printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts + run: | + printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts - name: Run only ProofDiff exact-target Vitest evidence shell: bash run: | From 2c701b1a6468e24528972d16e5e71f59ff791fa3 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 14 Aug 2026 21:34:54 +0800 Subject: [PATCH 3/4] Inspect real Vitest JSON observer shape --- .../workflows/runtime-adoption-validation.yml | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runtime-adoption-validation.yml b/.github/workflows/runtime-adoption-validation.yml index f1feddd..89afbb8 100644 --- a/.github/workflows/runtime-adoption-validation.yml +++ b/.github/workflows/runtime-adoption-validation.yml @@ -42,7 +42,31 @@ jobs: shell: bash run: | printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts - - name: Run only ProofDiff exact-target Vitest evidence + - name: Inspect real Vitest JSON schema for one exact target + shell: bash + working-directory: work/zod + run: | + pnpm exec vitest run packages/zod/src/v3/tests/string.test.ts --reporter=json --outputFile=proofdiff-vitest-diagnostic.json + node --input-type=module <<'NODE' + import { readFile } from 'node:fs/promises'; + const payload = JSON.parse(await readFile('proofdiff-vitest-diagnostic.json', 'utf8')); + console.log('Top-level keys:', Object.keys(payload).sort().join(', ')); + console.log('testResults length:', Array.isArray(payload.testResults) ? payload.testResults.length : 'not-array'); + for (const [index, suite] of (payload.testResults ?? []).slice(0, 3).entries()) { + console.log(`Suite ${index} keys:`, Object.keys(suite ?? {}).sort().join(', ')); + console.log(`Suite ${index} name:`, typeof suite?.name === 'string' ? suite.name : ''); + console.log(`Suite ${index} testFilePath:`, typeof suite?.testFilePath === 'string' ? suite.testFilePath : ''); + console.log(`Suite ${index} status:`, suite?.status ?? ''); + console.log(`Suite ${index} assertionResults:`, Array.isArray(suite?.assertionResults) ? suite.assertionResults.length : 'not-array'); + console.log(`Suite ${index} testResults:`, Array.isArray(suite?.testResults) ? suite.testResults.length : 'not-array'); + console.log(`Suite ${index} counts:`, JSON.stringify({ + numPassingTests: suite?.numPassingTests, + numFailingTests: suite?.numFailingTests, + numPendingTests: suite?.numPendingTests, + })); + } + NODE + - name: Observe current ProofDiff mapping on the same real repository shell: bash run: | node --input-type=module <<'NODE' @@ -77,19 +101,12 @@ jobs: const expectedObservation = result.targetObservations?.find((item) => item.path === expectedTest || item.runnerPath === expectedTest); if (!expectedObservation) throw new Error(`No exact observation for ${expectedTest}`); console.log('Expected target observation:', JSON.stringify(expectedObservation)); - if (expectedObservation.outcome !== 'passed') { - throw new Error(`Expected exact target to pass, got ${expectedObservation.outcome}`); - } + const observedCount = result.targetObservations?.filter((item) => item.outcome !== 'not-observed').length ?? 0; + console.log('Observed exact targets:', observedCount, '/', result.targetObservations?.length ?? 0); const assessment = report.assessments.find((item) => item.file.path === changedPath); if (!assessment) throw new Error(`Missing assessment for ${changedPath}`); console.log('Changed source assessment:', assessment.status); console.log('Strongest evidence:', assessment.evidenceBoundary?.strongestEvidence); - if (assessment.status !== 'verified') { - throw new Error(`Expected real-repo source assessment to be verified, got ${assessment.status}`); - } - if (assessment.evidenceBoundary?.strongestEvidence !== 'related-test-file-passed') { - throw new Error(`Unexpected strongest evidence: ${assessment.evidenceBoundary?.strongestEvidence}`); - } NODE vitest-monorepo-negative: From 0adc6f9b4b0ffb3b07f1386222f801ad0e5558e2 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 14 Aug 2026 21:44:51 +0800 Subject: [PATCH 4/4] Validate fixed Vitest observer on real repositories --- .../workflows/runtime-adoption-validation.yml | 102 ++++++++++-------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/.github/workflows/runtime-adoption-validation.yml b/.github/workflows/runtime-adoption-validation.yml index 89afbb8..e0151f8 100644 --- a/.github/workflows/runtime-adoption-validation.yml +++ b/.github/workflows/runtime-adoption-validation.yml @@ -9,22 +9,34 @@ on: permissions: contents: read +env: + PROOFDIFF_FIX_SHA: 1f2e8b00e1c3923d4d678a933bec80ad605593ed + jobs: zod-vitest: - name: Real repo / Zod / targeted Vitest + name: Real repo / Zod / fixed targeted Vitest runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: node-version: 24 - cache: npm - - run: npm ci - - run: npm run build - uses: pnpm/action-setup@v4 with: version: 10.12.1 run_install: false + - name: Acquire exact ProofDiff fix commit + shell: bash + run: | + mkdir -p work/proofdiff-fixed + cd work/proofdiff-fixed + git init -q + git remote add origin https://github.com/hzw0813/proofdiff.git + git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA" + git checkout --quiet --detach "$PROOFDIFF_FIX_SHA" + test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA" + npm ci + npm run build - name: Acquire pinned Zod shell: bash run: | @@ -42,37 +54,15 @@ jobs: shell: bash run: | printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> work/zod/packages/zod/src/v3/types.ts - - name: Inspect real Vitest JSON schema for one exact target - shell: bash - working-directory: work/zod - run: | - pnpm exec vitest run packages/zod/src/v3/tests/string.test.ts --reporter=json --outputFile=proofdiff-vitest-diagnostic.json - node --input-type=module <<'NODE' - import { readFile } from 'node:fs/promises'; - const payload = JSON.parse(await readFile('proofdiff-vitest-diagnostic.json', 'utf8')); - console.log('Top-level keys:', Object.keys(payload).sort().join(', ')); - console.log('testResults length:', Array.isArray(payload.testResults) ? payload.testResults.length : 'not-array'); - for (const [index, suite] of (payload.testResults ?? []).slice(0, 3).entries()) { - console.log(`Suite ${index} keys:`, Object.keys(suite ?? {}).sort().join(', ')); - console.log(`Suite ${index} name:`, typeof suite?.name === 'string' ? suite.name : ''); - console.log(`Suite ${index} testFilePath:`, typeof suite?.testFilePath === 'string' ? suite.testFilePath : ''); - console.log(`Suite ${index} status:`, suite?.status ?? ''); - console.log(`Suite ${index} assertionResults:`, Array.isArray(suite?.assertionResults) ? suite.assertionResults.length : 'not-array'); - console.log(`Suite ${index} testResults:`, Array.isArray(suite?.testResults) ? suite.testResults.length : 'not-array'); - console.log(`Suite ${index} counts:`, JSON.stringify({ - numPassingTests: suite?.numPassingTests, - numFailingTests: suite?.numFailingTests, - numPendingTests: suite?.numPendingTests, - })); - } - NODE - - name: Observe current ProofDiff mapping on the same real repository + - name: Require real exact-target evidence from fixed observer shell: bash run: | node --input-type=module <<'NODE' import path from 'node:path'; - import { analyzeRepository } from './dist/analyze.js'; + import { pathToFileURL } from 'node:url'; + const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href; + const { analyzeRepository } = await import(proofdiffModule); const root = path.resolve('work/zod'); const changedPath = 'packages/zod/src/v3/types.ts'; const expectedTest = 'packages/zod/src/v3/tests/string.test.ts'; @@ -84,9 +74,9 @@ jobs: if (!(target.targetFiles ?? []).includes(expectedTest)) { throw new Error(`Expected ${expectedTest} in targeted files; got ${(target.targetFiles ?? []).join(', ')}`); } + console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA); console.log('Targeted check discovered:', target.id); console.log('Target count:', target.targetFiles?.length ?? 0); - console.log('Includes expected test:', expectedTest); const report = await analyzeRepository({ repo: root, @@ -97,16 +87,31 @@ jobs: }); const result = report.checks.find((check) => check.id === targetId); if (!result) throw new Error(`Missing executed ${targetId}`); - console.log('Targeted check status:', result.status); - const expectedObservation = result.targetObservations?.find((item) => item.path === expectedTest || item.runnerPath === expectedTest); + if (result.status !== 'passed') throw new Error(`Targeted check failed: ${result.status}`); + + const expectedObservation = result.targetObservations?.find( + (item) => item.path === expectedTest || item.runnerPath === expectedTest, + ); if (!expectedObservation) throw new Error(`No exact observation for ${expectedTest}`); console.log('Expected target observation:', JSON.stringify(expectedObservation)); - const observedCount = result.targetObservations?.filter((item) => item.outcome !== 'not-observed').length ?? 0; - console.log('Observed exact targets:', observedCount, '/', result.targetObservations?.length ?? 0); + if (expectedObservation.outcome !== 'passed') { + throw new Error(`Expected exact target to pass, got ${expectedObservation.outcome}`); + } + + const observed = result.targetObservations?.filter((item) => item.outcome !== 'not-observed') ?? []; + console.log('Observed exact targets:', observed.length, '/', result.targetObservations?.length ?? 0); + if (observed.length === 0) throw new Error('Observer still produced no exact target observations'); + const assessment = report.assessments.find((item) => item.file.path === changedPath); if (!assessment) throw new Error(`Missing assessment for ${changedPath}`); console.log('Changed source assessment:', assessment.status); console.log('Strongest evidence:', assessment.evidenceBoundary?.strongestEvidence); + if (assessment.status !== 'verified') { + throw new Error(`Expected source to be verified, got ${assessment.status}`); + } + if (assessment.evidenceBoundary?.strongestEvidence !== 'related-test-file-passed') { + throw new Error(`Unexpected strongest evidence: ${assessment.evidenceBoundary?.strongestEvidence}`); + } NODE vitest-monorepo-negative: @@ -117,9 +122,18 @@ jobs: - uses: actions/setup-node@v7 with: node-version: 24 - cache: npm - - run: npm ci - - run: npm run build + - name: Acquire exact ProofDiff fix commit + shell: bash + run: | + mkdir -p work/proofdiff-fixed + cd work/proofdiff-fixed + git init -q + git remote add origin https://github.com/hzw0813/proofdiff.git + git fetch --quiet --depth=1 origin "$PROOFDIFF_FIX_SHA" + git checkout --quiet --detach "$PROOFDIFF_FIX_SHA" + test "$(git rev-parse HEAD)" = "$PROOFDIFF_FIX_SHA" + npm ci + npm run build - name: Acquire pinned Vitest monorepo without installing dependencies shell: bash run: | @@ -131,20 +145,24 @@ jobs: git checkout --quiet --detach 667c13954daff2d1bec8a866702da8934b9ce539 test "$(git rev-parse HEAD)" = "667c13954daff2d1bec8a866702da8934b9ce539" printf '\n// ProofDiff runtime adoption validation: syntax-preserving no-op.\n' >> packages/utils/src/error.ts - - name: Verify root workspace orchestration is not invented as exact Vitest evidence + - name: Require root workspace orchestration to remain opaque shell: bash run: | node --input-type=module <<'NODE' import path from 'node:path'; - import { analyzeRepository } from './dist/analyze.js'; + import { pathToFileURL } from 'node:url'; + const proofdiffModule = pathToFileURL(path.resolve('work/proofdiff-fixed/dist/analyze.js')).href; + const { analyzeRepository } = await import(proofdiffModule); const root = path.resolve('work/vitest'); const report = await analyzeRepository({ repo: root, runChecks: false }); const rootTest = report.discoveredChecks.find((check) => check.id === 'js:test:test'); if (!rootTest) throw new Error('Expected root test check discovery'); const invented = report.discoveredChecks.filter((check) => check.id.startsWith('js:test:test:targeted:')); - console.log('Root test origin:', rootTest.origin); + console.log('Fix commit:', process.env.PROOFDIFF_FIX_SHA); console.log('Root test command:', rootTest.command, ...(rootTest.args ?? [])); console.log('Invented exact-target checks:', invented.length); - if (invented.length !== 0) throw new Error(`Workspace orchestration unexpectedly became exact-target evidence: ${invented.map((item) => item.id).join(', ')}`); + if (invented.length !== 0) { + throw new Error(`Workspace orchestration unexpectedly became exact-target evidence: ${invented.map((item) => item.id).join(', ')}`); + } NODE