diff --git a/bin/xflow.js b/bin/xflow.js index 57188a4..c40ea8a 100755 --- a/bin/xflow.js +++ b/bin/xflow.js @@ -7027,10 +7027,11 @@ Usage: xflow audit [--base ] [--claims-file ] [--json] [--o Audit any git diff / PR without adopting evalid workflows: extract the test runs the change claims to have passed, replay each one in a clean clone at the change's head, and compare exit codes and pass/fail counts against the -claim. bun test claims are recognised, and the replay tree's node_modules -is checked against the lockfile (or the source snapshot when there is no -lockfile): a missing snapshot with unchanged dependencies is not a -mismatch, while a real inconsistency names the exact differing files. +claim. bun test, pnpm/yarn script runs, vitest and jest claims are +recognised, and the replay tree's node_modules is checked against the +lockfile (or the source snapshot when there is no lockfile): a missing +snapshot with unchanged dependencies is not a mismatch, while a real +inconsistency names the exact differing files. Diffs carrying an evalid proof seal are replayed through the existing i6b seal validator. When the diff touches Python test files and the audited repo carries the I6d plugin, the I6d assertion-execution gate also runs in diff --git a/src/core/audit.js b/src/core/audit.js index bf75f77..6a18540 100644 --- a/src/core/audit.js +++ b/src/core/audit.js @@ -58,34 +58,117 @@ * --locked when Cargo.lock exists so the locked versions are * honoured exactly). A failed supply is unverifiable, never a silent * run without dependencies — the same rule as a failed npm install. + * + * Claim harvesting now reads one command-family table to both recognise a + * claimed run and cut it out of the sentence. The two lists used to be + * written by hand and had drifted: `pnpm run test` was harvested as + * `npm run test` (the npm family matched inside the longer word), and + * `yarn run test` / `vitest run` / `jest` were not recognised at all, so + * a PR that quoted them was audited as no_claims. vitest and jest count + * only inside backticks — bare "we use jest" stays prose — and their + * replay summaries ("Tests 18 passed (18)" / "Tests: 18 passed, + * 18 total") are parsed so a count overstatement is still a mismatch. */ import { spawnSync } from 'node:child_process'; import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -/** Commands that count as claimed test runs. */ -export const TEST_COMMAND_PATTERNS = [ - /npm\s+test\b/, - /npm\s+run\s+[\w:.-]+/, - /pnpm\s+test\b/, - /yarn\s+test\b/, - /node\s+--test\b/, - /pytest\b/, - /python3?\s+-m\s+pytest\b/, - /python3?\s+-m\s+unittest\b/, - /go\s+test\b/, - /cargo\s+test\b/, - /mvn\s+(?:-[^ ]+\s+)*test\b/, - /npx\s+[\w@./-]+/, - /bun\s+test\b/, - /bun\s+run\s+[\w:.-]+/, +/** + * One row per command family: the shape a claim may take in prose + * (`detect`, what isTestCommand matches) and the runnable slice harvested + * out of it (`tail`). Both command lists are derived from this table, so a + * runner added here cannot be recognised by one list and truncated by the + * other. The two lists had drifted apart once: `pnpm run test` was + * harvested as `npm run test`, because the npm family matched from index 1 + * inside "pnpm" and the leftmost match won. The tails are therefore + * anchored on a token edge below, so a match can only start at the command. + * + * `bare: false` marks a family that only counts when the claim puts it in + * backticks (`vitest`, `jest`): "we use jest for unit tests" is prose about + * a runner, while "`jest --coverage`" is a command the author says they ran. + */ +const COMMAND_CHARS = String.raw`[\w@./:+\-='"]`; +const COMMAND_EDGE = String.raw`(? family.detect); + +function buildCommandTail(families) { + return new RegExp(`${COMMAND_EDGE}(?:${families.map((family) => family.tail).join('|')})`); +} + +/** Harvest inside backticks: every family, the bare-word ones included. */ +const COMMAND_TAIL = buildCommandTail(TEST_COMMAND_FAMILIES); +/** Harvest from prose: only families that read as a command on their own. */ +const COMMAND_TAIL_BARE = buildCommandTail(TEST_COMMAND_FAMILIES.filter((family) => family.bare !== false)); export function isTestCommand(text) { return TEST_COMMAND_PATTERNS.some((pattern) => pattern.test(text)); @@ -320,12 +403,15 @@ export function extractClaims(text) { const clauses = line.split(/(?:[.;]\s+|\s+;\s*)/); for (const clause of clauses) { const commands = new Set(); + // A fenced span is a command the author chose to quote, so every + // family counts there; unlocked prose only yields families that + // stand on their own (see TEST_COMMAND_FAMILIES' bare flag). const spans = [...clause.matchAll(/`([^`]+)`/g)].map((span) => span[1]); for (const span of spans) { const found = span.match(COMMAND_TAIL); if (found) commands.add(normalizeCommand(found[0])); } - const bare = clause.match(COMMAND_TAIL); + const bare = clause.match(COMMAND_TAIL_BARE); if (bare) commands.add(normalizeCommand(bare[0])); if (commands.size === 0) continue; const counts = parseClaimCounts(clause); @@ -415,11 +501,34 @@ export function parseDiffFiles(diffText) { return { files, testFiles: files.filter(isTestFile), seals }; } +/** + * Read "N passed" / "N failed" out of one vitest or jest summary line. + * Both print the counts in either order, separated by "|" or ",", and the + * total either in parentheses (vitest) or as "N total" (jest). A line + * without a count word is not a summary line and yields null. + */ +function parseSummaryLine(line) { + const passed = /\b(\d+)\s+passed\b/.exec(line); + const failed = /\b(\d+)\s+failed\b/.exec(line); + if (!passed && !failed) return null; + const passedCount = passed ? Number(passed[1]) : 0; + const failedCount = failed ? Number(failed[1]) : 0; + const parenthesized = /\((\d+)\)\s*$/.exec(line); + const totalToken = /\b(\d+)\s+total\b/.exec(line); + const total = parenthesized + ? Number(parenthesized[1]) + : totalToken + ? Number(totalToken[1]) + : passedCount + failedCount; + return { passed: passedCount, failed: failedCount, total }; +} + /** * Parse pass/fail counts out of a test runner's output. * node --test ("# pass 18", "# fail 0"), pytest ("18 passed, 0 failed"), * python -m unittest ("Ran 18 tests" + "OK" / "FAILED (failures=1, errors=0)"), - * mocha ("18 passing"), npm-wrapped node --test (same "# pass" lines). + * mocha ("18 passing"), npm-wrapped node --test (same "# pass" lines), + * vitest ("Tests 18 passed (18)") and jest ("Tests: 18 passed, 18 total"). */ export function parseReplayCounts(output, command) { const text = String(output || ''); @@ -452,6 +561,20 @@ export function parseReplayCounts(output, command) { return { passed: bunPass, failed: bunFail, total }; } if (bunPasses.length > 1 || bunFails.length > 1) return null; + // vitest and jest print one "Tests" summary per project. The marker word + // is mandatory (unlike bun's count-first pair), so a test whose name + // starts with "Tests " cannot fire it: + // vitest: " Tests 18 passed | 2 failed (20)" + // jest: "Tests: 18 passed, 2 failed, 20 total" + // A monorepo run prints one block per project, which is not a single + // total, so several markers report unparsed like the multi-block node + // case above. (Shape verified against vitest 3 and jest 30 output.) + const summaryMarkers = text.match(/^[ \t]*Tests(?::[ \t]+|[ \t]+)(.+)$/gm); + if (summaryMarkers && summaryMarkers.length > 1) return null; + if (summaryMarkers) { + const summary = parseSummaryLine(summaryMarkers[0]); + if (summary) return summary; + } // go test -v prints one "--- PASS:" / "--- FAIL:" line per test. // Only top-level tests count: a test name without "/" (e.g. // "TestAdd"). Subtest lines carry "/" in the name diff --git a/test/xflow-audit.test.js b/test/xflow-audit.test.js index 311ccf9..c01ba0c 100644 --- a/test/xflow-audit.test.js +++ b/test/xflow-audit.test.js @@ -210,6 +210,40 @@ describe('extractClaims', () => { assert.equal(claims[1].claimedPassed, 18); assert.equal(claims[1].claimedFailed, 0); }); + + test('keeps pnpm/yarn script runs whole instead of trimming them to npm', () => { + // `pnpm run test` used to come back as `npm run test`: the npm family + // matched from index 1 inside "pnpm" and the leftmost match won, so the + // replay ran a different command than the one claimed. `yarn run test` + // was not recognised at all, which made a body that named its run + // audit as no_claims. + const body = [ + 'Full suite: `pnpm run test`: 50 files, 440 tests, 439 passed, 0 failed.', + 'Also `yarn run test`: exit 0.', + ].join('\n'); + const claims = extractClaims(body); + assert.deepEqual(claims.map((claim) => claim.command), ['pnpm run test', 'yarn run test']); + assert.equal(claims[0].claimedPassed, 439); + assert.equal(claims[1].claimedExit, 0); + }); + + test('reads vitest and jest claims from backticks, never from prose', () => { + // These are the runners a JS PR most often names. A fenced span is a + // command the author says they ran; the same word in prose is a + // statement about tooling ("we use jest"), and replaying `jest` from it + // would run a binary the repo may not even depend on. + assert.deepEqual(extractClaims('We use vitest and jest for unit tests.'), []); + const body = [ + 'Unit tests: `vitest run` — 18 passed, 0 failed.', + 'Coverage: `yarn jest --coverage` — exit 0.', + ].join('\n'); + const claims = extractClaims(body); + // The package-manager prefix stays on the command: replaying the bare + // runner would run something other than what the author claimed. + assert.deepEqual(claims.map((claim) => claim.command), ['vitest run', 'yarn jest --coverage']); + assert.equal(claims[0].claimedPassed, 18); + assert.equal(claims[1].claimedExit, 0); + }); }); describe('resolveClaimCwd (M2 slice 2, #116)', () => { @@ -368,6 +402,31 @@ describe('parseReplayCounts', () => { { passed: 3, failed: 1, total: 4 }, ); }); + + test('reads vitest and jest summary lines, and leaves a multi-project run unparsed', () => { + // vitest 3 prints one count-first "Tests" line with the total in + // parentheses; jest 30 prints the same counts with "N total" instead. + // Both orders show up (failed first when a test fails). + assert.deepEqual( + parseReplayCounts(' Test Files 2 passed (2)\n Tests 18 passed (18)\n Duration 1.2s\n'), + { passed: 18, failed: 0, total: 18 }, + ); + assert.deepEqual( + parseReplayCounts(' Tests 1 failed | 17 passed (18)\n'), + { passed: 17, failed: 1, total: 18 }, + ); + assert.deepEqual(parseReplayCounts('Tests: 18 passed, 18 total\n'), { passed: 18, failed: 0, total: 18 }); + assert.deepEqual( + parseReplayCounts('Tests: 2 failed, 16 passed, 20 total, 2 todo\n'), + { passed: 16, failed: 2, total: 20 }, + ); + // One block per project: several blocks are not a suite total, the same + // rule the multi-block node --test case above follows. + assert.equal(parseReplayCounts(' Tests 18 passed (18)\n Tests 4 passed (4)\n'), null); + // A "Tests" marker without counts (a runner that printed nothing) is not + // a summary and must not invent a zero-test total. + assert.equal(parseReplayCounts('Tests: no test files found\n'), null); + }); }); describe('node_modules snapshot consistency (M6 slice 1, #132)', () => {