From d14b39a5ddfd3ba5a923a1368226caa82ef1ebbc Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sun, 26 Jul 2026 11:25:21 -0400 Subject: [PATCH] ci: enforce aggregate test coverage --- .changeset/lemon-dodos-carry.md | 2 + .github/workflows/ci.yml | 12 ++++- .github/workflows/pr-ci.yml | 12 ++++- AGENTS.md | 1 + bunfig.coverage.toml | 5 ++ package.json | 1 + scripts/check-coverage.test.ts | 49 +++++++++++++++++++ scripts/check-coverage.ts | 83 +++++++++++++++++++++++++++++++++ test/README.md | 6 +++ 9 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 .changeset/lemon-dodos-carry.md create mode 100644 bunfig.coverage.toml create mode 100644 scripts/check-coverage.test.ts create mode 100644 scripts/check-coverage.ts diff --git a/.changeset/lemon-dodos-carry.md b/.changeset/lemon-dodos-carry.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/lemon-dodos-carry.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e16d71cc..76c4bf8d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,16 @@ jobs: - name: Theme contrast check run: bun run test:theme-contrast - - name: Test suite - run: bun run test + - name: Test suite with coverage + run: bun run test:coverage + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-main + path: coverage/lcov.info + if-no-files-found: warn - name: PTY integration tests run: bun run test:integration diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 75d9d883a..6f8f2ad72 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -175,8 +175,16 @@ jobs: - name: Theme contrast check run: bun run test:theme-contrast - - name: Test suite - run: bun run test + - name: Test suite with coverage + run: bun run test:coverage + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-pr + path: coverage/lcov.info + if-no-files-found: warn - name: PTY integration tests run: bun run test:integration diff --git a/AGENTS.md b/AGENTS.md index bfdc0ae08..583368ae7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,7 @@ CLI input - fast smoke test: `bun run src/main.tsx -- diff /tmp/before.ts /tmp/after.ts` - typecheck: `bun run typecheck` - tests: `bun test` +- tests with CI coverage thresholds: `bun run test:coverage` - PTY integration tests: `bun run test:integration` - TTY smoke test: `bun run test:tty-smoke` - format: `bun run format` diff --git a/bunfig.coverage.toml b/bunfig.coverage.toml new file mode 100644 index 000000000..5f788f6ca --- /dev/null +++ b/bunfig.coverage.toml @@ -0,0 +1,5 @@ +[test] +coverage = true +coverageReporter = ["text", "lcov"] +coverageDir = "coverage" +coveragePathIgnorePatterns = ["test/**"] diff --git a/package.json b/package.json index 9eac23d0b..fe4d6159c 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "release:version": "bunx @changesets/cli@2.31.0 version", "prepare": "simple-git-hooks", "test": "\"${npm_execpath:-bun}\" test ./src ./packages ./scripts ./test/cli ./test/session", + "test:coverage": "\"${npm_execpath:-bun}\" --config=./bunfig.coverage.toml test ./src ./packages ./scripts ./test/cli ./test/session && bun run ./scripts/check-coverage.ts", "test:theme-contrast": "bun test src/ui/themes.test.ts --test-name-pattern contrast", "test:integration": "\"${npm_execpath:-bun}\" test ./test/pty", "test:tty-smoke": "HUNK_RUN_TTY_SMOKE=1 \"${npm_execpath:-bun}\" test ./test/smoke", diff --git a/scripts/check-coverage.test.ts b/scripts/check-coverage.test.ts new file mode 100644 index 000000000..e391b6977 --- /dev/null +++ b/scripts/check-coverage.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; +import { + findCoverageThresholdFailures, + formatCoverageSummary, + parseLcovCoverageTotals, +} from "./check-coverage"; + +const report = `TN: +SF:src/first.ts +FNF:2 +FNH:2 +LF:10 +LH:9 +end_of_record +SF:src/second.ts +FNF:3 +FNH:2 +LF:20 +LH:18 +end_of_record +`; + +describe("coverage threshold", () => { + test("aggregates line and function totals across LCOV records", () => { + expect(parseLcovCoverageTotals(report)).toEqual({ + lines: { hit: 27, found: 30 }, + functions: { hit: 4, found: 5 }, + }); + }); + + test("reports each aggregate below the configured threshold", () => { + const totals = parseLcovCoverageTotals(report); + + expect(findCoverageThresholdFailures(totals)).toEqual(["functions: 80.00% is below 90%"]); + expect(findCoverageThresholdFailures(totals, 0.75)).toEqual([]); + }); + + test("formats totals for CI logs", () => { + expect(formatCoverageSummary(parseLcovCoverageTotals(report))).toBe( + "Coverage totals: 90.00% (27/30) lines, 80.00% (4/5) functions", + ); + }); + + test("rejects reports without usable coverage counters", () => { + expect(() => parseLcovCoverageTotals("TN:\nend_of_record\n")).toThrow( + "LCOV report contains no lines coverage data.", + ); + }); +}); diff --git a/scripts/check-coverage.ts b/scripts/check-coverage.ts new file mode 100644 index 000000000..31363f472 --- /dev/null +++ b/scripts/check-coverage.ts @@ -0,0 +1,83 @@ +#!/usr/bin/env bun + +export interface LcovCoverageTotals { + lines: { hit: number; found: number }; + functions: { hit: number; found: number }; +} + +export const MINIMUM_COVERAGE = 0.9; + +/** Parse aggregate line and function totals from an LCOV report. */ +export function parseLcovCoverageTotals(report: string): LcovCoverageTotals { + const totals: LcovCoverageTotals = { + lines: { hit: 0, found: 0 }, + functions: { hit: 0, found: 0 }, + }; + + for (const line of report.split(/\r?\n/)) { + const separator = line.indexOf(":"); + if (separator < 0) continue; + + const value = Number(line.slice(separator + 1)); + if (!Number.isSafeInteger(value) || value < 0) continue; + + switch (line.slice(0, separator)) { + case "LH": + totals.lines.hit += value; + break; + case "LF": + totals.lines.found += value; + break; + case "FNH": + totals.functions.hit += value; + break; + case "FNF": + totals.functions.found += value; + break; + } + } + + for (const [name, total] of Object.entries(totals)) { + if (total.found === 0) { + throw new Error(`LCOV report contains no ${name} coverage data.`); + } + if (total.hit > total.found) { + throw new Error(`LCOV report has more hit than found ${name}.`); + } + } + + return totals; +} + +/** Return human-readable failures for totals below the required ratio. */ +export function findCoverageThresholdFailures( + totals: LcovCoverageTotals, + minimum = MINIMUM_COVERAGE, +): string[] { + return Object.entries(totals).flatMap(([name, total]) => { + const ratio = total.hit / total.found; + return ratio < minimum + ? [`${name}: ${(ratio * 100).toFixed(2)}% is below ${(minimum * 100).toFixed(0)}%`] + : []; + }); +} + +/** Format one concise aggregate coverage summary for CI logs. */ +export function formatCoverageSummary(totals: LcovCoverageTotals): string { + const format = ({ hit, found }: { hit: number; found: number }) => + `${((hit / found) * 100).toFixed(2)}% (${hit}/${found})`; + return `Coverage totals: ${format(totals.lines)} lines, ${format(totals.functions)} functions`; +} + +if (import.meta.main) { + const reportPath = process.argv[2] ?? "coverage/lcov.info"; + const totals = parseLcovCoverageTotals(await Bun.file(reportPath).text()); + console.log(formatCoverageSummary(totals)); + + const failures = findCoverageThresholdFailures(totals); + if (failures.length > 0) { + throw new Error( + `Coverage threshold failed:\n${failures.map((failure) => `- ${failure}`).join("\n")}`, + ); + } +} diff --git a/test/README.md b/test/README.md index 05325d2b4..5864de125 100644 --- a/test/README.md +++ b/test/README.md @@ -35,3 +35,9 @@ These tests do not belong to a single source file. They usually verify product-l If a test mainly targets one module or helper, keep it colocated in `src/`. If it needs a real repo, subprocess, daemon, PTY, or transcript-level assertion, it likely belongs under `test/`. + +## Coverage + +Run `bun run test:coverage` to execute the main test suite, print Bun's coverage table, and write `coverage/lcov.info`. CI requires at least 90% line and function coverage across loaded production modules and uploads the LCOV report as a workflow artifact. + +Bun only measures modules loaded by the test process. Black-box subprocess, PTY, and TTY smoke coverage remains enforced by their dedicated CI steps but is not merged into the LCOV report.