From b0c162a4868cc854f8313a540b85c3f1ffcbe3d6 Mon Sep 17 00:00:00 2001 From: Haider Date: Mon, 16 Nov 2026 01:34:28 +0530 Subject: [PATCH 1/3] ci: split the TypeScript tests across parallel jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `TypeScript` job's "Run tests" step ran every test file in one bun process, one after another, and grew from about 6 minutes in June to 36 minutes. It was the whole critical path of a PR run. - `TypeScript (1/4 … 4/4)`: the main suite with `bun test --shard=N/4`, one process per job, so the `test/preload.ts` leak guard still holds - `TypeScript (CLI processes)`: the bounded subprocess pass, plus the five suites that start a fresh CLI per test (`learn/cli`, `learn/bootstrap-cli`, `tool-cli/cli`, `rules/host-cli`, `bridge-merge-e2e`), which the shards leave out with `--path-ignore-patterns` - `SDK codegen`: the codegen round-trip check in its own job - `script/github/check-bun-test-summary.sh`: the shared pass/fail summary check (bun can segfault on exit after passing, so the summary is parsed instead of the exit code) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XM3J9mB75wZRnzTSGcW2kc --- .github/workflows/ci.yml | 193 ++++++++++++++++-------- script/github/check-bun-test-summary.sh | 41 +++++ 2 files changed, 167 insertions(+), 67 deletions(-) create mode 100755 script/github/check-bun-test-summary.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a33850e799..2fed937274 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,12 +202,80 @@ jobs: # Main TypeScript tests — excludes driver E2E tests (separate job) and # cloud credential tests (local-only). # --------------------------------------------------------------------------- + # The main suite runs one bun process per job, one test file after another (test/preload.ts relies + # on that), so it is split by file across a matrix of shards. Suites that start a real CLI process + # run in `typescript-processes` beside them, and the SDK codegen check in `sdk-codegen`. typescript: - name: TypeScript + name: TypeScript (${{ matrix.shard }}/4) needs: changes if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.14" + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} + restore-keys: | + bun-${{ runner.os }}- + + - name: Configure git for tests + run: | + git config --global user.name "CI" + git config --global user.email "ci@test.local" + + - name: Install dependencies + run: bun install + + - name: Install merge tooling deps (script/upstream) + # The script/upstream/ workspace is its own bun project (depends on + # minimatch). Root `bun install` doesn't reach it. Tests in + # test/upstream/bridge-merge-e2e.test.ts shell out to + # `bun run script/upstream/analyze.ts --branding` which requires this. + # Mirrors the install step in the marker-guard job below. + run: bun install + working-directory: script/upstream + + - name: Run tests + working-directory: packages/opencode + # Cloud E2E tests (Snowflake, BigQuery, Databricks) auto-skip when + # ALTIMATE_CODE_CONN_* env vars are not set. Docker E2E tests auto-skip + # when Docker is not available. No exclusion needed — skipIf handles it. + # --timeout 90000: under CPU contention a handful of slower tests (real + # fs/spawn/git-bootstrap work) get starved and exceed a tight timeout + # NON-deterministically. 90s gives ~3x headroom over the worst observed. + # + # OPENCODE_SKIP_SUBPROCESS=1 makes cliIt skip the subprocess tests, and the + # --path-ignore-patterns leave out the suites that start a CLI per test; both + # run in `typescript-processes`. Keep that list in sync with PROCESS_SUITES there. + shell: bash + run: | + # Redirect bun output to a file, then cat for CI visibility (avoids tee/SIGTERM-flush issues). + OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 --shard=${{ matrix.shard }}/4 \ + --path-ignore-patterns='test/altimate/learn/cli.test.ts' --path-ignore-patterns='test/altimate/learn/bootstrap-cli.test.ts' --path-ignore-patterns='test/altimate/tool-cli/cli.test.ts' --path-ignore-patterns='test/altimate/rules/host-cli.test.ts' --path-ignore-patterns='test/upstream/bridge-merge-e2e.test.ts' \ + > /tmp/test-main.txt 2>&1 || true + cat /tmp/test-main.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-main.txt + + typescript-processes: + name: TypeScript (CLI processes) + needs: changes + if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -255,6 +323,62 @@ jobs: # hang needs sustained CPU pressure across many concurrent mock round-trips to manifest. # altimate_change end + + - name: Run tests + working-directory: packages/opencode + shell: bash + run: | + # Two runs: + # - SUBPROCESS: test/cli/{acp,smokes,serve} + run-process/mcp-add/help-snapshots, with + # --max-concurrency=2 (default 20). These each spawn a real CLI + an in-process mock LLM + # server; at 20-concurrent the runner is CPU-starved and the mock round-trips time out (the + # load-variance flakes). NOTE: do NOT add --parallel=1 — a single worker hosts the mock + # server AND runs the tests on one event loop, so the mock gets starved and round-trips + # time out anyway. Default --parallel spreads the mocks across workers. + # - PROCESS_SUITES: suites that start a fresh `bun run src` CLI per test; left out of the + # `typescript` shards (keep the lists in sync). + # (Bun 1.3.x can segfault during cleanup after all tests pass — the summary is parsed, not the + # exit code.) + SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/cli/auth-login-cancel.test.ts" + PROCESS_SUITES="test/altimate/learn/cli.test.ts test/altimate/learn/bootstrap-cli.test.ts test/altimate/tool-cli/cli.test.ts test/altimate/rules/host-cli.test.ts test/upstream/bridge-merge-e2e.test.ts" + + bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true + bun test --timeout 90000 $PROCESS_SUITES > /tmp/test-process.txt 2>&1 || true + echo "===== SUBPROCESS SUITE (bounded) ====="; cat /tmp/test-sub.txt + echo "===== CLI PROCESS SUITES ====="; cat /tmp/test-process.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-sub.txt /tmp/test-process.txt + + sdk-codegen: + name: SDK codegen + needs: changes + if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.14" + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} + restore-keys: | + bun-${{ runner.os }}- + + - name: Configure git for tests + run: | + git config --global user.name "CI" + git config --global user.email "ci@test.local" + + - name: Install dependencies + run: bun install + - name: SDK codegen is reproducible # The v2 gen tree is committed AND regenerated on every release build # (script/build.ts, clean: true) with post-codegen patches re-applied. @@ -305,71 +429,6 @@ jobs: fi fi exit "$drift_rc" - - name: Run tests - working-directory: packages/opencode - # Cloud E2E tests (Snowflake, BigQuery, Databricks) auto-skip when - # ALTIMATE_CODE_CONN_* env vars are not set. Docker E2E tests auto-skip - # when Docker is not available. No exclusion needed — skipIf handles it. - # --timeout 90000: the full suite (9500+ tests across 379 files) runs in - # one parallel bun process. Under CPU contention a handful of slower tests - # (real fs/spawn/git-bootstrap work) get starved and exceed a tight timeout - # NON-deterministically — different tests each run — failing CI with - # "timed out after Nms" (observed 32s/51s at the old 30s limit). 90s gives - # ~3x headroom over the worst observed, killing the resource-contention - # flakiness without masking genuinely-hung tests. - # - # Bun 1.3.x has a known segfault during process cleanup after all tests - # pass (exit code 143/SIGTERM or 134/SIGABRT). We capture test output and - # check for real failures vs Bun crashes to avoid false CI failures. - shell: bash - run: | - # Redirect bun output to files, then cat for CI visibility (avoids tee/SIGTERM-flush issues). - # - # Two passes: - # - MAIN: the full suite EXCEPT the subprocess tests (OPENCODE_SKIP_SUBPROCESS=1 makes cliIt - # skip them), at default parallelism — fast. - # - SUBPROCESS: test/cli/{acp,smokes,serve} + run-process/mcp-add/help-snapshots, run in a - # DEDICATED pass with --max-concurrency=2 (default 20). These each spawn a real CLI + an - # in-process mock LLM server; at 20-concurrent the runner is CPU-starved and the mock - # round-trips time out (the load-variance flakes). NOTE: do NOT add --parallel=1 — a single - # worker hosts the mock server AND runs the tests on one event loop, so the mock gets starved - # and round-trips time out anyway. Default --parallel spreads the mocks across workers; the - # pass is isolated from the 11k main-suite tests so 2-per-worker is plenty bounded. - # (Bun 1.3.x can segfault during cleanup after all tests pass — we parse the summary, not the - # exit code. --timeout 90000 gives headroom for slow fs/spawn tests.) - SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/cli/auth-login-cancel.test.ts" - - OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 > /tmp/test-main.txt 2>&1 || true - bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true - echo "===== MAIN SUITE ====="; cat /tmp/test-main.txt - echo "===== SUBPROCESS SUITE (bounded) ====="; cat /tmp/test-sub.txt - - # Sum pass/fail across both passes (one summary line each) - sum_field() { awk -v f="$1" '$0 ~ ("^ *[0-9]+ " f "$"){s+=$1} END{print s+0}' /tmp/test-main.txt /tmp/test-sub.txt; } - PASS_COUNT=$(sum_field pass) - FAIL_COUNT=$(sum_field fail) - - echo "" - echo "--- Test Summary (main + subprocess) ---" - echo "pass=$PASS_COUNT fail=$FAIL_COUNT" - - # Real test failures — always fail CI - if [ "$FAIL_COUNT" != "0" ]; then - echo "::error::$FAIL_COUNT test(s) failed" - exit 1 - fi - - # Each pass must have produced a summary, else Bun crashed before finishing - if ! grep -qE "^ *[0-9]+ pass$" /tmp/test-main.txt || ! grep -qE "^ *[0-9]+ pass$" /tmp/test-sub.txt; then - echo "::error::Missing test summary in a pass — Bun may have crashed before running tests" - exit 1 - fi - - if [ "$PASS_COUNT" -gt 0 ] 2>/dev/null; then - exit 0 - fi - echo "::error::No test results found" - exit 1 # --------------------------------------------------------------------------- # Driver E2E tests — only when driver code changes. diff --git a/script/github/check-bun-test-summary.sh b/script/github/check-bun-test-summary.sh new file mode 100755 index 0000000000..6c407ff7bb --- /dev/null +++ b/script/github/check-bun-test-summary.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Checks the saved output of one or more `bun test` runs and fails CI on a real test failure. +# +# Bun 1.3.x can segfault during process cleanup after every test passed (exit 143/SIGTERM or +# 134/SIGABRT), so the CI steps ignore bun's exit code and parse its summary lines instead. Each +# file must hold one run's output. Fails when any run reports failures, when a run printed no +# summary (bun crashed before finishing), or when nothing passed at all. +# +# Usage: check-bun-test-summary.sh ... +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +sum_field() { awk -v f="$1" '$0 ~ ("^ *[0-9]+ " f "$"){s+=$1} END{print s+0}' "${@:2}"; } +PASS_COUNT=$(sum_field pass "$@") +FAIL_COUNT=$(sum_field fail "$@") + +echo "" +echo "--- Test Summary ($# runs) ---" +echo "pass=$PASS_COUNT fail=$FAIL_COUNT" + +if [ "$FAIL_COUNT" != "0" ]; then + echo "::error::$FAIL_COUNT test(s) failed" + exit 1 +fi + +for file in "$@"; do + if ! grep -qE "^ *[0-9]+ pass$" "$file"; then + echo "::error::Missing test summary in $file — Bun may have crashed before running tests" + exit 1 + fi +done + +if [ "$PASS_COUNT" -gt 0 ] 2>/dev/null; then + exit 0 +fi +echo "::error::No test results found" +exit 1 From a2b16761f1613838faec3ba2e6a814928af31125 Mon Sep 17 00:00:00 2001 From: Haider Date: Mon, 16 Nov 2026 01:53:54 +0530 Subject: [PATCH 2/3] ci: run the TypeScript tests when the test summary script changes The test jobs decide pass or fail with `script/github/check-bun-test-summary.sh`, so a change to it runs them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XM3J9mB75wZRnzTSGcW2kc --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fed937274..382f1a0013 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,8 @@ jobs: - 'script/check-tracker-leaks.ts' - 'script/upstream/**' # altimate_change end + # The test jobs decide pass/fail with this script. + - 'script/github/check-bun-test-summary.sh' # altimate_change start — a change to this workflow runs the Typecheck job on # its own PR, without also starting the 60-minute TypeScript test job. typecheck: From d50c8c6c91a0f4bcefb15c9b0422c4cd4d8d91c0 Mon Sep 17 00:00:00 2001 From: Haider Date: Mon, 16 Nov 2026 02:20:09 +0530 Subject: [PATCH 3/3] ci: split the CLI-process tests in two and address review comments - `TypeScript (CLI processes, solo|rest)`: `learn/cli` (about 6 min) runs on its own, and the subprocess pass plus the other four CLI suites run beside it; Bun runs these files one after another, so one job took 14.6 min - One list of CLI suites, `PROCESS_SUITES_SOLO` and `PROCESS_SUITES` at the top of the workflow, drives both the shards' `--path-ignore-patterns` and the CLI-process jobs - `check-bun-test-summary.sh` also fails on bun's " N error(s)" line (errors outside a test), with tests in `test/ci-bun-test-summary.test.ts` - The cold-start job's comment points at `typescript-processes`; the SDK codegen job drops the unused git identity step Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XM3J9mB75wZRnzTSGcW2kc --- .github/workflows/ci.yml | 50 +++++++++---- .../opencode/test/ci-bun-test-summary.test.ts | 73 +++++++++++++++++++ script/github/check-bun-test-summary.sh | 13 +++- 3 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 packages/opencode/test/ci-bun-test-summary.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 382f1a0013..e9a7d7150d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,16 @@ concurrency: permissions: contents: read +# Test suites that start a fresh CLI process per test. The `typescript` shards leave them out, and +# `typescript-processes` runs them: the slowest on its own (`solo`), the rest beside the subprocess pass. +env: + PROCESS_SUITES_SOLO: test/altimate/learn/cli.test.ts + PROCESS_SUITES: >- + test/altimate/learn/bootstrap-cli.test.ts + test/altimate/tool-cli/cli.test.ts + test/altimate/rules/host-cli.test.ts + test/upstream/bridge-merge-e2e.test.ts + jobs: # --------------------------------------------------------------------------- # Detect which areas of the codebase changed to skip unaffected jobs. @@ -261,23 +271,31 @@ jobs: # NON-deterministically. 90s gives ~3x headroom over the worst observed. # # OPENCODE_SKIP_SUBPROCESS=1 makes cliIt skip the subprocess tests, and the - # --path-ignore-patterns leave out the suites that start a CLI per test; both - # run in `typescript-processes`. Keep that list in sync with PROCESS_SUITES there. + # suites in PROCESS_SUITES_SOLO and PROCESS_SUITES (top of this file) are left out; + # both run in `typescript-processes`. shell: bash + env: + SHARD: ${{ matrix.shard }} run: | + IGNORE=() + for suite in $PROCESS_SUITES_SOLO $PROCESS_SUITES; do IGNORE+=("--path-ignore-patterns=$suite"); done # Redirect bun output to a file, then cat for CI visibility (avoids tee/SIGTERM-flush issues). - OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 --shard=${{ matrix.shard }}/4 \ - --path-ignore-patterns='test/altimate/learn/cli.test.ts' --path-ignore-patterns='test/altimate/learn/bootstrap-cli.test.ts' --path-ignore-patterns='test/altimate/tool-cli/cli.test.ts' --path-ignore-patterns='test/altimate/rules/host-cli.test.ts' --path-ignore-patterns='test/upstream/bridge-merge-e2e.test.ts' \ + OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 --shard="$SHARD/4" "${IGNORE[@]}" \ > /tmp/test-main.txt 2>&1 || true cat /tmp/test-main.txt bash ../../script/github/check-bun-test-summary.sh /tmp/test-main.txt typescript-processes: - name: TypeScript (CLI processes) + name: TypeScript (CLI processes, ${{ matrix.part }}) needs: changes if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + # Bun runs these files one after another, so they are split into two jobs of similar length. + part: [solo, rest] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -329,21 +347,26 @@ jobs: - name: Run tests working-directory: packages/opencode shell: bash + env: + PART: ${{ matrix.part }} run: | - # Two runs: + # `solo`: PROCESS_SUITES_SOLO. `rest`: two runs, # - SUBPROCESS: test/cli/{acp,smokes,serve} + run-process/mcp-add/help-snapshots, with # --max-concurrency=2 (default 20). These each spawn a real CLI + an in-process mock LLM # server; at 20-concurrent the runner is CPU-starved and the mock round-trips time out (the # load-variance flakes). NOTE: do NOT add --parallel=1 — a single worker hosts the mock # server AND runs the tests on one event loop, so the mock gets starved and round-trips # time out anyway. Default --parallel spreads the mocks across workers. - # - PROCESS_SUITES: suites that start a fresh `bun run src` CLI per test; left out of the - # `typescript` shards (keep the lists in sync). + # - PROCESS_SUITES: suites that start a fresh `bun run src` CLI per test. # (Bun 1.3.x can segfault during cleanup after all tests pass — the summary is parsed, not the # exit code.) + if [ "$PART" = "solo" ]; then + bun test --timeout 90000 $PROCESS_SUITES_SOLO > /tmp/test-process.txt 2>&1 || true + echo "===== CLI PROCESS SUITES (solo) ====="; cat /tmp/test-process.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-process.txt + exit + fi SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/cli/auth-login-cancel.test.ts" - PROCESS_SUITES="test/altimate/learn/cli.test.ts test/altimate/learn/bootstrap-cli.test.ts test/altimate/tool-cli/cli.test.ts test/altimate/rules/host-cli.test.ts test/upstream/bridge-merge-e2e.test.ts" - bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true bun test --timeout 90000 $PROCESS_SUITES > /tmp/test-process.txt 2>&1 || true echo "===== SUBPROCESS SUITE (bounded) ====="; cat /tmp/test-sub.txt @@ -373,11 +396,6 @@ jobs: restore-keys: | bun-${{ runner.os }}- - - name: Configure git for tests - run: | - git config --global user.name "CI" - git config --global user.email "ci@test.local" - - name: Install dependencies run: bun install @@ -866,7 +884,7 @@ jobs: MODELS_DEV_API_JSON: test/tool/fixtures/models-api.json # --version and PURE-mode tests never exercise ordinary config installs. This is the - # deliberate exception to the "no OPENCODE_TEST_CLI" NOTE in the `typescript` job above: a + # deliberate exception to the "no OPENCODE_TEST_CLI" NOTE in the `typescript-processes` job above: a # single bounded cold-start check with its own --timeout, not the general run+mock subprocess # suite, so the compiled-binary load hang that NOTE warns about cannot stall CI here. - name: Cold-start config regression (compiled, non-PURE) diff --git a/packages/opencode/test/ci-bun-test-summary.test.ts b/packages/opencode/test/ci-bun-test-summary.test.ts new file mode 100644 index 0000000000..21acb7eab9 --- /dev/null +++ b/packages/opencode/test/ci-bun-test-summary.test.ts @@ -0,0 +1,73 @@ +/** + * Every TypeScript CI job decides pass/fail with script/github/check-bun-test-summary.sh, which + * parses bun's summary lines instead of its exit code (bun can segfault on exit after passing). + * These tests drive the script with synthetic `bun test` output so a parsing change cannot + * silently turn a failing run green. + */ +import { describe, expect, it } from "bun:test" +import { spawnSync } from "node:child_process" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" + +const script = join(import.meta.dir, "../../../script/github/check-bun-test-summary.sh") + +function run(...outputs: string[]): number { + const dir = mkdtempSync(join(tmpdir(), "bun-summary-")) + try { + const files = outputs.map((output, i) => { + const file = join(dir, `run-${i}.txt`) + writeFileSync(file, output) + return file + }) + return spawnSync("bash", [script, ...files]).status ?? -1 + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +const summary = (pass: number, fail: number, extra = "") => + `(pass) a > b [1.00ms]\n\n ${pass} pass\n 3 skip\n ${fail} fail\n${extra}Ran ${pass + fail + 3} tests across 2 files. [1.00s]\n` + +describe("check-bun-test-summary.sh", () => { + it("passes a run with passing tests and no failures", () => { + expect(run(summary(12, 0))).toBe(0) + }) + + it("passes several clean runs together", () => { + expect(run(summary(12, 0), summary(3, 0))).toBe(0) + }) + + it("passes a run that crashed on exit after its summary (bun's cleanup segfault)", () => { + expect(run(summary(12, 0) + "Segmentation fault (core dumped)\n")).toBe(0) + }) + + it("fails a run with a failed test", () => { + expect(run(summary(11, 1))).toBe(1) + }) + + it("fails when any one of several runs failed", () => { + expect(run(summary(12, 0), summary(2, 1))).toBe(1) + }) + + it("fails a run with an error outside a test", () => { + expect(run(summary(12, 0, " 1 error\n"))).toBe(1) + expect(run(summary(12, 0, " 2 errors\n"))).toBe(1) + }) + + it("fails a run that printed no summary (bun crashed before finishing)", () => { + expect(run(summary(12, 0), "(pass) a > b [1.00ms]\nSegmentation fault\n")).toBe(1) + }) + + it("fails when nothing passed", () => { + expect(run(summary(0, 0))).toBe(1) + }) + + it("does not count test names that end in pass or fail", () => { + expect(run(`(pass) handles 3 fail\n${summary(4, 0)}`)).toBe(0) + }) + + it("rejects a call with no output files", () => { + expect(run()).toBe(2) + }) +}) diff --git a/script/github/check-bun-test-summary.sh b/script/github/check-bun-test-summary.sh index 6c407ff7bb..114b6fcc9a 100755 --- a/script/github/check-bun-test-summary.sh +++ b/script/github/check-bun-test-summary.sh @@ -3,8 +3,9 @@ # # Bun 1.3.x can segfault during process cleanup after every test passed (exit 143/SIGTERM or # 134/SIGABRT), so the CI steps ignore bun's exit code and parse its summary lines instead. Each -# file must hold one run's output. Fails when any run reports failures, when a run printed no -# summary (bun crashed before finishing), or when nothing passed at all. +# file must hold one run's output. Fails when any run reports failures or errors outside a test +# (bun's " N error(s)" line), when a run printed no summary (bun crashed before finishing), or when +# nothing passed at all. # # Usage: check-bun-test-summary.sh ... set -euo pipefail @@ -17,16 +18,22 @@ fi sum_field() { awk -v f="$1" '$0 ~ ("^ *[0-9]+ " f "$"){s+=$1} END{print s+0}' "${@:2}"; } PASS_COUNT=$(sum_field pass "$@") FAIL_COUNT=$(sum_field fail "$@") +ERROR_COUNT=$(( $(sum_field error "$@") + $(sum_field errors "$@") )) echo "" echo "--- Test Summary ($# runs) ---" -echo "pass=$PASS_COUNT fail=$FAIL_COUNT" +echo "pass=$PASS_COUNT fail=$FAIL_COUNT error=$ERROR_COUNT" if [ "$FAIL_COUNT" != "0" ]; then echo "::error::$FAIL_COUNT test(s) failed" exit 1 fi +if [ "$ERROR_COUNT" != "0" ]; then + echo "::error::$ERROR_COUNT error(s) outside a test" + exit 1 +fi + for file in "$@"; do if ! grep -qE "^ *[0-9]+ pass$" "$file"; then echo "::error::Missing test summary in $file — Bun may have crashed before running tests"