diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a33850e799..e9a7d7150d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,16 @@ concurrency: permissions: contents: read +# Test suites that start a fresh CLI process per test. The `typescript` shards leave them out, and +# `typescript-processes` runs them: the slowest on its own (`solo`), the rest beside the subprocess pass. +env: + PROCESS_SUITES_SOLO: test/altimate/learn/cli.test.ts + PROCESS_SUITES: >- + test/altimate/learn/bootstrap-cli.test.ts + test/altimate/tool-cli/cli.test.ts + test/altimate/rules/host-cli.test.ts + test/upstream/bridge-merge-e2e.test.ts + jobs: # --------------------------------------------------------------------------- # Detect which areas of the codebase changed to skip unaffected jobs. @@ -81,6 +91,8 @@ jobs: - 'script/check-tracker-leaks.ts' - 'script/upstream/**' # altimate_change end + # The test jobs decide pass/fail with this script. + - 'script/github/check-bun-test-summary.sh' # altimate_change start — a change to this workflow runs the Typecheck job on # its own PR, without also starting the 60-minute TypeScript test job. typecheck: @@ -202,12 +214,88 @@ jobs: # Main TypeScript tests — excludes driver E2E tests (separate job) and # cloud credential tests (local-only). # --------------------------------------------------------------------------- + # The main suite runs one bun process per job, one test file after another (test/preload.ts relies + # on that), so it is split by file across a matrix of shards. Suites that start a real CLI process + # run in `typescript-processes` beside them, and the SDK codegen check in `sdk-codegen`. typescript: - name: TypeScript + name: TypeScript (${{ matrix.shard }}/4) needs: changes if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.14" + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} + restore-keys: | + bun-${{ runner.os }}- + + - name: Configure git for tests + run: | + git config --global user.name "CI" + git config --global user.email "ci@test.local" + + - name: Install dependencies + run: bun install + + - name: Install merge tooling deps (script/upstream) + # The script/upstream/ workspace is its own bun project (depends on + # minimatch). Root `bun install` doesn't reach it. Tests in + # test/upstream/bridge-merge-e2e.test.ts shell out to + # `bun run script/upstream/analyze.ts --branding` which requires this. + # Mirrors the install step in the marker-guard job below. + run: bun install + working-directory: script/upstream + + - name: Run tests + working-directory: packages/opencode + # Cloud E2E tests (Snowflake, BigQuery, Databricks) auto-skip when + # ALTIMATE_CODE_CONN_* env vars are not set. Docker E2E tests auto-skip + # when Docker is not available. No exclusion needed — skipIf handles it. + # --timeout 90000: under CPU contention a handful of slower tests (real + # fs/spawn/git-bootstrap work) get starved and exceed a tight timeout + # NON-deterministically. 90s gives ~3x headroom over the worst observed. + # + # OPENCODE_SKIP_SUBPROCESS=1 makes cliIt skip the subprocess tests, and the + # suites in PROCESS_SUITES_SOLO and PROCESS_SUITES (top of this file) are left out; + # both run in `typescript-processes`. + shell: bash + env: + SHARD: ${{ matrix.shard }} + run: | + IGNORE=() + for suite in $PROCESS_SUITES_SOLO $PROCESS_SUITES; do IGNORE+=("--path-ignore-patterns=$suite"); done + # Redirect bun output to a file, then cat for CI visibility (avoids tee/SIGTERM-flush issues). + OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 --shard="$SHARD/4" "${IGNORE[@]}" \ + > /tmp/test-main.txt 2>&1 || true + cat /tmp/test-main.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-main.txt + + typescript-processes: + name: TypeScript (CLI processes, ${{ matrix.part }}) + needs: changes + if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + # Bun runs these files one after another, so they are split into two jobs of similar length. + part: [solo, rest] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -255,6 +343,62 @@ jobs: # hang needs sustained CPU pressure across many concurrent mock round-trips to manifest. # altimate_change end + + - name: Run tests + working-directory: packages/opencode + shell: bash + env: + PART: ${{ matrix.part }} + run: | + # `solo`: PROCESS_SUITES_SOLO. `rest`: two runs, + # - SUBPROCESS: test/cli/{acp,smokes,serve} + run-process/mcp-add/help-snapshots, with + # --max-concurrency=2 (default 20). These each spawn a real CLI + an in-process mock LLM + # server; at 20-concurrent the runner is CPU-starved and the mock round-trips time out (the + # load-variance flakes). NOTE: do NOT add --parallel=1 — a single worker hosts the mock + # server AND runs the tests on one event loop, so the mock gets starved and round-trips + # time out anyway. Default --parallel spreads the mocks across workers. + # - PROCESS_SUITES: suites that start a fresh `bun run src` CLI per test. + # (Bun 1.3.x can segfault during cleanup after all tests pass — the summary is parsed, not the + # exit code.) + if [ "$PART" = "solo" ]; then + bun test --timeout 90000 $PROCESS_SUITES_SOLO > /tmp/test-process.txt 2>&1 || true + echo "===== CLI PROCESS SUITES (solo) ====="; cat /tmp/test-process.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-process.txt + exit + fi + SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/cli/auth-login-cancel.test.ts" + bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true + bun test --timeout 90000 $PROCESS_SUITES > /tmp/test-process.txt 2>&1 || true + echo "===== SUBPROCESS SUITE (bounded) ====="; cat /tmp/test-sub.txt + echo "===== CLI PROCESS SUITES ====="; cat /tmp/test-process.txt + bash ../../script/github/check-bun-test-summary.sh /tmp/test-sub.txt /tmp/test-process.txt + + sdk-codegen: + name: SDK codegen + needs: changes + if: needs.changes.outputs.typescript == 'true' || github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.14" + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} + restore-keys: | + bun-${{ runner.os }}- + + - name: Install dependencies + run: bun install + - name: SDK codegen is reproducible # The v2 gen tree is committed AND regenerated on every release build # (script/build.ts, clean: true) with post-codegen patches re-applied. @@ -305,71 +449,6 @@ jobs: fi fi exit "$drift_rc" - - name: Run tests - working-directory: packages/opencode - # Cloud E2E tests (Snowflake, BigQuery, Databricks) auto-skip when - # ALTIMATE_CODE_CONN_* env vars are not set. Docker E2E tests auto-skip - # when Docker is not available. No exclusion needed — skipIf handles it. - # --timeout 90000: the full suite (9500+ tests across 379 files) runs in - # one parallel bun process. Under CPU contention a handful of slower tests - # (real fs/spawn/git-bootstrap work) get starved and exceed a tight timeout - # NON-deterministically — different tests each run — failing CI with - # "timed out after Nms" (observed 32s/51s at the old 30s limit). 90s gives - # ~3x headroom over the worst observed, killing the resource-contention - # flakiness without masking genuinely-hung tests. - # - # Bun 1.3.x has a known segfault during process cleanup after all tests - # pass (exit code 143/SIGTERM or 134/SIGABRT). We capture test output and - # check for real failures vs Bun crashes to avoid false CI failures. - shell: bash - run: | - # Redirect bun output to files, then cat for CI visibility (avoids tee/SIGTERM-flush issues). - # - # Two passes: - # - MAIN: the full suite EXCEPT the subprocess tests (OPENCODE_SKIP_SUBPROCESS=1 makes cliIt - # skip them), at default parallelism — fast. - # - SUBPROCESS: test/cli/{acp,smokes,serve} + run-process/mcp-add/help-snapshots, run in a - # DEDICATED pass with --max-concurrency=2 (default 20). These each spawn a real CLI + an - # in-process mock LLM server; at 20-concurrent the runner is CPU-starved and the mock - # round-trips time out (the load-variance flakes). NOTE: do NOT add --parallel=1 — a single - # worker hosts the mock server AND runs the tests on one event loop, so the mock gets starved - # and round-trips time out anyway. Default --parallel spreads the mocks across workers; the - # pass is isolated from the 11k main-suite tests so 2-per-worker is plenty bounded. - # (Bun 1.3.x can segfault during cleanup after all tests pass — we parse the summary, not the - # exit code. --timeout 90000 gives headroom for slow fs/spawn tests.) - SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/cli/auth-login-cancel.test.ts" - - OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 > /tmp/test-main.txt 2>&1 || true - bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true - echo "===== MAIN SUITE ====="; cat /tmp/test-main.txt - echo "===== SUBPROCESS SUITE (bounded) ====="; cat /tmp/test-sub.txt - - # Sum pass/fail across both passes (one summary line each) - sum_field() { awk -v f="$1" '$0 ~ ("^ *[0-9]+ " f "$"){s+=$1} END{print s+0}' /tmp/test-main.txt /tmp/test-sub.txt; } - PASS_COUNT=$(sum_field pass) - FAIL_COUNT=$(sum_field fail) - - echo "" - echo "--- Test Summary (main + subprocess) ---" - echo "pass=$PASS_COUNT fail=$FAIL_COUNT" - - # Real test failures — always fail CI - if [ "$FAIL_COUNT" != "0" ]; then - echo "::error::$FAIL_COUNT test(s) failed" - exit 1 - fi - - # Each pass must have produced a summary, else Bun crashed before finishing - if ! grep -qE "^ *[0-9]+ pass$" /tmp/test-main.txt || ! grep -qE "^ *[0-9]+ pass$" /tmp/test-sub.txt; then - echo "::error::Missing test summary in a pass — Bun may have crashed before running tests" - exit 1 - fi - - if [ "$PASS_COUNT" -gt 0 ] 2>/dev/null; then - exit 0 - fi - echo "::error::No test results found" - exit 1 # --------------------------------------------------------------------------- # Driver E2E tests — only when driver code changes. @@ -805,7 +884,7 @@ jobs: MODELS_DEV_API_JSON: test/tool/fixtures/models-api.json # --version and PURE-mode tests never exercise ordinary config installs. This is the - # deliberate exception to the "no OPENCODE_TEST_CLI" NOTE in the `typescript` job above: a + # deliberate exception to the "no OPENCODE_TEST_CLI" NOTE in the `typescript-processes` job above: a # single bounded cold-start check with its own --timeout, not the general run+mock subprocess # suite, so the compiled-binary load hang that NOTE warns about cannot stall CI here. - name: Cold-start config regression (compiled, non-PURE) diff --git a/packages/opencode/test/ci-bun-test-summary.test.ts b/packages/opencode/test/ci-bun-test-summary.test.ts new file mode 100644 index 0000000000..21acb7eab9 --- /dev/null +++ b/packages/opencode/test/ci-bun-test-summary.test.ts @@ -0,0 +1,73 @@ +/** + * Every TypeScript CI job decides pass/fail with script/github/check-bun-test-summary.sh, which + * parses bun's summary lines instead of its exit code (bun can segfault on exit after passing). + * These tests drive the script with synthetic `bun test` output so a parsing change cannot + * silently turn a failing run green. + */ +import { describe, expect, it } from "bun:test" +import { spawnSync } from "node:child_process" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" + +const script = join(import.meta.dir, "../../../script/github/check-bun-test-summary.sh") + +function run(...outputs: string[]): number { + const dir = mkdtempSync(join(tmpdir(), "bun-summary-")) + try { + const files = outputs.map((output, i) => { + const file = join(dir, `run-${i}.txt`) + writeFileSync(file, output) + return file + }) + return spawnSync("bash", [script, ...files]).status ?? -1 + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +const summary = (pass: number, fail: number, extra = "") => + `(pass) a > b [1.00ms]\n\n ${pass} pass\n 3 skip\n ${fail} fail\n${extra}Ran ${pass + fail + 3} tests across 2 files. [1.00s]\n` + +describe("check-bun-test-summary.sh", () => { + it("passes a run with passing tests and no failures", () => { + expect(run(summary(12, 0))).toBe(0) + }) + + it("passes several clean runs together", () => { + expect(run(summary(12, 0), summary(3, 0))).toBe(0) + }) + + it("passes a run that crashed on exit after its summary (bun's cleanup segfault)", () => { + expect(run(summary(12, 0) + "Segmentation fault (core dumped)\n")).toBe(0) + }) + + it("fails a run with a failed test", () => { + expect(run(summary(11, 1))).toBe(1) + }) + + it("fails when any one of several runs failed", () => { + expect(run(summary(12, 0), summary(2, 1))).toBe(1) + }) + + it("fails a run with an error outside a test", () => { + expect(run(summary(12, 0, " 1 error\n"))).toBe(1) + expect(run(summary(12, 0, " 2 errors\n"))).toBe(1) + }) + + it("fails a run that printed no summary (bun crashed before finishing)", () => { + expect(run(summary(12, 0), "(pass) a > b [1.00ms]\nSegmentation fault\n")).toBe(1) + }) + + it("fails when nothing passed", () => { + expect(run(summary(0, 0))).toBe(1) + }) + + it("does not count test names that end in pass or fail", () => { + expect(run(`(pass) handles 3 fail\n${summary(4, 0)}`)).toBe(0) + }) + + it("rejects a call with no output files", () => { + expect(run()).toBe(2) + }) +}) diff --git a/script/github/check-bun-test-summary.sh b/script/github/check-bun-test-summary.sh new file mode 100755 index 0000000000..114b6fcc9a --- /dev/null +++ b/script/github/check-bun-test-summary.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Checks the saved output of one or more `bun test` runs and fails CI on a real test failure. +# +# Bun 1.3.x can segfault during process cleanup after every test passed (exit 143/SIGTERM or +# 134/SIGABRT), so the CI steps ignore bun's exit code and parse its summary lines instead. Each +# file must hold one run's output. Fails when any run reports failures or errors outside a test +# (bun's " N error(s)" line), when a run printed no summary (bun crashed before finishing), or when +# nothing passed at all. +# +# Usage: check-bun-test-summary.sh ... +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +sum_field() { awk -v f="$1" '$0 ~ ("^ *[0-9]+ " f "$"){s+=$1} END{print s+0}' "${@:2}"; } +PASS_COUNT=$(sum_field pass "$@") +FAIL_COUNT=$(sum_field fail "$@") +ERROR_COUNT=$(( $(sum_field error "$@") + $(sum_field errors "$@") )) + +echo "" +echo "--- Test Summary ($# runs) ---" +echo "pass=$PASS_COUNT fail=$FAIL_COUNT error=$ERROR_COUNT" + +if [ "$FAIL_COUNT" != "0" ]; then + echo "::error::$FAIL_COUNT test(s) failed" + exit 1 +fi + +if [ "$ERROR_COUNT" != "0" ]; then + echo "::error::$ERROR_COUNT error(s) outside a test" + exit 1 +fi + +for file in "$@"; do + if ! grep -qE "^ *[0-9]+ pass$" "$file"; then + echo "::error::Missing test summary in $file — Bun may have crashed before running tests" + exit 1 + fi +done + +if [ "$PASS_COUNT" -gt 0 ] 2>/dev/null; then + exit 0 +fi +echo "::error::No test results found" +exit 1