diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0909632066..79ad796498 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -721,6 +721,40 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - uses: taiki-e/install-action@nextest if: needs.changes.outputs.heavy == 'true' && (matrix.os != 'ubuntu-latest' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') + - name: Hold this machine's build lock (self-hosted) + # The self-hosted Mac is also a developer machine: local agents build + # through scripts/dev-cargo.sh, which holds this lock, and two Cargo + # builds at once exhaust its memory. Opt-in: the runner's `.env` names + # the same file as CODEWHALE_BUILD_LOCK_FILE. The holder is a + # background process; the runner kills orphans when the job ends, so a + # cancelled or crashed job cannot leave the lock held. + # (`env.*` in `if:` cannot see the runner's `.env`, so the opt-in is + # checked in the script.) + if: needs.changes.outputs.heavy == 'true' && runner.environment == 'self-hosted' + shell: bash + run: | + if [ -z "${CODEWHALE_BUILD_LOCK_FILE:-}" ]; then + echo "CODEWHALE_BUILD_LOCK_FILE is not set on this runner; building without the machine lock." + exit 0 + fi + hold="$RUNNER_TEMP/cw-build-lock.hold" + ready="$RUNNER_TEMP/cw-build-lock.ready" + log="$RUNNER_TEMP/cw-build-lock.log" + touch "$hold" + rm -f "$ready" + # The single quotes are deliberate: the inner sh expands $1/$2. + # shellcheck disable=SC2016 + nohup python3 scripts/build-lock.py "$CODEWHALE_BUILD_LOCK_FILE" -- \ + sh -c 'touch "$1"; while [ -e "$2" ]; do sleep 2; done' _ "$ready" "$hold" \ + >"$log" 2>&1 & + shown=0 + until [ -e "$ready" ]; do + if [ "$shown" -eq 0 ] && [ -s "$log" ]; then cat "$log"; shown=1; fi + sleep 2 + done + cat "$log" + # Scripts inside this job already run under the lock. + echo "CODEWHALE_BUILD_LOCK_HELD=1" >> "$GITHUB_ENV" - name: Run tests # Same test binaries as `cargo test`, run by cargo-nextest: one # process per test, all runner cores busy, slow tests named instead @@ -789,6 +823,10 @@ jobs: - name: Linux test location (CNB) if: needs.changes.outputs.heavy == 'true' && matrix.os == 'ubuntu-latest' && github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' run: echo "Linux workspace tests run on CNB for non-PR release/main pushes; pull requests run directly on Ubuntu." + - name: Release this machine's build lock (self-hosted) + if: always() && runner.environment == 'self-hosted' + shell: bash + run: rm -f "$RUNNER_TEMP/cw-build-lock.hold" macos-budget: # Fork PRs build cold on a GitHub-hosted Mac, and the RSS budget and the diff --git a/docs/BUILD_PERFORMANCE.md b/docs/BUILD_PERFORMANCE.md index 9092ffd2fd..74eec88823 100644 --- a/docs/BUILD_PERFORMANCE.md +++ b/docs/BUILD_PERFORMANCE.md @@ -349,6 +349,20 @@ TUI-DOG-017) — left as they are. 6. Then `fleet/`, `tools/`, `core/engine` — each behind the crate boundary its tests already respect, measured with the A0 table. +## One build per machine + +`scripts/dev-cargo.sh` and `scripts/dev-test.sh` hold an exclusive machine-wide +build lock (`/build.lock`, via `scripts/build-lock.py`) for the +whole Cargo invocation. Cargo's own lock is per target directory, so two +agents building into different target dirs still ran concurrently and exhausted +memory. A second build waits and prints who holds the lock. Set +`CODEWHALE_BUILD_LOCK=0` to skip it, or `CODEWHALE_BUILD_LOCK_FILE` to name the +lock file. A self-hosted CI runner on the same machine joins the lock when its +`.env` sets `CODEWHALE_BUILD_LOCK_FILE` to the same path: the macOS Test job +then holds it from the first test build to the end of the job. The lock is advisory: Cargo started +directly, outside these scripts, does not take it, and on platforms without +`fcntl` (Windows) the build runs unlocked after a warning. + ## What changed (this lane) 1. **`scripts/dev-cache.sh` / `scripts/dev-cargo.sh` activate the measured diff --git a/scripts/build-lock.py b/scripts/build-lock.py new file mode 100755 index 0000000000..c4d3e2711e --- /dev/null +++ b/scripts/build-lock.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Run one command while holding this machine's Codewhale build lock. + + scripts/build-lock.py -- [args...] + +Several agents share one checkout and one memory-constrained machine. Cargo's +own lock is per target directory, so builds into different target dirs still +run concurrently and exhaust memory. This holds an exclusive flock on a lock +file under the persistent cache root for the lifetime of the command, and +says who holds it while waiting. + +The command runs as a child, never via exec: a daemon it spawns (an sccache +server, say) must not inherit the descriptor and pin the lock after the build +exits. Nested invocations see CODEWHALE_BUILD_LOCK_HELD and run unlocked. + +Known limitation: advisory only. Cargo started outside scripts/dev-cargo.sh +does not take the lock, and on a platform without fcntl (Windows) the +command runs unlocked after a warning. +""" + +import os +import signal +import subprocess +import sys +import time + + +def main() -> int: + if len(sys.argv) < 4 or sys.argv[2] != "--": + print("usage: build-lock.py -- [args...]", file=sys.stderr) + return 2 + lock_path, command = sys.argv[1], sys.argv[3:] + env = dict(os.environ, CODEWHALE_BUILD_LOCK_HELD="1") + try: + import fcntl + except ImportError: + print("build-lock: fcntl unavailable; running without the machine build lock", file=sys.stderr) + return subprocess.call(command, env=env) + + os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True) + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + holder = os.pread(fd, 512, 0).decode(errors="replace").strip() or "holder unknown" + print( + f"build-lock: waiting for another Cargo build on this machine ({holder}); " + "set CODEWHALE_BUILD_LOCK=0 to skip", + file=sys.stderr, + flush=True, + ) + started = time.monotonic() + fcntl.flock(fd, fcntl.LOCK_EX) + print( + f"build-lock: acquired after {time.monotonic() - started:.0f}s", + file=sys.stderr, + flush=True, + ) + os.ftruncate(fd, 0) + summary = " ".join(command[:5]) + os.pwrite(fd, f"pid {os.getpid()} in {os.getcwd()}: {summary}\n".encode(), 0) + + child = subprocess.Popen(command, env=env) + for forwarded in (signal.SIGTERM, signal.SIGHUP): + signal.signal(forwarded, lambda signum, _frame: child.send_signal(signum)) + # SIGINT reaches the whole foreground process group already; only keep + # waiting so the lock outlives the child's own shutdown. + signal.signal(signal.SIGINT, signal.SIG_IGN) + status = child.wait() + return 128 - status if status < 0 else status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/dev-cache.sh b/scripts/dev-cache.sh index 23e9a1efac..364d8e4407 100755 --- a/scripts/dev-cache.sh +++ b/scripts/dev-cache.sh @@ -332,10 +332,19 @@ codewhale_dev_cache_exec_cargo() { case ${CODEWHALE_DEV_CACHE_MODE:-} in isolated-build-dir|force-isolated) if [ -n "${CARGO_BUILD_BUILD_DIR:-}" ]; then - exec cargo --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@" + set -- --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@" fi ;; esac + # One Cargo build per machine: separate target dirs do not make concurrent + # builds safe on a memory-constrained host (scripts/build-lock.py). + _cw_lock_script=${repo_root:-.}/scripts/build-lock.py + if ! codewhale_dev_cache_falsey "${CODEWHALE_BUILD_LOCK:-1}" \ + && [ -z "${CODEWHALE_BUILD_LOCK_HELD:-}" ] \ + && [ -f "$_cw_lock_script" ] \ + && command -v python3 >/dev/null 2>&1; then + exec python3 "$_cw_lock_script" "${CODEWHALE_BUILD_LOCK_FILE:-$(codewhale_dev_cache_root)/build.lock}" -- cargo "$@" + fi exec cargo "$@" }