From 39ee2fdfc2c4780610ebbc750c4ffac1152d6922 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 23 Sep 2026 08:51:00 -0700 Subject: [PATCH 1/3] build: hold one machine-wide lock per dev Cargo build Several agents share this checkout on a memory-constrained Mac, and Cargo's own lock is per target directory, so builds into different target dirs ran concurrently and needed manual coordination. scripts/dev-cargo.sh and scripts/dev-test.sh (through the shared codewhale_dev_cache_exec_cargo) now run Cargo under an exclusive flock on /build.lock. A waiting build prints the holder's pid, directory and command. Cargo runs as a child of the lock holder, never via exec, so a daemon it starts (an sccache server) cannot inherit the descriptor and pin the lock. Nested runs skip it through CODEWHALE_BUILD_LOCK_HELD; CODEWHALE_BUILD_LOCK=0 opts out. Advisory only; documented in docs/BUILD_PERFORMANCE.md. Evidence: two concurrent holders serialized (the second waited 2s, named the first, then ran and propagated exit status 7); a nested call skipped the lock; codewhale_dev_cache_exec_cargo --version ran under it and recorded the holder. scripts/dev-cache.test.sh: all 22 checks passed. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/BUILD_PERFORMANCE.md | 11 ++++++ scripts/build-lock.py | 74 +++++++++++++++++++++++++++++++++++++++ scripts/dev-cache.sh | 11 +++++- 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100755 scripts/build-lock.py diff --git a/docs/BUILD_PERFORMANCE.md b/docs/BUILD_PERFORMANCE.md index 9092ffd2fd..55fc3f4223 100644 --- a/docs/BUILD_PERFORMANCE.md +++ b/docs/BUILD_PERFORMANCE.md @@ -349,6 +349,17 @@ TUI-DOG-017) — left as they are. 6. Then `fleet/`, `tools/`, `core/engine` — each behind the crate boundary its tests already respect, measured with the A0 table. +## One build per machine + +`scripts/dev-cargo.sh` and `scripts/dev-test.sh` hold an exclusive machine-wide +build lock (`/build.lock`, via `scripts/build-lock.py`) for the +whole Cargo invocation. Cargo's own lock is per target directory, so two +agents building into different target dirs still ran concurrently and exhausted +memory. A second build waits and prints who holds the lock. Set +`CODEWHALE_BUILD_LOCK=0` to skip it. The lock is advisory: Cargo started +directly, outside these scripts, does not take it, and on platforms without +`fcntl` (Windows) the build runs unlocked after a warning. + ## What changed (this lane) 1. **`scripts/dev-cache.sh` / `scripts/dev-cargo.sh` activate the measured diff --git a/scripts/build-lock.py b/scripts/build-lock.py new file mode 100755 index 0000000000..c4d3e2711e --- /dev/null +++ b/scripts/build-lock.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Run one command while holding this machine's Codewhale build lock. + + scripts/build-lock.py -- [args...] + +Several agents share one checkout and one memory-constrained machine. Cargo's +own lock is per target directory, so builds into different target dirs still +run concurrently and exhaust memory. This holds an exclusive flock on a lock +file under the persistent cache root for the lifetime of the command, and +says who holds it while waiting. + +The command runs as a child, never via exec: a daemon it spawns (an sccache +server, say) must not inherit the descriptor and pin the lock after the build +exits. Nested invocations see CODEWHALE_BUILD_LOCK_HELD and run unlocked. + +Known limitation: advisory only. Cargo started outside scripts/dev-cargo.sh +does not take the lock, and on a platform without fcntl (Windows) the +command runs unlocked after a warning. +""" + +import os +import signal +import subprocess +import sys +import time + + +def main() -> int: + if len(sys.argv) < 4 or sys.argv[2] != "--": + print("usage: build-lock.py -- [args...]", file=sys.stderr) + return 2 + lock_path, command = sys.argv[1], sys.argv[3:] + env = dict(os.environ, CODEWHALE_BUILD_LOCK_HELD="1") + try: + import fcntl + except ImportError: + print("build-lock: fcntl unavailable; running without the machine build lock", file=sys.stderr) + return subprocess.call(command, env=env) + + os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True) + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + holder = os.pread(fd, 512, 0).decode(errors="replace").strip() or "holder unknown" + print( + f"build-lock: waiting for another Cargo build on this machine ({holder}); " + "set CODEWHALE_BUILD_LOCK=0 to skip", + file=sys.stderr, + flush=True, + ) + started = time.monotonic() + fcntl.flock(fd, fcntl.LOCK_EX) + print( + f"build-lock: acquired after {time.monotonic() - started:.0f}s", + file=sys.stderr, + flush=True, + ) + os.ftruncate(fd, 0) + summary = " ".join(command[:5]) + os.pwrite(fd, f"pid {os.getpid()} in {os.getcwd()}: {summary}\n".encode(), 0) + + child = subprocess.Popen(command, env=env) + for forwarded in (signal.SIGTERM, signal.SIGHUP): + signal.signal(forwarded, lambda signum, _frame: child.send_signal(signum)) + # SIGINT reaches the whole foreground process group already; only keep + # waiting so the lock outlives the child's own shutdown. + signal.signal(signal.SIGINT, signal.SIG_IGN) + status = child.wait() + return 128 - status if status < 0 else status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/dev-cache.sh b/scripts/dev-cache.sh index 23e9a1efac..1b56aee41a 100755 --- a/scripts/dev-cache.sh +++ b/scripts/dev-cache.sh @@ -332,10 +332,19 @@ codewhale_dev_cache_exec_cargo() { case ${CODEWHALE_DEV_CACHE_MODE:-} in isolated-build-dir|force-isolated) if [ -n "${CARGO_BUILD_BUILD_DIR:-}" ]; then - exec cargo --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@" + set -- --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@" fi ;; esac + # One Cargo build per machine: separate target dirs do not make concurrent + # builds safe on a memory-constrained host (scripts/build-lock.py). + _cw_lock_script=${repo_root:-.}/scripts/build-lock.py + if ! codewhale_dev_cache_falsey "${CODEWHALE_BUILD_LOCK:-1}" \ + && [ -z "${CODEWHALE_BUILD_LOCK_HELD:-}" ] \ + && [ -f "$_cw_lock_script" ] \ + && command -v python3 >/dev/null 2>&1; then + exec python3 "$_cw_lock_script" "$(codewhale_dev_cache_root)/build.lock" -- cargo "$@" + fi exec cargo "$@" } From e3f6be37cf1daec904edcf2b2c9f5e3221c8f4ea Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 23 Sep 2026 10:14:01 -0700 Subject: [PATCH 2/3] ci: the self-hosted Mac's Test job joins the machine build lock The self-hosted macOS runner is also the developer machine, and its trusted Test job runs Cargo directly, so it still built concurrently with local agents (observed: `cargo run -p codewhale-tui -- eval` from the runner beside local work). The Test job now holds the same machine lock from before the first test build to the end of the job, on the self-hosted runner only and only when the runner's `.env` sets CODEWHALE_BUILD_LOCK_FILE; otherwise it logs that it builds unlocked. The holder is a background process released by an always() step, and the runner kills orphans at job end, so a cancelled or crashed job cannot pin the lock. The job exports CODEWHALE_BUILD_LOCK_HELD so scripts inside it do not wait on their own lock. dev-cargo.sh accepts the same CODEWHALE_BUILD_LOCK_FILE override. Evidence: the step script extracted from ci.yml was run locally against a contended lock: it waited for a running holder (3 s), then a second build waited while it held the lock and ran once the hold file was removed; with the variable unset it exits 0 without locking. YAML parses; dev-cache.sh passes `sh -n`. Hosted acceptance needs a self-hosted run after the runner's .env is set and the runner restarted. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++++++ docs/BUILD_PERFORMANCE.md | 5 ++++- scripts/dev-cache.sh | 2 +- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70d9b2f044..9202d0660c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -719,6 +719,38 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - uses: taiki-e/install-action@nextest if: needs.changes.outputs.heavy == 'true' && (matrix.os != 'ubuntu-latest' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') + - name: Hold this machine's build lock (self-hosted) + # The self-hosted Mac is also a developer machine: local agents build + # through scripts/dev-cargo.sh, which holds this lock, and two Cargo + # builds at once exhaust its memory. Opt-in: the runner's `.env` names + # the same file as CODEWHALE_BUILD_LOCK_FILE. The holder is a + # background process; the runner kills orphans when the job ends, so a + # cancelled or crashed job cannot leave the lock held. + # (`env.*` in `if:` cannot see the runner's `.env`, so the opt-in is + # checked in the script.) + if: needs.changes.outputs.heavy == 'true' && runner.environment == 'self-hosted' + shell: bash + run: | + if [ -z "${CODEWHALE_BUILD_LOCK_FILE:-}" ]; then + echo "CODEWHALE_BUILD_LOCK_FILE is not set on this runner; building without the machine lock." + exit 0 + fi + hold="$RUNNER_TEMP/cw-build-lock.hold" + ready="$RUNNER_TEMP/cw-build-lock.ready" + log="$RUNNER_TEMP/cw-build-lock.log" + touch "$hold" + rm -f "$ready" + nohup python3 scripts/build-lock.py "$CODEWHALE_BUILD_LOCK_FILE" -- \ + sh -c 'touch "$1"; while [ -e "$2" ]; do sleep 2; done' _ "$ready" "$hold" \ + >"$log" 2>&1 & + shown=0 + until [ -e "$ready" ]; do + if [ "$shown" -eq 0 ] && [ -s "$log" ]; then cat "$log"; shown=1; fi + sleep 2 + done + cat "$log" + # Scripts inside this job already run under the lock. + echo "CODEWHALE_BUILD_LOCK_HELD=1" >> "$GITHUB_ENV" - name: Run tests # Same test binaries as `cargo test`, run by cargo-nextest: one # process per test, all runner cores busy, slow tests named instead @@ -786,6 +818,10 @@ jobs: - name: Linux test location (CNB) if: needs.changes.outputs.heavy == 'true' && matrix.os == 'ubuntu-latest' && github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' run: echo "Linux workspace tests run on CNB for non-PR release/main pushes; pull requests run directly on Ubuntu." + - name: Release this machine's build lock (self-hosted) + if: always() && runner.environment == 'self-hosted' + shell: bash + run: rm -f "$RUNNER_TEMP/cw-build-lock.hold" macos-budget: # Fork PRs build cold on a GitHub-hosted Mac, and the RSS budget and the diff --git a/docs/BUILD_PERFORMANCE.md b/docs/BUILD_PERFORMANCE.md index 55fc3f4223..74eec88823 100644 --- a/docs/BUILD_PERFORMANCE.md +++ b/docs/BUILD_PERFORMANCE.md @@ -356,7 +356,10 @@ build lock (`/build.lock`, via `scripts/build-lock.py`) for the whole Cargo invocation. Cargo's own lock is per target directory, so two agents building into different target dirs still ran concurrently and exhausted memory. A second build waits and prints who holds the lock. Set -`CODEWHALE_BUILD_LOCK=0` to skip it. The lock is advisory: Cargo started +`CODEWHALE_BUILD_LOCK=0` to skip it, or `CODEWHALE_BUILD_LOCK_FILE` to name the +lock file. A self-hosted CI runner on the same machine joins the lock when its +`.env` sets `CODEWHALE_BUILD_LOCK_FILE` to the same path: the macOS Test job +then holds it from the first test build to the end of the job. The lock is advisory: Cargo started directly, outside these scripts, does not take it, and on platforms without `fcntl` (Windows) the build runs unlocked after a warning. diff --git a/scripts/dev-cache.sh b/scripts/dev-cache.sh index 1b56aee41a..364d8e4407 100755 --- a/scripts/dev-cache.sh +++ b/scripts/dev-cache.sh @@ -343,7 +343,7 @@ codewhale_dev_cache_exec_cargo() { && [ -z "${CODEWHALE_BUILD_LOCK_HELD:-}" ] \ && [ -f "$_cw_lock_script" ] \ && command -v python3 >/dev/null 2>&1; then - exec python3 "$_cw_lock_script" "$(codewhale_dev_cache_root)/build.lock" -- cargo "$@" + exec python3 "$_cw_lock_script" "${CODEWHALE_BUILD_LOCK_FILE:-$(codewhale_dev_cache_root)/build.lock}" -- cargo "$@" fi exec cargo "$@" } From b20c8eea8adc89ecb84f7bf95413c429508bea5b Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 23 Sep 2026 12:30:44 -0700 Subject: [PATCH 3/3] ci: mark the build-lock holder's single quotes as intended actionlint failed on #6440 with SC2016: the holder's `sh -c 'touch "$1"; while [ -e "$2" ]...'` script is single-quoted on purpose, so the inner shell expands its own positional arguments. A shellcheck directive above the command says so. Evidence: `actionlint -ignore SC2129 -ignore SC2221 -ignore SC2222 .github/workflows/ci.yml` (the CI job's flags) passes clean locally. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9202d0660c..7fb37c85d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -740,6 +740,8 @@ jobs: log="$RUNNER_TEMP/cw-build-lock.log" touch "$hold" rm -f "$ready" + # The single quotes are deliberate: the inner sh expands $1/$2. + # shellcheck disable=SC2016 nohup python3 scripts/build-lock.py "$CODEWHALE_BUILD_LOCK_FILE" -- \ sh -c 'touch "$1"; while [ -e "$2" ]; do sleep 2; done' _ "$ready" "$hold" \ >"$log" 2>&1 &