build: hold one machine-wide lock per dev Cargo build

.github/workflows/ci.yml

@@ -721,6 +721,40 @@ jobs:
           save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: taiki-e/install-action@nextest
         if: needs.changes.outputs.heavy == 'true' && (matrix.os != 'ubuntu-latest' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request')
+      - name: Hold this machine's build lock (self-hosted)
+        # The self-hosted Mac is also a developer machine: local agents build
+        # through scripts/dev-cargo.sh, which holds this lock, and two Cargo
+        # builds at once exhaust its memory. Opt-in: the runner's `.env` names
+        # the same file as CODEWHALE_BUILD_LOCK_FILE. The holder is a
+        # background process; the runner kills orphans when the job ends, so a
+        # cancelled or crashed job cannot leave the lock held.
+        # (`env.*` in `if:` cannot see the runner's `.env`, so the opt-in is
+        # checked in the script.)
+        if: needs.changes.outputs.heavy == 'true' && runner.environment == 'self-hosted'
+        shell: bash
+        run: |
+          if [ -z "${CODEWHALE_BUILD_LOCK_FILE:-}" ]; then
+            echo "CODEWHALE_BUILD_LOCK_FILE is not set on this runner; building without the machine lock."
+            exit 0
+          fi
+          hold="$RUNNER_TEMP/cw-build-lock.hold"
+          ready="$RUNNER_TEMP/cw-build-lock.ready"
+          log="$RUNNER_TEMP/cw-build-lock.log"
+          touch "$hold"
+          rm -f "$ready"
+          # The single quotes are deliberate: the inner sh expands $1/$2.
+          # shellcheck disable=SC2016
+          nohup python3 scripts/build-lock.py "$CODEWHALE_BUILD_LOCK_FILE" -- \
+            sh -c 'touch "$1"; while [ -e "$2" ]; do sleep 2; done' _ "$ready" "$hold" \
+            >"$log" 2>&1 &
+          shown=0
+          until [ -e "$ready" ]; do
+            if [ "$shown" -eq 0 ] && [ -s "$log" ]; then cat "$log"; shown=1; fi
+            sleep 2
+          done
+          cat "$log"
+          # Scripts inside this job already run under the lock.
+          echo "CODEWHALE_BUILD_LOCK_HELD=1" >> "$GITHUB_ENV"
       - name: Run tests
         # Same test binaries as `cargo test`, run by cargo-nextest: one
         # process per test, all runner cores busy, slow tests named instead
@@ -789,6 +823,10 @@ jobs:
       - name: Linux test location (CNB)
         if: needs.changes.outputs.heavy == 'true' && matrix.os == 'ubuntu-latest' && github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request'
         run: echo "Linux workspace tests run on CNB for non-PR release/main pushes; pull requests run directly on Ubuntu."
+      - name: Release this machine's build lock (self-hosted)
+        if: always() && runner.environment == 'self-hosted'
+        shell: bash
+        run: rm -f "$RUNNER_TEMP/cw-build-lock.hold"
 
   macos-budget:
     # Fork PRs build cold on a GitHub-hosted Mac, and the RSS budget and the

docs/BUILD_PERFORMANCE.md

@@ -349,6 +349,20 @@ TUI-DOG-017) — left as they are.
 6. Then `fleet/`, `tools/`, `core/engine` — each behind the crate boundary
    its tests already respect, measured with the A0 table.
 
+## One build per machine
+
+`scripts/dev-cargo.sh` and `scripts/dev-test.sh` hold an exclusive machine-wide
+build lock (`<cache root>/build.lock`, via `scripts/build-lock.py`) for the
+whole Cargo invocation. Cargo's own lock is per target directory, so two
+agents building into different target dirs still ran concurrently and exhausted
+memory. A second build waits and prints who holds the lock. Set
+`CODEWHALE_BUILD_LOCK=0` to skip it, or `CODEWHALE_BUILD_LOCK_FILE` to name the
+lock file. A self-hosted CI runner on the same machine joins the lock when its
+`.env` sets `CODEWHALE_BUILD_LOCK_FILE` to the same path: the macOS Test job
+then holds it from the first test build to the end of the job. The lock is advisory: Cargo started
+directly, outside these scripts, does not take it, and on platforms without
+`fcntl` (Windows) the build runs unlocked after a warning.
+
 ## What changed (this lane)
 
 1. **`scripts/dev-cache.sh` / `scripts/dev-cargo.sh` activate the measured

scripts/build-lock.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Run one command while holding this machine's Codewhale build lock.
+
+    scripts/build-lock.py <lock-file> -- <command> [args...]
+
+Several agents share one checkout and one memory-constrained machine. Cargo's
+own lock is per target directory, so builds into different target dirs still
+run concurrently and exhaust memory. This holds an exclusive flock on a lock
+file under the persistent cache root for the lifetime of the command, and
+says who holds it while waiting.
+
+The command runs as a child, never via exec: a daemon it spawns (an sccache
+server, say) must not inherit the descriptor and pin the lock after the build
+exits. Nested invocations see CODEWHALE_BUILD_LOCK_HELD and run unlocked.
+
+Known limitation: advisory only. Cargo started outside scripts/dev-cargo.sh
+does not take the lock, and on a platform without fcntl (Windows) the
+command runs unlocked after a warning.
+"""
+
+import os
+import signal
+import subprocess
+import sys
+import time
+
+
+def main() -> int:
+    if len(sys.argv) < 4 or sys.argv[2] != "--":
+        print("usage: build-lock.py <lock-file> -- <command> [args...]", file=sys.stderr)
+        return 2
+    lock_path, command = sys.argv[1], sys.argv[3:]
+    env = dict(os.environ, CODEWHALE_BUILD_LOCK_HELD="1")
+    try:
+        import fcntl
+    except ImportError:
+        print("build-lock: fcntl unavailable; running without the machine build lock", file=sys.stderr)
+        return subprocess.call(command, env=env)
+
+    os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True)
+    fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        holder = os.pread(fd, 512, 0).decode(errors="replace").strip() or "holder unknown"
+        print(
+            f"build-lock: waiting for another Cargo build on this machine ({holder}); "
+            "set CODEWHALE_BUILD_LOCK=0 to skip",
+            file=sys.stderr,
+            flush=True,
+        )
+        started = time.monotonic()
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        print(
+            f"build-lock: acquired after {time.monotonic() - started:.0f}s",
+            file=sys.stderr,
+            flush=True,
+        )
+    os.ftruncate(fd, 0)
+    summary = " ".join(command[:5])
+    os.pwrite(fd, f"pid {os.getpid()} in {os.getcwd()}: {summary}\n".encode(), 0)
+
+    child = subprocess.Popen(command, env=env)
+    for forwarded in (signal.SIGTERM, signal.SIGHUP):
+        signal.signal(forwarded, lambda signum, _frame: child.send_signal(signum))
+    # SIGINT reaches the whole foreground process group already; only keep
+    # waiting so the lock outlives the child's own shutdown.
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    status = child.wait()
+    return 128 - status if status < 0 else status
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/dev-cache.sh

@@ -332,10 +332,19 @@ codewhale_dev_cache_exec_cargo() {
   case ${CODEWHALE_DEV_CACHE_MODE:-} in
     isolated-build-dir|force-isolated)
       if [ -n "${CARGO_BUILD_BUILD_DIR:-}" ]; then
-        exec cargo --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@"
+        set -- --config "build.build-dir = \"${CARGO_BUILD_BUILD_DIR}\"" "$@"
       fi
       ;;
   esac
+  # One Cargo build per machine: separate target dirs do not make concurrent
+  # builds safe on a memory-constrained host (scripts/build-lock.py).
+  _cw_lock_script=${repo_root:-.}/scripts/build-lock.py
+  if ! codewhale_dev_cache_falsey "${CODEWHALE_BUILD_LOCK:-1}" \
+    && [ -z "${CODEWHALE_BUILD_LOCK_HELD:-}" ] \
+    && [ -f "$_cw_lock_script" ] \
+    && command -v python3 >/dev/null 2>&1; then
+    exec python3 "$_cw_lock_script" "${CODEWHALE_BUILD_LOCK_FILE:-$(codewhale_dev_cache_root)/build.lock}" -- cargo "$@"
+  fi
   exec cargo "$@"
 }