From 07f7f3a8a6d492c6109c8b5dcebaf5402c2f0e83 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 15 Sep 2026 17:09:46 +0200 Subject: [PATCH 1/2] fix(cluster): pin and cache the bootstrap toolchain outside the dir the wipe deletes (#392) #388 cached the actions-runner tarball, but the same playbook fetched four more tools from the internet on every run into the directory it wipes: uv, a python build, ansible-core and ansible.posix. Each sat behind a skip-if-present guard that could therefore never skip, none retried, and ansible-core and ansible.posix were unpinned. On 2026-09-15 a Galaxy timeout on one host (API time-to-first-byte 23s while GitHub on the same host did 48 MB/s) failed nightly 34981852841 although the other two hosts had already registered their runners. Pinning comes first. Caching an unpinned install silently freezes whichever version installed first, with nothing recording which. uv 0.12.15, python 3.13, ansible-core 2.21.4 and ansible.posix 2.2.2 are pinned, and each cache directory is keyed by its pin so a bump installs fresh. All four were field-proven by the 14:29Z bootstrap on all three hosts. Six cluster workflows reach the venv and the collections through paths they hardcode under runner_root, not through anything this playbook exports, so moving the tools would have broken every tier. The two legacy paths stay alive as symlinks into the cache. Teardown's wipe is ansible.builtin.file state=absent, which for a directory is shutil.rmtree; that unlinks a symlink and never follows it -- measured on Linux CPython 3.13 -- so the wipe drops the links and keeps the cache. uv venv now runs with --managed-python. Without it uv prefers any matching python already on PATH over downloading into UV_PYTHON_INSTALL_DIR; an end-to-end run caught the "cached" venv linked to the test controller's python, and a fresh container could not execute it. The cluster only escaped because its system python is 3.14. UV_PYTHON_INSTALL_DIR and UV_CACHE_DIR also move uv's python and cache under /tmp, which the playbook's own pristine rule always required and uv's defaults under $HOME quietly broke. ansible.posix is trusted from the cache only if its MANIFEST.json names the pinned version; otherwise it is fetched from Galaxy, and if Galaxy will not serve it, built from the collection's GitHub source tag. It publishes no release assets, so its git source is the only non-Galaxy route, and git is present on every host. TestBootstrapToolchainIsPinnedAndCachedOutsideTheWipedDir asserts the pins, the cache locations, the manifest check, --managed-python, and the six-workflow symlink contract from both ends. --- ansible/runner-setup.yml | 193 +++++++++++++++++++++----- workflow_runner_tarball_cache_test.go | 119 ++++++++++++++++ 2 files changed, 280 insertions(+), 32 deletions(-) diff --git a/ansible/runner-setup.yml b/ansible/runner-setup.yml index c49275a4..19aad1b1 100644 --- a/ansible/runner-setup.yml +++ b/ansible/runner-setup.yml @@ -48,6 +48,26 @@ # runner_root on a host called "cache", and the wipe would take # the cache with it. runner_cache_dir: "/tmp/celeris-runner-tarballs" + # probatorium#392: every tool the bootstrap fetches is pinned and cached + # outside runner_root, keyed by its pin. Pinning has to come first -- + # caching an unpinned install silently freezes whichever version happened + # to install first, with nothing recording which -- and keying each cache + # directory by its pin is what makes a bump install fresh instead of + # reusing a stale copy. All four pins were field-proven by the + # 2026-09-15 14:29Z bootstrap on all three hosts. + uv_version: "0.12.15" + python_version: "3.13" + ansible_core_version: "2.21.4" + ansible_posix_version: "2.2.2" + tool_cache_dir: "{{ runner_cache_dir }}/tools" + uv_dir: "{{ tool_cache_dir }}/uv-{{ uv_version }}" + # uv otherwise puts its python under $HOME/.local/share/uv/python and its + # cache under $HOME/.cache/uv -- outside /tmp, which quietly broke this + # playbook's own "nothing lands outside /tmp" rule. Both live here now. + uv_python_install_dir: "{{ tool_cache_dir }}/uv-python" + uv_cache_dir: "{{ tool_cache_dir }}/uv-cache" + ansible_venv_dir: "{{ tool_cache_dir }}/ansible-venv-py{{ python_version }}-core{{ ansible_core_version }}" + ansible_collections_dir: "{{ tool_cache_dir }}/ansible-collections-posix{{ ansible_posix_version }}" runner_arch_map: x86_64: x64 aarch64: arm64 @@ -193,65 +213,174 @@ # as a secret regardless. no_log: true - - name: Install uv (static binary, self-contained) + - name: Ensure tool cache dir + ansible.builtin.file: + path: "{{ tool_cache_dir }}" + state: directory + mode: '0755' + + - name: Install uv (pinned, cached across runs) # mage Deploy / Validate / Soak shell out to ansible-playbook. # The cluster hosts don't have ansible system-installed (the # pristine rule forbids). Plain `python3 -m venv` fails on the # cluster because Ubuntu 26.04 ships python3.14 without # python3.14-venv / ensurepip, and there's no system pip either. # - # Use uv instead — single static binary that bootstraps its own + # Use uv instead -- single static binary that bootstraps its own # python + venv without depending on system python-venv. The # installer drops uv + uvx directly into UV_INSTALL_DIR (NOT a # bin/ subdir), so the binary lives at /uv. # - # Lives inside runner_root so runner-teardown.yml wipes it - # along with the rest of the runner dir. + # The versioned installer URL really pins: its script sets + # APP_VERSION to that release. A cached uv is only trusted if it + # reports the pinned version, so a partial install re-fetches + # instead of being skipped as present. ansible.builtin.shell: | set -e - if [ -x "{{ runner_root }}/uv/uv" ]; then - echo "uv already present" + if [ -x "{{ uv_dir }}/uv" ] && "{{ uv_dir }}/uv" --version | grep -qF "uv {{ uv_version }}"; then + echo "uv {{ uv_version }} already cached" exit 0 fi - curl -LsSf https://astral.sh/uv/install.sh \ - | env UV_INSTALL_DIR="{{ runner_root }}/uv" \ + rm -rf "{{ uv_dir }}" + curl -LsSf "https://astral.sh/uv/{{ uv_version }}/install.sh" \ + | env UV_INSTALL_DIR="{{ uv_dir }}" \ UV_NO_MODIFY_PATH=1 \ INSTALLER_NO_MODIFY_PATH=1 \ sh + "{{ uv_dir }}/uv" --version | grep -qF "uv {{ uv_version }}" args: executable: /bin/bash - changed_when: true + register: uv_install + changed_when: "'already cached' not in uv_install.stdout" + retries: 5 + delay: 10 + until: uv_install is succeeded - - name: Create ansible venv via uv (auto-downloads python if needed) + - name: Create ansible venv via uv (pinned python, cached across runs) # uv venv brings its own python distribution if the system lacks - # one with ensurepip — exactly the cluster's situation. + # one with ensurepip -- exactly the cluster's situation. --clear + # only matters when the guard misses, i.e. a venv at this path + # lost its python: rebuild it rather than trip over the leftovers. + # + # --managed-python is what keeps that python inside the cache. + # Without it uv prefers any matching python already on PATH over + # downloading into UV_PYTHON_INSTALL_DIR, and the "cached" venv + # then depends on a python that lives somewhere else entirely. An + # end-to-end run caught exactly that: the venv linked to the test + # controller's python, and a fresh container could not execute it. + # The cluster only escapes today because its system python is 3.14. ansible.builtin.command: cmd: >- - {{ runner_root }}/uv/uv venv - --python 3.13 - {{ runner_root }}/ansible-venv - creates: "{{ runner_root }}/ansible-venv/bin/python" + {{ uv_dir }}/uv venv --clear --managed-python + --python {{ python_version }} + {{ ansible_venv_dir }} + creates: "{{ ansible_venv_dir }}/bin/python" + environment: + UV_PYTHON_INSTALL_DIR: "{{ uv_python_install_dir }}" + UV_CACHE_DIR: "{{ uv_cache_dir }}" + register: ansible_venv_create + retries: 5 + delay: 10 + until: ansible_venv_create is succeeded - - name: Install ansible-core into the venv via uv pip + - name: Install ansible-core into the venv via uv pip (pinned, cached across runs) ansible.builtin.command: cmd: >- - {{ runner_root }}/uv/uv pip install - --python {{ runner_root }}/ansible-venv/bin/python - ansible-core - creates: "{{ runner_root }}/ansible-venv/bin/ansible-playbook" + {{ uv_dir }}/uv pip install + --python {{ ansible_venv_dir }}/bin/python + ansible-core=={{ ansible_core_version }} + creates: "{{ ansible_venv_dir }}/bin/ansible-playbook" + environment: + UV_PYTHON_INSTALL_DIR: "{{ uv_python_install_dir }}" + UV_CACHE_DIR: "{{ uv_cache_dir }}" + register: ansible_core_install + retries: 5 + delay: 10 + until: ansible_core_install is succeeded - - name: Install ansible.posix collection - # cleanup.yml + deploy.yml reference ansible.posix.sysctl / - # ansible.posix.synchronize. ansible-core ships without bundled - # collections, so install into a runner-scoped collections dir - # (kept under runner_root for teardown wipe). ANSIBLE_COLLECTIONS_PATH - # gets exported in the Start runner task so the playbooks find it. - ansible.builtin.command: - cmd: >- - {{ runner_root }}/ansible-venv/bin/ansible-galaxy collection install - --collections-path {{ runner_root }}/ansible-collections - ansible.posix - creates: "{{ runner_root }}/ansible-collections/ansible_collections/ansible/posix" + # cleanup.yml + deploy.yml reference ansible.posix.sysctl / + # ansible.posix.synchronize, and ansible-core ships without bundled + # collections. On 2026-09-15 galaxy.ansible.com gave one host nothing + # for 2m29s (API time-to-first-byte 23s, while GitHub on the same host + # did 48 MB/s) and that one host failed the whole bootstrap. So trust + # the cache only if its own manifest names the pinned version; + # otherwise fetch from Galaxy, and if Galaxy will not serve it, build it + # from the collection's GitHub source tag. ansible.posix publishes no + # release assets on GitHub, so its git source is the only non-Galaxy + # route; git is present on every cluster host. + - name: Install ansible.posix (pinned; Galaxy, then GitHub source; cached across runs) + block: + - name: Read the cached ansible.posix manifest + ansible.builtin.slurp: + src: "{{ ansible_collections_dir }}/ansible_collections/ansible/posix/MANIFEST.json" + register: ansible_posix_manifest + + - name: Check the cached ansible.posix is the pinned version + ansible.builtin.assert: + that: + - (ansible_posix_manifest.content | b64decode | from_json).collection_info.version == ansible_posix_version + quiet: true + rescue: + - name: Discard any partial or mismatched ansible.posix + ansible.builtin.file: + path: "{{ ansible_collections_dir }}" + state: absent + + - name: Fetch ansible.posix, from Galaxy or its GitHub source + block: + - name: Install ansible.posix from Galaxy + ansible.builtin.command: + cmd: >- + {{ ansible_venv_dir }}/bin/ansible-galaxy collection install + --collections-path {{ ansible_collections_dir }} + ansible.posix:=={{ ansible_posix_version }} + register: ansible_posix_galaxy + retries: 3 + delay: 15 + until: ansible_posix_galaxy is succeeded + rescue: + - name: Install ansible.posix from its GitHub source tag + ansible.builtin.command: + cmd: >- + {{ ansible_venv_dir }}/bin/ansible-galaxy collection install --force + --collections-path {{ ansible_collections_dir }} + git+https://github.com/ansible-collections/ansible.posix.git,{{ ansible_posix_version }} + register: ansible_posix_git + retries: 3 + delay: 15 + until: ansible_posix_git is succeeded + + - name: Read the installed ansible.posix manifest + ansible.builtin.slurp: + src: "{{ ansible_collections_dir }}/ansible_collections/ansible/posix/MANIFEST.json" + register: ansible_posix_manifest + + - name: Check the installed ansible.posix is the pinned version + ansible.builtin.assert: + that: + - (ansible_posix_manifest.content | b64decode | from_json).collection_info.version == ansible_posix_version + fail_msg: "ansible.posix installed but its MANIFEST.json does not name {{ ansible_posix_version }}" + + - name: Point the legacy runner_root tool paths at the cache + # Six cluster workflows (nightly, weekend, benchmark, checkptr, + # race, pr-tier) never read the exports in "Start runner": each + # re-derives RUNNER_ROOT=/tmp/actions-runner-$(hostname -s) and puts + # ${RUNNER_ROOT}/ansible-venv/bin on GITHUB_PATH and + # ${RUNNER_ROOT}/ansible-collections into ANSIBLE_COLLECTIONS_PATH. + # Keeping those two paths alive as symlinks into the cache leaves + # all six untouched. Teardown's "Wipe runner dir" is + # ansible.builtin.file state=absent, which for a directory is + # shutil.rmtree -- that unlinks a symlink and never follows it + # (measured on Linux CPython 3.13: avoids_symlink_attacks=True and + # the target survives), so the wipe drops the links, not the cache. + ansible.builtin.file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + force: true + loop: + - { src: "{{ ansible_venv_dir }}", dest: "{{ runner_root }}/ansible-venv" } + - { src: "{{ ansible_collections_dir }}", dest: "{{ runner_root }}/ansible-collections" } - name: Start runner (nohup) # Prepend the runner-scoped ansible venv to PATH so any job diff --git a/workflow_runner_tarball_cache_test.go b/workflow_runner_tarball_cache_test.go index a20a40f4..cf376e9b 100644 --- a/workflow_runner_tarball_cache_test.go +++ b/workflow_runner_tarball_cache_test.go @@ -137,3 +137,122 @@ func readPlaybook(t *testing.T, name string) string { } return string(b) } + +// probatorium#392: #388 cached the runner tarball, but the same playbook +// fetched four more tools on every run into the directory it wipes -- uv, a +// python build, ansible-core and ansible.posix -- behind skip-if-present +// guards that therefore never skipped, with no retries, and with ansible-core +// and ansible.posix unpinned. A Galaxy timeout on one host failed nightly +// 34981852841 although the other two hosts had already registered. +// +// Pinning comes first: caching an unpinned install silently freezes whichever +// version happened to install first, with nothing recording which. Keying +// each cache directory by its pin is what makes a bump install fresh. +// +// Six cluster workflows reach the venv and the collections through hardcoded +// runner_root paths, not through anything the playbook exports, so the +// playbook keeps those two paths alive as symlinks into the cache. A path +// written in one file and consumed in six breaks silently, so the contract is +// asserted here from both ends. +func TestBootstrapToolchainIsPinnedAndCachedOutsideTheWipedDir(t *testing.T) { + setup := readPlaybook(t, "runner-setup.yml") + + for _, p := range []string{"uv_version", "python_version", "ansible_core_version", "ansible_posix_version"} { + re := regexp.MustCompile(`(?m)^\s*` + p + `:\s*"\d+\.\d+(\.\d+)?"\s*$`) + if !re.MatchString(setup) { + t.Errorf("%s is not pinned to a concrete version: an unpinned tool that is also "+ + "cached freezes whichever version installed first, with nothing recording which", p) + } + } + + for what, want := range map[string]string{ + "uv installer": "astral.sh/uv/{{ uv_version }}/install.sh", + "venv python": "--python {{ python_version }}", + "ansible-core": "ansible-core=={{ ansible_core_version }}", + "ansible.posix from Galaxy": "ansible.posix:=={{ ansible_posix_version }}", + "ansible.posix from GitHub": "ansible.posix.git,{{ ansible_posix_version }}", + } { + if !strings.Contains(setup, want) { + t.Errorf("%s does not install its pin (want %q in runner-setup.yml)", what, want) + } + } + + // No tool may keep its install target or its skip guard under runner_root. + for _, f := range []string{ + "{{ runner_root }}/uv/", + "--collections-path {{ runner_root }}", + `creates: "{{ runner_root }}/ansible-venv`, + `creates: "{{ runner_root }}/ansible-collections`, + } { + if strings.Contains(setup, f) { + t.Errorf("runner-setup.yml still has %q: that target is inside the directory the "+ + "bootstrap wipes, so its skip guard can never skip and the fetch repeats every run", f) + } + } + + for _, want := range []string{ + `tool_cache_dir: "{{ runner_cache_dir }}/`, + `uv_dir: "{{ tool_cache_dir }}/`, + `ansible_venv_dir: "{{ tool_cache_dir }}/`, + `ansible_collections_dir: "{{ tool_cache_dir }}/`, + } { + if !strings.Contains(setup, want) { + t.Errorf("want %q: every cached tool must live under runner_cache_dir, which "+ + "TestRunnerTarballIsCachedOutsideTheWipedDir already keeps teardown away from", want) + } + } + + // Without --managed-python uv prefers a matching python already on PATH + // over downloading into UV_PYTHON_INSTALL_DIR, so the cached venv would + // depend on a python outside the cache. An end-to-end run caught it. + if !strings.Contains(setup, "uv venv --clear --managed-python") { + t.Error("the ansible venv is not created with --managed-python: uv may build it on a " + + "python found on PATH, outside the cache, and the cached venv breaks when that python goes") + } + + // A cache that skips on presence alone turns one bad install into every + // later run's problem, so ansible.posix is checked against its manifest. + if !strings.Contains(setup, "MANIFEST.json") { + t.Error("ansible.posix has no integrity check against its MANIFEST.json: a partial " + + "or mismatched cached collection would be skipped as present, forever") + } + + // The six-workflow contract, from both ends. + links := map[string]string{ + "ansible-venv": `src: "{{ ansible_venv_dir }}", dest: "{{ runner_root }}/ansible-venv"`, + "ansible-collections": `src: "{{ ansible_collections_dir }}", dest: "{{ runner_root }}/ansible-collections"`, + } + consumers := map[string]int{} + entries, err := os.ReadDir(filepath.Join(".github", "workflows")) + if err != nil { + t.Fatalf("read workflows: %v", err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + b, rerr := os.ReadFile(filepath.Join(".github", "workflows", e.Name())) + if rerr != nil { + t.Errorf("%s: %v", e.Name(), rerr) + continue + } + for path := range links { + if strings.Contains(string(b), "${RUNNER_ROOT}/"+path) { + consumers[path]++ + } + } + } + for path, want := range links { + if consumers[path] == 0 { + t.Errorf("no workflow reads ${RUNNER_ROOT}/%s any more: this contract check is "+ + "guarding nothing, so update or delete it", path) + continue + } + if !strings.Contains(setup, want) || !strings.Contains(setup, "state: link") { + t.Errorf("%d workflow(s) read ${RUNNER_ROOT}/%s but runner-setup.yml does not link it "+ + "into the cache (want %q): every one of those tiers loses its ansible", consumers[path], path, want) + } + } + t.Logf("toolchain contract: %d workflow(s) read ansible-venv, %d read ansible-collections", + consumers["ansible-venv"], consumers["ansible-collections"]) +} From 2f8f518b57aad4901d71ed84f84710f76c198efa Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 15 Sep 2026 18:03:17 +0200 Subject: [PATCH 2/2] fix(cluster): trust the cached toolchain only when verified, never on presence Review of this branch found that the first cut still trusted three cached states on presence alone, which is the exact failure the change exists to remove, and each of them survives the wipe now that the cache outlives the run that left it: - Ansible's `creates:` checks with glob.glob, which counts a DANGLING symlink as present. A venv whose python was deleted was skipped and failed later inside a job instead of being rebuilt. The self-heal the comment promised never ran. - uv writes a wheel's entry-point scripts before its package data, metadata and RECORD, so bin/ansible-playbook can exist over an install the bootstrap timeout killed half way. - ansible-galaxy writes MANIFEST.json before any other collection file, so a manifest naming the pinned version proves nothing about the files after it. The venv and ansible-core are now one cached unit, trusted only when a completion stamp written last exists AND ansible-playbook actually runs and reports the pinned core; a miss rebuilds it from scratch. ansible.posix is trusted only with a completion stamp AND `ansible-galaxy collection verify --offline`, which checks every file against its recorded hash, AND a MANIFEST.json version equal to the pin: verify does not hash MANIFEST.json itself, and the end-to-end run caught a cached manifest altered to 0.0.0 passing it. ansible-core's dependencies were unpinned and would have been frozen by the cache at whatever resolved first. They are now resolved with `--exclude-newer 2026-09-15T00:00:00Z`, and the cutoff is part of the venv's directory name, so moving it builds a fresh venv. Each Galaxy attempt is bounded by coreutils timeout with one retry. retries alone never bound a stalled transfer: at the 2m29s stall that failed nightly 34981852841, four unbounded attempts would have used ~11 minutes of a 25-minute bootstrap before the fallback ran. The fallback has its own timeout and installs commit e98d9a07 rather than the 2.2.2 tag, which upstream could move. Smaller fixes from the same review: ANSIBLE_HOME keeps ansible-galaxy's temp dirs and API cache inside the tool cache, so "nothing lands outside /tmp" is true for it too; the cache directories carry an explicit owner; the link task drops force, which switched off ansible's only check that a link target exists, and a stat now fails the play if a target is missing; changed_when no longer masks a module failure. RUNNER_BOOTSTRAP.md documents the persistent tool cache instead of claiming nothing persists, and the teardown check is renamed to say it confirms the runner dir only. TestBootstrapToolchainCacheIsTrustedOnlyWhenVerified and TestCachedAnsiblePosixCheckComparesTheManifestVersion guard all of it. Its first version failed on the good tree: a substring check for "force: true" matched the comment explaining why there is no force. It now matches force as a YAML key in any truthy spelling. --- .../actions/cluster-runner-down/action.yml | 6 +- ansible/RUNNER_BOOTSTRAP.md | 48 ++++- ansible/runner-setup.yml | 203 ++++++++++++------ ansible/runner-teardown.yml | 5 +- workflow_runner_tarball_cache_test.go | 102 ++++++++- 5 files changed, 295 insertions(+), 69 deletions(-) diff --git a/.github/actions/cluster-runner-down/action.yml b/.github/actions/cluster-runner-down/action.yml index a0ff81c9..738a62b7 100644 --- a/.github/actions/cluster-runner-down/action.yml +++ b/.github/actions/cluster-runner-down/action.yml @@ -94,8 +94,10 @@ runs: sudo apt-get install -y --no-install-recommends ansible-core # cleanup.yml (driven below) uses ansible.posix.sysctl, which ansible-core # does NOT bundle. Provision it explicitly so the off-cluster reclamation - # is deterministic regardless of what the runner image happens to ship — - # mirrors runner-setup.yml's provisioning on the self-hosted runners. + # is deterministic regardless of what the runner image happens to ship. + # This is NOT the self-hosted runners' provisioning: runner-setup.yml pins + # ansible.posix, caches it and falls back to its GitHub source. This is a + # plain install on the GitHub-hosted image, which carries ansible.posix. # Installs to ~/.ansible/collections, already on ansible's default path. ansible-galaxy collection install ansible.posix diff --git a/ansible/RUNNER_BOOTSTRAP.md b/ansible/RUNNER_BOOTSTRAP.md index 9b530d26..9b9a2e25 100644 --- a/ansible/RUNNER_BOOTSTRAP.md +++ b/ansible/RUNNER_BOOTSTRAP.md @@ -3,9 +3,11 @@ The matrix-tier workflows (`matrix-{pr,nightly,weekend}-tier.yml`) provision **ephemeral** GitHub Actions self-hosted runners on the three cluster hosts at the start of every run and tear them down -when the run finishes — even on cancel/failure. Nothing persists on -the cluster between tier runs; the pristine rule (no Go install, no -runner daemon, no `~/actions-runner` dir) is preserved. +when the run finishes — even on cancel/failure. The runner itself does +not persist between tier runs, and the pristine rule (no Go install, no +runner daemon, no `~/actions-runner` dir) is preserved. One thing does +persist by design: the version-keyed tool cache described under +[Persistent tool cache](#persistent-tool-cache). The dev-side machinery is in: @@ -121,7 +123,8 @@ finish. 4. Watch the three jobs progress: `setup` (~3 min) → `matrix` (10 min) → `teardown` (~2 min). 5. Confirm no `/tmp/actions-runner-*` dirs remain on any cluster - host afterward. + host afterward. `/tmp/celeris-runner-tarballs/` is expected to + remain: it is the tool cache, not a leftover. ## Operator overrides @@ -129,4 +132,39 @@ If you want to keep runners alive across multiple workflow runs (useful when iterating on a tier locally), set the input `wait-for-manifest-clear: "false"` and skip the `teardown` job. Not intended for production — the pristine rule means we don't leave -state lying around between scheduled runs. +state lying around between scheduled runs. The tool cache below is the +one deliberate exception. + +## Persistent tool cache + +Every tier run needs the actions-runner tarball, uv, a python build, an +ansible-core venv and the `ansible.posix` collection. Fetching them fresh +on every run made each tier depend on four internet services at once +(probatorium#387, #392), so they live in a cache that survives teardown: + +``` +/tmp/celeris-runner-tarballs/ + actions-runner-linux--.tar.gz + tools/ + uv-/ the uv binary + uv-python/ uv-cache/ uv's python builds and wheel cache + ansible-venv-py-core-deps/ + ansible-collections-posix/ + ansible-home/ ansible-galaxy temp dirs and API cache +``` + +- **Keyed by pin.** Every directory name carries the version it holds, so a + bump in `runner-setup.yml` installs fresh instead of reusing a stale copy. + Old versions stay until a reboot; they are small (tens of MB each). +- **Trusted only when verified, never on presence.** uv must report its + pinned version. The venv needs a completion stamp *and* a working + `ansible-playbook --version` naming the pinned core. `ansible.posix` needs + a completion stamp *and* `ansible-galaxy collection verify --offline`. Any + miss discards that tool and reinstalls it. +- **Reached through symlinks.** Six cluster workflows hardcode + `/tmp/actions-runner-/ansible-venv` and `.../ansible-collections`; + the bootstrap recreates those as links into the cache on every run. + Teardown removes the links and leaves the cache. +- **Cleared by a reboot**, because `/tmp` is RAM-backed on these hosts. To + purge it by hand, run `rm -rf /tmp/celeris-runner-tarballs` on each host + while no cluster run is in progress; the next bootstrap repopulates it. diff --git a/ansible/runner-setup.yml b/ansible/runner-setup.yml index 19aad1b1..74bac651 100644 --- a/ansible/runner-setup.yml +++ b/ansible/runner-setup.yml @@ -5,6 +5,10 @@ # changes. The companion runner-teardown.yml wipes the directory and # de-registers from GitHub. # +# One thing persists by design: the version-keyed tool cache under +# /tmp/celeris-runner-tarballs -- the runner tarball, uv, a python build, +# the ansible venv and ansible.posix. Teardown leaves it; a reboot clears it. +# # This playbook follows the cluster-pristine rule: the runner ships # its own runtime tarball + supervises itself with `nohup ./run.sh` # (no systemd unit, no user lingering). When teardown runs the @@ -62,12 +66,32 @@ tool_cache_dir: "{{ runner_cache_dir }}/tools" uv_dir: "{{ tool_cache_dir }}/uv-{{ uv_version }}" # uv otherwise puts its python under $HOME/.local/share/uv/python and its - # cache under $HOME/.cache/uv -- outside /tmp, which quietly broke this - # playbook's own "nothing lands outside /tmp" rule. Both live here now. + # cache under $HOME/.cache/uv, and ansible-galaxy puts its download temp + # dirs and API cache under $HOME/.ansible -- all outside /tmp, which quietly + # broke this playbook's own "nothing lands outside /tmp" rule. All three + # live under the tool cache now. uv_python_install_dir: "{{ tool_cache_dir }}/uv-python" uv_cache_dir: "{{ tool_cache_dir }}/uv-cache" - ansible_venv_dir: "{{ tool_cache_dir }}/ansible-venv-py{{ python_version }}-core{{ ansible_core_version }}" + ansible_home_dir: "{{ tool_cache_dir }}/ansible-home" + # ansible-core is pinned; its dependencies (jinja2, PyYAML, cryptography, + # packaging, resolvelib) are frozen by resolving only what PyPI held at this + # cutoff. ansible-core 2.21.4 was uploaded 2026-09-08T16:51:47Z. The cutoff + # is part of the venv's directory name, so moving it builds a fresh venv + # instead of trusting one resolved under a different cutoff. + ansible_deps_exclude_newer: "2026-09-15T00:00:00Z" + ansible_venv_dir: "{{ tool_cache_dir }}/ansible-venv-py{{ python_version }}-core{{ ansible_core_version }}-deps{{ ansible_deps_exclude_newer | regex_replace('[^0-9]', '') }}" ansible_collections_dir: "{{ tool_cache_dir }}/ansible-collections-posix{{ ansible_posix_version }}" + # The GitHub-source fallback installs this COMMIT, not the 2.2.2 tag: a tag + # can be moved or deleted upstream, and the manifest check after the install + # reads its version from the same repository, so a moved tag would pass it. + # 2.2.2 is a lightweight tag on exactly this commit. + ansible_posix_commit: "e98d9a0756458be1ac710988498000973889075c" + # Every Galaxy attempt is bounded. retries never bound a stalled transfer: + # at the 2m29s stall that failed nightly 34981852841, four unbounded + # attempts would spend ~11 minutes of a 25-minute bootstrap before the + # GitHub fallback ran. The fallback clones and builds, so it gets its own. + galaxy_attempt_timeout: 120 + galaxy_git_fallback_timeout: 300 runner_arch_map: x86_64: x64 aarch64: arm64 @@ -117,6 +141,7 @@ path: "{{ runner_cache_dir }}" state: directory mode: '0755' + owner: "{{ ansible_user_id }}" # 215 MiB per host. Pointing dest inside runner_root made force:false # dead code -- the wipe above had just deleted the file it was meant @@ -214,10 +239,14 @@ no_log: true - name: Ensure tool cache dir + # owner: the cache is under world-writable /tmp and the playbook executes + # what it finds there. On a directory another local user created first a + # non-root chown fails and the play stops, instead of trusting it. ansible.builtin.file: path: "{{ tool_cache_dir }}" state: directory mode: '0755' + owner: "{{ ansible_user_id }}" - name: Install uv (pinned, cached across runs) # mage Deploy / Validate / Soak shell out to ansible-playbook. @@ -251,49 +280,55 @@ args: executable: /bin/bash register: uv_install - changed_when: "'already cached' not in uv_install.stdout" + changed_when: "'already cached' not in (uv_install.stdout | default(''))" retries: 5 delay: 10 until: uv_install is succeeded - - name: Create ansible venv via uv (pinned python, cached across runs) - # uv venv brings its own python distribution if the system lacks - # one with ensurepip -- exactly the cluster's situation. --clear - # only matters when the guard misses, i.e. a venv at this path - # lost its python: rebuild it rather than trip over the leftovers. + - name: Install ansible-core into a uv venv (pinned python, core and dependencies; cached across runs) + # mage Deploy / Validate / Soak shell out to ansible-playbook, and the + # cluster hosts have no system ansible (the pristine rule forbids it). # - # --managed-python is what keeps that python inside the cache. - # Without it uv prefers any matching python already on PATH over - # downloading into UV_PYTHON_INSTALL_DIR, and the "cached" venv - # then depends on a python that lives somewhere else entirely. An - # end-to-end run caught exactly that: the venv linked to the test - # controller's python, and a fresh container could not execute it. - # The cluster only escapes today because its system python is 3.14. - ansible.builtin.command: - cmd: >- - {{ uv_dir }}/uv venv --clear --managed-python - --python {{ python_version }} - {{ ansible_venv_dir }} - creates: "{{ ansible_venv_dir }}/bin/python" - environment: - UV_PYTHON_INSTALL_DIR: "{{ uv_python_install_dir }}" - UV_CACHE_DIR: "{{ uv_cache_dir }}" - register: ansible_venv_create - retries: 5 - delay: 10 - until: ansible_venv_create is succeeded - - - name: Install ansible-core into the venv via uv pip (pinned, cached across runs) - ansible.builtin.command: - cmd: >- - {{ uv_dir }}/uv pip install - --python {{ ansible_venv_dir }}/bin/python - ansible-core=={{ ansible_core_version }} - creates: "{{ ansible_venv_dir }}/bin/ansible-playbook" + # The venv and ansible-core are ONE cached unit, trusted only when a + # completion stamp written after a verified install exists AND + # ansible-playbook actually runs and reports the pinned core. A presence + # check is not enough, twice over (probatorium#393 review): + # - Ansible's `creates:` checks with glob.glob, which counts a DANGLING + # bin/python symlink as present, so a venv whose interpreter was + # deleted would be skipped and fail later inside a job. + # - uv writes a wheel's entry-point scripts before its package data, + # metadata and RECORD, so bin/ansible-playbook can exist over an + # install the bootstrap timeout killed half way. + # The probe executes the interpreter and imports ansible, which neither a + # dangling link nor a partial install can pass, and the stamp is written + # last. A miss rebuilds the whole venv from scratch. + # + # --managed-python is what keeps that python inside the cache. Without it + # uv prefers any matching python already on PATH over downloading into + # UV_PYTHON_INSTALL_DIR, and the "cached" venv then depends on a python + # that lives somewhere else entirely. An end-to-end run caught exactly + # that: the venv linked to the test controller's python, and a fresh + # container could not execute it. The cluster only escapes today because + # its system python is 3.14. + ansible.builtin.shell: | + set -e + stamp="{{ ansible_venv_dir }}/.celeris-installed" + if [ -f "$stamp" ] && "{{ ansible_venv_dir }}/bin/ansible-playbook" --version 2>/dev/null | head -1 | grep -qF "[core {{ ansible_core_version }}]"; then + echo "ansible-core {{ ansible_core_version }} already cached" + exit 0 + fi + rm -rf "{{ ansible_venv_dir }}" + {{ uv_dir }}/uv venv --clear --managed-python --python {{ python_version }} {{ ansible_venv_dir }} + {{ uv_dir }}/uv pip install --python {{ ansible_venv_dir }}/bin/python --exclude-newer {{ ansible_deps_exclude_newer }} ansible-core=={{ ansible_core_version }} + "{{ ansible_venv_dir }}/bin/ansible-playbook" --version | head -1 | grep -qF "[core {{ ansible_core_version }}]" + touch "$stamp" + args: + executable: /bin/bash environment: UV_PYTHON_INSTALL_DIR: "{{ uv_python_install_dir }}" UV_CACHE_DIR: "{{ uv_cache_dir }}" register: ansible_core_install + changed_when: "'already cached' not in (ansible_core_install.stdout | default(''))" retries: 5 delay: 10 until: ansible_core_install is succeeded @@ -302,24 +337,39 @@ # ansible.posix.synchronize, and ansible-core ships without bundled # collections. On 2026-09-15 galaxy.ansible.com gave one host nothing # for 2m29s (API time-to-first-byte 23s, while GitHub on the same host - # did 48 MB/s) and that one host failed the whole bootstrap. So trust - # the cache only if its own manifest names the pinned version; - # otherwise fetch from Galaxy, and if Galaxy will not serve it, build it - # from the collection's GitHub source tag. ansible.posix publishes no - # release assets on GitHub, so its git source is the only non-Galaxy - # route; git is present on every cluster host. + # did 48 MB/s) and that one host failed the whole bootstrap. + # + # The cache is trusted only when a completion stamp written after a + # verified install exists AND `ansible-galaxy collection verify --offline` + # finds every file matching its recorded hash. The manifest alone proves + # nothing: ansible-galaxy writes MANIFEST.json BEFORE any other collection + # file, so an install killed half way leaves a manifest naming the pin over + # a partial collection. + # + # A miss fetches from Galaxy, and if Galaxy will not serve it, builds the + # collection from its GitHub source at a pinned COMMIT. ansible.posix + # publishes no release assets, so its git source is the only non-Galaxy + # route; git is present on every cluster host. Each attempt is bounded by + # coreutils timeout, because retries never bound a stall. ANSIBLE_HOME keeps + # ansible-galaxy's download temp dirs and API cache inside the tool cache. - name: Install ansible.posix (pinned; Galaxy, then GitHub source; cached across runs) + environment: + ANSIBLE_HOME: "{{ ansible_home_dir }}" block: - - name: Read the cached ansible.posix manifest - ansible.builtin.slurp: - src: "{{ ansible_collections_dir }}/ansible_collections/ansible/posix/MANIFEST.json" - register: ansible_posix_manifest - - - name: Check the cached ansible.posix is the pinned version - ansible.builtin.assert: - that: - - (ansible_posix_manifest.content | b64decode | from_json).collection_info.version == ansible_posix_version - quiet: true + - name: Check the cached ansible.posix is complete and the pinned version + ansible.builtin.shell: | + set -e + test -f "{{ ansible_collections_dir }}/.celeris-installed-posix{{ ansible_posix_version }}" + # verify --offline hashes every file but not MANIFEST.json itself, so a + # manifest whose version was altered passes it (measured end to end). + "{{ ansible_venv_dir }}/bin/python" -c 'import json, sys; sys.exit(json.load(open(sys.argv[1]))["collection_info"]["version"] != sys.argv[2])' \ + "{{ ansible_collections_dir }}/ansible_collections/ansible/posix/MANIFEST.json" "{{ ansible_posix_version }}" + "{{ ansible_venv_dir }}/bin/ansible-galaxy" collection verify --offline \ + --collections-path "{{ ansible_collections_dir }}" \ + "ansible.posix:=={{ ansible_posix_version }}" + args: + executable: /bin/bash + changed_when: false rescue: - name: Discard any partial or mismatched ansible.posix ansible.builtin.file: @@ -331,22 +381,25 @@ - name: Install ansible.posix from Galaxy ansible.builtin.command: cmd: >- + timeout {{ galaxy_attempt_timeout }} {{ ansible_venv_dir }}/bin/ansible-galaxy collection install + --timeout 30 --collections-path {{ ansible_collections_dir }} ansible.posix:=={{ ansible_posix_version }} register: ansible_posix_galaxy - retries: 3 + retries: 1 delay: 15 until: ansible_posix_galaxy is succeeded rescue: - - name: Install ansible.posix from its GitHub source tag + - name: Install ansible.posix from its GitHub source commit ansible.builtin.command: cmd: >- + timeout {{ galaxy_git_fallback_timeout }} {{ ansible_venv_dir }}/bin/ansible-galaxy collection install --force --collections-path {{ ansible_collections_dir }} - git+https://github.com/ansible-collections/ansible.posix.git,{{ ansible_posix_version }} + git+https://github.com/ansible-collections/ansible.posix.git,{{ ansible_posix_commit }} register: ansible_posix_git - retries: 3 + retries: 2 delay: 15 until: ansible_posix_git is succeeded @@ -361,6 +414,29 @@ - (ansible_posix_manifest.content | b64decode | from_json).collection_info.version == ansible_posix_version fail_msg: "ansible.posix installed but its MANIFEST.json does not name {{ ansible_posix_version }}" + - name: Verify every installed ansible.posix file against its recorded hash + ansible.builtin.command: + cmd: >- + {{ ansible_venv_dir }}/bin/ansible-galaxy collection verify --offline + --collections-path {{ ansible_collections_dir }} + ansible.posix:=={{ ansible_posix_version }} + changed_when: false + + - name: Stamp ansible.posix as completely installed + ansible.builtin.file: + path: "{{ ansible_collections_dir }}/.celeris-installed-posix{{ ansible_posix_version }}" + state: touch + mode: '0644' + + - name: Confirm the cached tool directories exist before linking to them + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ ansible_venv_dir }}" + - "{{ ansible_collections_dir }}" + register: cached_tool_dir + failed_when: not (cached_tool_dir.stat.isdir | default(false)) + - name: Point the legacy runner_root tool paths at the cache # Six cluster workflows (nightly, weekend, benchmark, checkptr, # race, pr-tier) never read the exports in "Start runner": each @@ -373,26 +449,33 @@ # shutil.rmtree -- that unlinks a symlink and never follows it # (measured on Linux CPython 3.13: avoids_symlink_attacks=True and # the target survives), so the wipe drops the links, not the cache. + # + # No force: true. The wipe above leaves both paths absent, replacing a + # stale link needs no force anyway, and force would switch off ansible's + # only check that the target exists: a missing cache dir would become a + # dangling link and a WARNING instead of a failure. ansible.builtin.file: src: "{{ item.src }}" dest: "{{ item.dest }}" state: link - force: true loop: - { src: "{{ ansible_venv_dir }}", dest: "{{ runner_root }}/ansible-venv" } - { src: "{{ ansible_collections_dir }}", dest: "{{ runner_root }}/ansible-collections" } - name: Start runner (nohup) - # Prepend the runner-scoped ansible venv to PATH so any job + # Prepend the ansible venv to PATH so any job # that lands on this runner finds ansible-playbook without # needing an explicit per-job install step. The export goes # into the runner's process environment because run.sh # inherits the shell's PATH. # # ANSIBLE_COLLECTIONS_PATH points ansible-playbook at the - # runner-scoped collections dir (ansible.posix lives there) — + # collections dir (ansible.posix lives there) — # without it ansible-galaxy's collection install is invisible # to ansible-playbook at runtime. + # + # Both paths are symlinks into the host's version-keyed tool cache; + # see "Point the legacy runner_root tool paths at the cache". ansible.builtin.shell: | cd {{ runner_root }} export PATH="{{ runner_root }}/ansible-venv/bin:$PATH" diff --git a/ansible/runner-teardown.yml b/ansible/runner-teardown.yml index 3965beba..44f199bb 100644 --- a/ansible/runner-teardown.yml +++ b/ansible/runner-teardown.yml @@ -103,7 +103,10 @@ path: "{{ manifest_path | default('/tmp/celeris-bench-manifest.json') }}" state: absent - - name: Confirm pristine state + # The runner dir must be gone. The version-keyed tool cache under + # /tmp/celeris-runner-tarballs is NOT removed and is not checked here: it + # survives teardown on purpose, so this confirms the runner dir only. + - name: Confirm the runner dir is gone ansible.builtin.stat: path: "{{ runner_root }}" register: post_teardown diff --git a/workflow_runner_tarball_cache_test.go b/workflow_runner_tarball_cache_test.go index cf376e9b..4b51753f 100644 --- a/workflow_runner_tarball_cache_test.go +++ b/workflow_runner_tarball_cache_test.go @@ -170,7 +170,7 @@ func TestBootstrapToolchainIsPinnedAndCachedOutsideTheWipedDir(t *testing.T) { "venv python": "--python {{ python_version }}", "ansible-core": "ansible-core=={{ ansible_core_version }}", "ansible.posix from Galaxy": "ansible.posix:=={{ ansible_posix_version }}", - "ansible.posix from GitHub": "ansible.posix.git,{{ ansible_posix_version }}", + "ansible.posix from GitHub": "ansible.posix.git,{{ ansible_posix_commit }}", } { if !strings.Contains(setup, want) { t.Errorf("%s does not install its pin (want %q in runner-setup.yml)", what, want) @@ -256,3 +256,103 @@ func TestBootstrapToolchainIsPinnedAndCachedOutsideTheWipedDir(t *testing.T) { t.Logf("toolchain contract: %d workflow(s) read ansible-venv, %d read ansible-collections", consumers["ansible-venv"], consumers["ansible-collections"]) } + +// probatorium#393 review: a cache that skips on presence is a liability, and the +// first cut still skipped on presence in three places. Ansible's `creates:` +// checks with glob.glob, which counts a DANGLING bin/python symlink as present; +// uv writes a wheel's entry-point scripts before its package data, metadata and +// RECORD, so bin/ansible-playbook can exist over a half-finished install; and +// ansible-galaxy writes MANIFEST.json before any collection file, so a manifest +// naming the pin proves nothing about the files after it. A job killed by the +// bootstrap timeout leaves exactly those states, and the cache now outlives the +// run that left them. +// +// So each cached tool is trusted only when a stamp written after a verified +// install exists AND a check that exercises the tool agrees: the venv is probed +// by running ansible-playbook, and ansible.posix by verifying every file's hash. +func TestBootstrapToolchainCacheIsTrustedOnlyWhenVerified(t *testing.T) { + setup := readPlaybook(t, "runner-setup.yml") + + for name, re := range map[string]*regexp.Regexp{ + "ansible_deps_exclude_newer": regexp.MustCompile(`(?m)^\s*ansible_deps_exclude_newer:\s*"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"\s*$`), + "ansible_posix_commit": regexp.MustCompile(`(?m)^\s*ansible_posix_commit:\s*"[0-9a-f]{40}"\s*$`), + "galaxy_attempt_timeout": regexp.MustCompile(`(?m)^\s*galaxy_attempt_timeout:\s*\d+\s*$`), + "galaxy_git_fallback_timeout": regexp.MustCompile(`(?m)^\s*galaxy_git_fallback_timeout:\s*\d+\s*$`), + } { + if !re.MatchString(setup) { + t.Errorf("%s is not set to a concrete value", name) + } + } + + for what, want := range map[string]string{ + "ansible-core dependencies frozen at a cutoff": "--exclude-newer {{ ansible_deps_exclude_newer }}", + "the cutoff keys the venv directory": `-core{{ ansible_core_version }}-deps{{ ansible_deps_exclude_newer`, + "venv trusted only with a completion stamp": "{{ ansible_venv_dir }}/.celeris-installed", + "venv trusted only if ansible-playbook runs": "[core {{ ansible_core_version }}]", + "ansible.posix verified file by file": "collection verify --offline", + "ansible.posix trusted only with a stamp": ".celeris-installed-posix{{ ansible_posix_version }}", + "each Galaxy attempt is bounded": "timeout {{ galaxy_attempt_timeout }}", + "the GitHub fallback is bounded": "timeout {{ galaxy_git_fallback_timeout }}", + "ansible-galaxy keeps its temp and cache in the tool cache": `ANSIBLE_HOME: "{{ ansible_home_dir }}"`, + "ANSIBLE_HOME lives in the tool cache": `ansible_home_dir: "{{ tool_cache_dir }}/`, + } { + if !strings.Contains(setup, want) { + t.Errorf("%s: want %q in runner-setup.yml", what, want) + } + } + + // A presence guard on the venv is exactly what the review caught. + if strings.Contains(setup, `creates: "{{ ansible_venv_dir }}`) { + t.Error("the venv is still guarded by creates:, which glob.glob satisfies with a dangling " + + "bin/python symlink and with a half-installed bin/ansible-playbook") + } + + // force: true on the link task switches off ansible's only check that the + // link target exists, so a missing cache dir would become a dangling link + // and a WARNING instead of a failure. + const linkTask = "- name: Point the legacy runner_root tool paths at the cache" + i := strings.Index(setup, linkTask) + if i < 0 { + t.Fatal("runner-setup.yml has no link task to check") + } + task := setup[i+len(linkTask):] + if j := strings.Index(task, "\n - name:"); j >= 0 { + task = task[:j] + } + // Match force as a YAML key, not as text: the task's own comment explains + // why there is no force, and a substring check tripped on that sentence. + // Any truthy spelling counts; ansible accepts yes/on/true in any case. + forceKey := regexp.MustCompile(`(?mi)^\s*force:\s*(true|yes|on)\s*$`) + if forceKey.MatchString(task) { + t.Error("the link task sets force: true, which turns off the check that its target exists") + } +} + +// ansible-galaxy collection verify --offline checks every file against the +// hashes recorded in FILES.json, but it does not hash MANIFEST.json itself. +// The round-3 end-to-end run measured the consequence: a cached manifest whose +// version had been altered to 0.0.0 passed the cached check, and the play +// reported success over it. So the cached check must compare the manifest's +// version to the pin as well as run verify. +func TestCachedAnsiblePosixCheckComparesTheManifestVersion(t *testing.T) { + setup := readPlaybook(t, "runner-setup.yml") + const name = "- name: Check the cached ansible.posix is complete and the pinned version" + i := strings.Index(setup, name) + if i < 0 { + t.Fatal("runner-setup.yml has no cached ansible.posix check") + } + task := setup[i+len(name):] + if j := strings.Index(task, "\n - name:"); j >= 0 { + task = task[:j] + } + for what, want := range map[string]string{ + "the completion stamp": ".celeris-installed-posix{{ ansible_posix_version }}", + "a file-by-file hash check": "collection verify --offline", + "the manifest version against the pin": `["collection_info"]["version"] != sys.argv[2]`, + } { + if !strings.Contains(task, want) { + t.Errorf("the cached ansible.posix check lacks %s (want %q): verify --offline does not hash "+ + "MANIFEST.json, so a manifest naming the wrong version passes it", what, want) + } + } +}