diff --git a/.github/actions/nexus-bootstrap/action.yml b/.github/actions/nexus-bootstrap/action.yml index 04fa30c2..6493b173 100644 --- a/.github/actions/nexus-bootstrap/action.yml +++ b/.github/actions/nexus-bootstrap/action.yml @@ -78,16 +78,19 @@ runs: - name: Sync nexus_deploy runtime deps # --no-dev: ruff/mypy/pytest are not needed at deploy time. shell: bash - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared # Needed wherever SSH goes over Cloudflare Access, i.e. anything # calling nexus_deploy.setup.configure_ssh. + # Pinned, and without sudo: a Forgejo job image has none (#884). shell: bash - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + run: bash "$GITHUB_WORKSPACE/.github/scripts/install-tool.sh" cloudflared - name: Install SSH key shell: bash diff --git a/.github/scripts/install-tool.sh b/.github/scripts/install-tool.sh new file mode 100644 index 00000000..7e482da5 --- /dev/null +++ b/.github/scripts/install-tool.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# ============================================================================= +# install-tool.sh — install a pinned tool that a job image may not have +# ============================================================================= +# The lifecycle workflows were written for GitHub's `ubuntu-latest`, which +# ships jq, the AWS CLI and sudo. A Conductor tenant fork runs them on a +# Forgejo runner whose job image is `node:22-bookworm`, which has none of +# these (#884). Each job therefore installs what it needs with this script, +# the way install-opentofu.sh installs OpenTofu: a pinned version, a pinned +# checksum, a directory under $RUNNER_TEMP, and no sudo. It works the same on +# both runners. +# +# Usage: install-tool.sh +# +# Installs into $RUNNER_TEMP/tools/--/ and, when +# GITHUB_PATH is set, puts that directory on PATH for the following steps. +# +# Where the checksums come from (checked 2026-09-17, amd64 and arm64): +# jq 1.8.2 the release's sha256sum.txt, equal to the asset digest +# GitHub records for the release, and to the download +# cloudflared 2026.9.1 +# the checksums in the release notes, equal to GitHub's +# asset digest, and to the download +# AWS CLI 2.36.47 the zip's PGP signature verified against the key the +# AWS CLI install guide publishes (fingerprint FB5D B77F +# D5C1 18B8 0511 ADA8 A631 0ACC 4672 475C) -> "Good +# signature"; the same signature on the other +# architecture's zip -> "BAD signature" +# +# To move a tool to another version: repeat that check, then replace its +# version and both checksums below. Replace, do not add. +# ============================================================================= +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "❌ usage: install-tool.sh " >&2 + exit 1 +fi +TOOL="$1" + +if [ "$(uname -s)" != "Linux" ]; then + echo "❌ install-tool.sh supports Linux runners only (got $(uname -s))." >&2 + exit 1 +fi + +case "$(uname -m)" in + x86_64 | amd64) ARCH=amd64 ;; + aarch64 | arm64) ARCH=arm64 ;; + *) + echo "❌ No $TOOL build pinned for architecture $(uname -m)." >&2 + exit 1 + ;; +esac + +# The pins. VERSION, the download URL, its checksum per architecture, the +# command the tool installs, and the text its version output must contain. +case "$TOOL/$ARCH" in + jq/amd64 | jq/arm64) + VERSION="1.8.2" + FILE="jq-linux-$ARCH" + URL="https://github.com/jqlang/jq/releases/download/jq-$VERSION/$FILE" + case "$ARCH" in + amd64) SHA256="b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" ;; + arm64) SHA256="8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309" ;; + esac + COMMAND="jq" + EXPECT="jq-$VERSION" + ;; + cloudflared/amd64 | cloudflared/arm64) + VERSION="2026.9.1" + FILE="cloudflared-linux-$ARCH" + URL="https://github.com/cloudflare/cloudflared/releases/download/$VERSION/$FILE" + case "$ARCH" in + amd64) SHA256="03f1f25d1cc93b9ad6c60569d44060bc4f17ed97075760ed8cfca4b12dcd68cc" ;; + arm64) SHA256="3d97437c71848bd8df68041e12436b484a661d95073ea1937f01a845ce88faa3" ;; + esac + COMMAND="cloudflared" + EXPECT="cloudflared version $VERSION " + ;; + awscli/amd64 | awscli/arm64) + VERSION="2.36.47" + case "$ARCH" in + amd64) FILE="awscli-exe-linux-x86_64-$VERSION.zip" + SHA256="44ab93f1c6e90e0a34dfe5da6b798d99f2f6b41825e97022b3d409d93a65c615" ;; + arm64) FILE="awscli-exe-linux-aarch64-$VERSION.zip" + SHA256="118ef53b6f5c652471fd299af01c058901c1a53db69c6b71a7bf3ebd93590969" ;; + esac + URL="https://awscli.amazonaws.com/$FILE" + COMMAND="aws" + EXPECT="aws-cli/$VERSION " + ;; + *) + echo "❌ Unknown tool '$TOOL' (expected jq, cloudflared or awscli)." >&2 + exit 1 + ;; +esac + +DEST="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/tools/$TOOL-$VERSION-$ARCH" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +if ! curl -fsSL --retry 3 --retry-delay 2 -o "$WORK/$FILE" "$URL"; then + echo "❌ Could not download $URL" >&2 + exit 1 +fi + +# `sha256sum` prints " "; the expansion keeps only the hash +# without a pipe (#883). +if ! SUM=$(sha256sum "$WORK/$FILE"); then + echo "❌ Could not compute the checksum of $FILE." >&2 + exit 1 +fi +ACTUAL=${SUM%% *} +if [ "$ACTUAL" != "$SHA256" ]; then + echo "❌ Checksum mismatch for $FILE — refusing to install it." >&2 + echo " expected $SHA256" >&2 + echo " got $ACTUAL" >&2 + exit 1 +fi + +mkdir -p "$DEST" +case "$TOOL" in + awscli) + # The official installer, without root: it takes an install directory + # and a bin directory, both under $DEST. unzip is in both job images; + # Python's zipfile is the fallback, and python3 is in both as well. + mkdir -p "$WORK/extract" + if command -v unzip >/dev/null; then + unzip -q "$WORK/$FILE" -d "$WORK/extract" + else + python3 -m zipfile -e "$WORK/$FILE" "$WORK/extract" + chmod -R u+x "$WORK/extract/aws" + fi + "$WORK/extract/aws/install" --install-dir "$DEST/aws-cli" --bin-dir "$DEST/bin" --update >/dev/null + BIN_DIR="$DEST/bin" + ;; + *) + install -m 0755 "$WORK/$FILE" "$DEST/$COMMAND" + BIN_DIR="$DEST" + ;; +esac + +# Read back what was installed. Whole output first, no pipe (#883). +if ! INSTALLED=$("$BIN_DIR/$COMMAND" --version 2>&1); then + echo "❌ $BIN_DIR/$COMMAND --version failed: ${INSTALLED:-(no output)}" >&2 + exit 1 +fi +INSTALLED=${INSTALLED%%$'\n'*} +case "$INSTALLED" in + "$EXPECT"*) ;; + *) + echo "❌ Installed $COMMAND reports '$INSTALLED', expected it to start with '$EXPECT'." >&2 + exit 1 + ;; +esac + +if [ -n "${GITHUB_PATH:-}" ]; then + echo "$BIN_DIR" >> "$GITHUB_PATH" +fi +echo "✅ $INSTALLED installed at $BIN_DIR" diff --git a/.github/workflows/destroy-all.yml b/.github/workflows/destroy-all.yml index 5a7fcea2..9ae5fdc2 100644 --- a/.github/workflows/destroy-all.yml +++ b/.github/workflows/destroy-all.yml @@ -81,6 +81,16 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + + - name: Install AWS CLI + # Deletes the R2 buckets when delete_data=DESTROY. The official v2 + # installer, pinned and without root (#884). + run: bash .github/scripts/install-tool.sh awscli + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -119,7 +129,12 @@ jobs: - name: Sync nexus_deploy runtime deps # --no-dev: ruff/mypy/pytest are not needed here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Generate SSH key for OpenTofu run: | diff --git a/.github/workflows/job-image-toolchain.yaml b/.github/workflows/job-image-toolchain.yaml new file mode 100644 index 00000000..b45115cc --- /dev/null +++ b/.github/workflows/job-image-toolchain.yaml @@ -0,0 +1,106 @@ +name: "CI: Job image toolchain" + +# ============================================================================= +# The lifecycle toolchain, installed inside the Forgejo runner's job image. +# ============================================================================= +# A Conductor tenant fork runs the lifecycle workflows on a Forgejo runner +# whose job image is node:22-bookworm (stacks/forgejo-runner/runner-config.yml). +# That image has no sudo, jq or AWS CLI (#884), so every lifecycle job +# installs them itself. This job runs those installs in the same image on +# GitHub, so a change that breaks them fails here, on the pull request, and +# not in a tenant's first deploy. +# +# It installs; it does not deploy. Whether a whole lifecycle run gets +# through is only shown by one, on a fork. +# ============================================================================= + +on: + pull_request: + paths: + - ".github/scripts/install-tool.sh" + - ".github/scripts/install-opentofu.sh" + - ".github/actions/**" + - ".github/workflows/job-image-toolchain.yaml" + - "stacks/forgejo-runner/runner-config.yml" + - "pyproject.toml" + - "uv.lock" + workflow_dispatch: + +concurrency: + group: job-image-toolchain-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + toolchain: + name: Toolchain in node:22-bookworm + runs-on: ubuntu-latest + # Must match the image the runner maps its labels to; + # tests/unit/test_job_toolchain.py keeps the two in step. + container: + image: node:22-bookworm + # A container job on GitHub defaults to `sh`; the Forgejo runner runs + # these steps with bash, and so does this job. + defaults: + run: + shell: bash + steps: + - name: Checkout + # Pinned as #652 asks for added `uses:` lines. The same commit is on + # data.forgejo.org, which a Forgejo runner resolves actions from. + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: What the image lacks + # Informational: the reason the steps below exist. + run: | + for tool in sudo jq aws cloudflared tofu uv; do + if command -v "$tool" >/dev/null; then + echo " $tool: present in the image" + else + echo " $tool: missing from the image" + fi + done + echo " running as uid $(id -u)" + + - name: Install jq + run: bash .github/scripts/install-tool.sh jq + + - name: Install cloudflared + run: bash .github/scripts/install-tool.sh cloudflared + + - name: Install AWS CLI + run: bash .github/scripts/install-tool.sh awscli + + - name: Install OpenTofu + run: bash .github/scripts/install-opentofu.sh + + - name: Install uv + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 + with: + version: "0.11.7" + enable-cache: true + + - name: Sync nexus_deploy runtime deps + run: | + uv sync --frozen --no-dev + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + + - name: Every tool answers from PATH + run: | + set -euo pipefail + jq --version + cloudflared --version + aws --version + tofu version + uv --version + python3 -c 'import yaml; print("PyYAML", yaml.__version__)' + # The script the lifecycle workflows run with python3 and PyYAML. + # Rendering the real services.yaml, not only importing the module. + export ENABLED_SERVICES="forgejo,grafana" FIREWALL_RULES="" + mkdir -p tofu/stack && : > tofu/stack/config.tfvars + python3 .github/scripts/generate-services-tfvars.py + grep -q '^services = {' tofu/stack/config.tfvars diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 8f8d90ed..4942ebce 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -30,6 +30,10 @@ name: "CI: Python Tests" +# The tests also guard files outside src/: the lifecycle workflows and +# actions (test_job_toolchain.py, test_install_opentofu.py) and the scripts +# under .github/scripts/. A pull request that changes only those must run +# them too. on: pull_request: paths: @@ -40,7 +44,9 @@ on: - "pyproject.toml" - "uv.lock" - ".python-version" - - ".github/workflows/python-tests.yml" + - ".github/workflows/**" + - ".github/actions/**" + - ".github/scripts/**" - "scripts/*.sh" push: branches: [main] @@ -52,7 +58,9 @@ on: - "pyproject.toml" - "uv.lock" - ".python-version" - - ".github/workflows/python-tests.yml" + - ".github/workflows/**" + - ".github/actions/**" + - ".github/scripts/**" - "scripts/*.sh" # Cancel previous runs of this workflow on the same ref. The CI is fast diff --git a/.github/workflows/setup-control-plane.yaml b/.github/workflows/setup-control-plane.yaml index f2e5d4dc..e53eac05 100644 --- a/.github/workflows/setup-control-plane.yaml +++ b/.github/workflows/setup-control-plane.yaml @@ -150,6 +150,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -184,10 +189,30 @@ jobs: run: bash .github/scripts/install-opentofu.sh - name: Install cloudflared + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared + + - name: Install AWS CLI + # Validates the R2 credentials and cleans up buckets below. The + # official v2 installer, pinned and without root (#884). + run: bash .github/scripts/install-tool.sh awscli + + - name: Install uv + # sync-deployed-state.sh and sync-firewall-rules.sh import PyYAML; + # the project environment provides it (#884). Pinned as #652 asks for + # added `uses:` lines; the commit is also on data.forgejo.org. + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 + with: + version: "0.11.7" + enable-cache: true + + - name: Sync nexus_deploy runtime deps run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Generate SSH key for server access id: ssh_key diff --git a/.github/workflows/spin-up.yml b/.github/workflows/spin-up.yml index d7d247e3..b330bc8f 100644 --- a/.github/workflows/spin-up.yml +++ b/.github/workflows/spin-up.yml @@ -152,6 +152,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -229,13 +234,16 @@ jobs: # match pyproject.toml (which CI's python-tests already # enforces); `--no-dev` skips ruff/mypy/pytest etc. since we # only need runtime here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared - name: Setup SSH key for server access run: | diff --git a/.github/workflows/teardown-snapshot.yml b/.github/workflows/teardown-snapshot.yml index 09c70ffc..d4ca3cc2 100644 --- a/.github/workflows/teardown-snapshot.yml +++ b/.github/workflows/teardown-snapshot.yml @@ -98,6 +98,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml index d406738f..55e5b1fd 100644 --- a/.github/workflows/teardown.yml +++ b/.github/workflows/teardown.yml @@ -47,6 +47,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -98,16 +103,19 @@ jobs: - name: Sync nexus_deploy runtime deps # Pre-resolve the lockfile-pinned runtime deps. `--no-dev` # skips ruff/mypy/pytest since we only need runtime here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared # Needed by the SSH-over-Cloudflare-Access setup in # `nexus_deploy.pipeline.run_snapshot` (configure_ssh writes # a Host block that uses ProxyCommand=cloudflared). - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared - name: Install SSH key from GitHub Secrets # Teardown's pre-destroy s3-snapshot step needs SSH into the diff --git a/CLAUDE.md b/CLAUDE.md index 92f34ae0..a211fcf8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -317,6 +317,18 @@ harmless, because the value is only compared against a success code; in a hook that *prints* the code or branches on it with `case`, it is not. The answer usually reveals that your case differs. +### Tools in lifecycle workflows + +The lifecycle workflows also run on a Forgejo runner whose job image is +`node:22-bookworm`: root, no `sudo`, no `jq`, no AWS CLI (#884). So a +lifecycle job installs every tool beyond that image in an earlier step: +OpenTofu with `.github/scripts/install-opentofu.sh`; jq, cloudflared and the +AWS CLI with `.github/scripts/install-tool.sh `; uv with +`astral-sh/setup-uv`. PyYAML comes from the project environment, whose +`.venv/bin` goes onto `PATH` after `uv sync`. Never `sudo`, never `apt-get`. +`tests/unit/test_job_toolchain.py` enforces it, and +`docs/stacks/forgejo-runner.md` lists what the image contains. + ### Service Account Naming Convention All service accounts MUST use the `nexus-` prefix to prevent default username guessing: diff --git a/docs/stacks/forgejo-runner.md b/docs/stacks/forgejo-runner.md index def92827..d496f0c6 100644 --- a/docs/stacks/forgejo-runner.md +++ b/docs/stacks/forgejo-runner.md @@ -146,6 +146,40 @@ That is **not** GitHub's large `ubuntu` runner image. A workflow that assumes Python, Go, Docker CLI or the AWS CLI is preinstalled will need to install them first. +### What the job image contains + +Probed on 2026-09-17 inside the stack's own daemon: + +| Present | Missing | +|---|---| +| `bash`, `curl`, `wget`, `git`, `ssh`, `ssh-keygen`, `python3`, `unzip`, `tar`, `node`, `npm` | `sudo`, `jq`, `aws`, `gh`, `pip` | + +Jobs run as root, so nothing needs `sudo`, and nothing may use it. + +Nexus-Stack's own lifecycle workflows (Initial Setup, Setup Control +Plane, Spin Up, Teardown, Destroy All and the snapshot pair) install what +they need beyond that list themselves, so they run unchanged on GitHub +and on this runner (#884): + +| Tool | Installed by | +|---|---| +| OpenTofu | `.github/scripts/install-opentofu.sh` | +| jq, cloudflared, AWS CLI v2 | `.github/scripts/install-tool.sh `, pinned versions and checksums, into `$RUNNER_TEMP`, without root | +| uv | `astral-sh/setup-uv` | +| PyYAML | the project environment: after `uv sync`, `.venv/bin` goes first on `PATH`, so `python3` finds it | + +Two checks keep it that way: + +- `tests/unit/test_job_toolchain.py` fails when a lifecycle job uses one + of these tools before a step of the same job installed it, or uses + `sudo` or `apt-get` at all. It follows local actions and the scripts a + step calls. +- `.github/workflows/job-image-toolchain.yaml` runs the installs in + `node:22-bookworm` on every pull request that touches them. + +Neither proves that a whole lifecycle run passes on this runner. Only a +real run on a fork shows that. + Other known differences from GitHub Actions: - `permissions:` and `continue-on-error:` on a job are ignored. diff --git a/tests/unit/test_install_tool.py b/tests/unit/test_install_tool.py new file mode 100644 index 00000000..a6d743da --- /dev/null +++ b/tests/unit/test_install_tool.py @@ -0,0 +1,178 @@ +"""install-tool.sh installs pinned jq, cloudflared and the AWS CLI (#884). + +The refusals are run under bash with `curl` and `uname` replaced. The +success path cannot be faked, because the checksums are pinned to the real +release files. It runs for real in `.github/workflows/job-image-toolchain.yaml`, +inside the Forgejo runner's job image. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / ".github" / "scripts" / "install-tool.sh" +TOOLS = ("jq", "cloudflared", "awscli") + +_FAKE_CURL = """#!/usr/bin/env bash +out="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) out="$2"; shift 2 ;; + http*) echo "$1" >> "$FAKE_LOG"; shift ;; + *) shift ;; + esac +done +printf 'not the real file' > "$out" +""" + +_FAKE_UNAME = """#!/usr/bin/env bash +case "$1" in + -s) echo "$FAKE_UNAME_S" ;; + -m) echo "$FAKE_UNAME_M" ;; + *) echo "$FAKE_UNAME_S" ;; +esac +""" + +# A real sha256, so the mismatch is computed rather than faked; macOS has no +# `sha256sum`, CI does, and both must reach the same comparison. +_SHA256SUM = f"""#!{sys.executable} +import hashlib, sys +for path in sys.argv[1:]: + print(hashlib.sha256(open(path, "rb").read()).hexdigest() + " " + path) +""" + + +def _run( + tmp_path: Path, *args: str, uname_s: str = "Linux", uname_m: str = "x86_64" +) -> tuple[int, str, str, list[str]]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + for name, body in (("curl", _FAKE_CURL), ("uname", _FAKE_UNAME), ("sha256sum", _SHA256SUM)): + (bin_dir / name).write_text(body) + (bin_dir / name).chmod(0o755) + gh_path = tmp_path / "github_path" + gh_path.write_text("") + log = tmp_path / "curl.log" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "FAKE_LOG": str(log), + "FAKE_UNAME_S": uname_s, + "FAKE_UNAME_M": uname_m, + "GITHUB_PATH": str(gh_path), + "RUNNER_TEMP": str(tmp_path / "runner-temp"), + } + done = subprocess.run( + ["bash", str(SCRIPT), *args], capture_output=True, text=True, env=env, check=False + ) + urls = log.read_text().splitlines() if log.exists() else [] + return done.returncode, done.stderr, gh_path.read_text(), urls + + +def _pins() -> dict[tuple[str, str], tuple[str, str]]: + """(tool, arch) -> (version, sha256), read from the script.""" + text = SCRIPT.read_text() + out = {} + for tool in TOOLS: + block = re.search(rf"^ {tool}/amd64 \| {tool}/arm64\)(.*?)^ ;;", text, re.M | re.S) + assert block, tool + body = block.group(1) + version = re.search(r'VERSION="([^"]+)"', body) + assert version, tool + for arch in ("amd64", "arm64"): + sha = re.search(rf'{arch}\).*?SHA256="([0-9a-f]+)"', body, re.S) + assert sha, (tool, arch) + out[(tool, arch)] = (version.group(1), sha.group(1)) + return out + + +def test_every_tool_has_a_version_and_two_well_formed_checksums() -> None: + pins = _pins() + assert len(pins) == len(TOOLS) * 2 + for (tool, arch), (version, sha) in pins.items(): + assert re.fullmatch(r"[0-9]+(\.[0-9]+)+", version), (tool, version) + assert re.fullmatch(r"[0-9a-f]{64}", sha), (tool, arch) + # One checksum per file: amd64 and arm64 must differ. + for tool in TOOLS: + assert pins[(tool, "amd64")][1] != pins[(tool, "arm64")][1], tool + + +@pytest.mark.parametrize("args", [(), ("jq", "cloudflared"), ("yq",), ("../evil",)]) +def test_the_script_refuses_anything_but_one_known_tool( + tmp_path: Path, args: tuple[str, ...] +) -> None: + rc, err, gh_path, urls = _run(tmp_path, *args) + assert rc == 1 + assert "usage" in err or "Unknown tool" in err + assert urls == [] + assert gh_path == "" + + +def test_the_script_refuses_a_non_linux_runner(tmp_path: Path) -> None: + rc, err, gh_path, urls = _run(tmp_path, "jq", uname_s="Darwin") + assert rc == 1 + assert "Linux runners only" in err + assert (gh_path, urls) == ("", []) + + +def test_the_script_refuses_an_unpinned_architecture(tmp_path: Path) -> None: + rc, err, gh_path, urls = _run(tmp_path, "jq", uname_m="riscv64") + assert rc == 1 + assert "No jq build pinned" in err + assert (gh_path, urls) == ("", []) + + +_EXPECTED_URLS = { + ("jq", "amd64"): "https://github.com/jqlang/jq/releases/download/jq-{v}/jq-linux-amd64", + ("jq", "arm64"): "https://github.com/jqlang/jq/releases/download/jq-{v}/jq-linux-arm64", + ( + "cloudflared", + "amd64", + ): "https://github.com/cloudflare/cloudflared/releases/download/{v}/cloudflared-linux-amd64", + ( + "cloudflared", + "arm64", + ): "https://github.com/cloudflare/cloudflared/releases/download/{v}/cloudflared-linux-arm64", + ("awscli", "amd64"): "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{v}.zip", + ("awscli", "arm64"): "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{v}.zip", +} + + +@pytest.mark.parametrize(("tool", "arch"), sorted(_EXPECTED_URLS)) +def test_a_file_that_does_not_match_its_pin_is_not_installed( + tmp_path: Path, tool: str, arch: str +) -> None: + """The refusal that protects anything. The URL is asserted, so the test + is known to have reached the comparison for this tool and architecture.""" + version, sha = _pins()[(tool, arch)] + uname_m = {"amd64": "x86_64", "arm64": "aarch64"}[arch] + rc, err, gh_path, urls = _run(tmp_path, tool, uname_m=uname_m) + assert rc == 1, err + assert "Checksum mismatch" in err + assert urls == [_EXPECTED_URLS[(tool, arch)].format(v=version)] + assert sha in err + assert hashlib.sha256(b"not the real file").hexdigest() in err + assert gh_path == "" + assert not list((tmp_path / "runner-temp").rglob("*")), "nothing may be installed" + + +def test_the_script_uses_no_sudo_and_no_early_reader() -> None: + code = "\n".join(line.split("#", 1)[0] for line in SCRIPT.read_text().splitlines()) + assert not re.search(r"(? None: + calls = [] + for path in [*(REPO_ROOT / ".github").rglob("*.yml"), *(REPO_ROOT / ".github").rglob("*.yaml")]: + calls += re.findall(r"install-tool\.sh\"?[ \t]+(\S+)", path.read_text()) + assert calls, "no workflow calls install-tool.sh" + assert set(calls) <= set(TOOLS), calls diff --git a/tests/unit/test_job_toolchain.py b/tests/unit/test_job_toolchain.py new file mode 100644 index 00000000..33d91fff --- /dev/null +++ b/tests/unit/test_job_toolchain.py @@ -0,0 +1,279 @@ +"""Lifecycle jobs install every tool they use beyond the job image (#884). + +A Conductor tenant fork runs the lifecycle workflows on a Forgejo runner +whose job image is `node:22-bookworm`. Probed on 2026-09-17: it runs as +root and has curl, wget, git, ssh, ssh-keygen, python3, unzip, tar, node +and npm, but no sudo, jq, aws, gh, or pip. The first real run failed on +`sudo: command not found`; jq, the AWS CLI and PyYAML were next in line. + +The rule: a lifecycle job may only use, beyond that baseline, what an +earlier step of the same job installed. Local composite actions and the +scripts a step calls are followed, so a tool used inside +`log-to-d1.sh` counts where that script runs. + +`sudo` and `apt-get` are never allowed: the job image has no sudo, and +GitHub's runner is not root. + +`gh` is used only by `scripts/repo-secret.sh`, on github.com; on a forge +the script uses curl. The workflow `update-from-upstream.yml` refuses to +run anywhere but GitHub and is therefore not a lifecycle job here. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +LIFECYCLE = [ + "initial-setup.yaml", + "setup-control-plane.yaml", + "spin-up.yml", + "teardown.yml", + "destroy-all.yml", + "spin-up-snapshot.yml", + "teardown-snapshot.yml", +] + +# Tools outside the job image's baseline, and what makes them available. +# "python-yaml" stands for `import yaml` under the job's python3. +WATCHED = ("jq", "aws", "cloudflared", "tofu", "uv", "uvx", "gh", "yq", "pip", "pip3") +FORBIDDEN = ("sudo", "apt-get", "apt") + +# Uses that are fine without an install, with the reason. +EXEMPT = { + ("scripts/repo-secret.sh", "gh"): "used only on github.com; a forge takes the curl branch", +} + +INSTALLERS = {"install-tool.sh", "install-opentofu.sh"} + +_SCRIPT_REF = re.compile(r"(?:\.github/scripts|(? dict[str, Any]: + data = yaml.safe_load(path.read_text()) + assert isinstance(data, dict), path + return data + + +def _code(text: str) -> str: + return "\n".join(line.split("#", 1)[0] for line in text.splitlines()) + + +def _commands(text: str, names: tuple[str, ...]) -> set[str]: + code = _code(text) + return { + name + for name in names + if re.search(rf"(? set[tuple[str, str]]: + """(source, tool) pairs a referenced script needs.""" + path = REPO_ROOT / rel + # The installers name the tools they install; that is not a use. + if not path.is_file() or path.name in INSTALLERS: + return set() + text = path.read_text() + found = {(rel, t) for t in _commands(text, WATCHED + FORBIDDEN)} + if rel.endswith(".py") and re.search(r"^\s*import yaml", text, re.M): + found.add((rel, "python-yaml")) + if rel.endswith(".sh") and re.search(r"^\s*import yaml", text, re.M): + found.add((rel, "python-yaml")) + return found + + +def _provides(step: dict[str, Any]) -> set[str]: + run = _code(str(step.get("run", ""))) + uses = str(step.get("uses", "")) + out: set[str] = set() + if "install-opentofu.sh" in run: + out.add("tofu") + for tool in re.findall(r"install-tool\.sh\"?[ \t]+(\w+)", run): + out.add({"awscli": "aws"}.get(tool, tool)) + if uses.startswith("astral-sh/setup-uv"): + out |= {"uv", "uvx"} + if "uv sync" in run and ".venv/bin" in run and "GITHUB_PATH" in run: + out.add("python-yaml") + return out + + +# SSH to the server goes through Cloudflare Access: ~/.ssh/config names +# cloudflared as the ProxyCommand (nexus_deploy.setup.configure_ssh). So a +# step needs cloudflared when it runs `ssh … nexus`, or a nexus_deploy +# command that opens that connection. +_SSH_VIA_ACCESS = re.compile( + r"(? set[tuple[str, str]]: + run = str(step.get("run", "")) + found = {("step", t) for t in _commands(run, WATCHED + FORBIDDEN)} + if _SSH_VIA_ACCESS.search(_code(run)): + found.add(("ssh over Access", "cloudflared")) + for rel in set(_SCRIPT_REF.findall(_code(run))): + found |= _script_uses(rel) + return found + + +def _expanded_steps(steps: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for step in steps: + uses = str(step.get("uses", "")) + if uses.startswith("./"): + base = REPO_ROOT / uses + action = base / "action.yml" if (base / "action.yml").exists() else base / "action.yaml" + out.extend(_expanded_steps((_load(action).get("runs") or {}).get("steps") or [])) + else: + out.append(step) + return out + + +def _jobs() -> list[tuple[str, str, list[dict[str, Any]]]]: + jobs = [] + for name in LIFECYCLE: + for job_name, job in (_load(WORKFLOWS / name).get("jobs") or {}).items(): + if "steps" in job: + jobs.append((name, job_name, _expanded_steps(job["steps"]))) + return jobs + + +def _is_installer_line(line: str) -> bool: + return any(name in line for name in INSTALLERS) + + +def _violations(steps: list[dict[str, Any]]) -> list[str]: + """Walk each step line by line, so a tool used before the line that + installs it is caught within one step too. An installer line only + provides; its arguments are not uses.""" + available: set[str] = set() + problems = [] + for index, step in enumerate(steps): + label = step.get("name") or step.get("uses") or f"step {index}" + run = str(step.get("run", "")) + # A step-level provider (setup-uv) is available to its own step. + available |= _provides({"uses": step.get("uses", "")}) + for line in run.splitlines(): + if not _is_installer_line(line): + for source, tool in sorted(_uses({"run": line})): + if (source, tool) in EXEMPT: + continue + if tool in FORBIDDEN: + problems.append(f"{label}: {tool} ({source}) is never available") + elif tool not in available: + problems.append(f"{label}: {tool} ({source}) used before it is installed") + available |= _provides({"run": line}) + if ".venv/bin" in line and "GITHUB_PATH" in line and "uv sync" in run: + available.add("python-yaml") + return problems + + +@pytest.mark.parametrize( + ("workflow", "job", "steps"), _jobs(), ids=lambda v: v if isinstance(v, str) else "" +) +def test_lifecycle_job_installs_what_it_uses( + workflow: str, job: str, steps: list[dict[str, Any]] +) -> None: + problems = _violations(steps) + assert not problems, f"{workflow}:{job}\n " + "\n ".join(problems) + + +def test_the_rule_sees_the_tools_it_is_about() -> None: + """Guards the guard: if the detection stopped matching, every job + would pass on nothing.""" + seen: dict[str, set[str]] = {} + for workflow, _job, steps in _jobs(): + for step in steps: + seen.setdefault(workflow, set()).update(tool for _, tool in _uses(step)) + assert {"jq", "aws", "python-yaml", "tofu"} <= seen["setup-control-plane.yaml"] + assert {"jq", "cloudflared", "python-yaml", "uv"} <= seen["spin-up.yml"] + assert {"jq", "aws", "python-yaml"} <= seen["destroy-all.yml"] + + +@pytest.mark.parametrize( + ("steps", "expected"), + [ + ([{"run": "sudo mv x /usr/local/bin/"}], ["sudo"]), + ([{"run": "echo '{}' | jq ."}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh jq"}, {"run": "jq . f"}], []), + ([{"run": "jq . f"}, {"run": "bash .github/scripts/install-tool.sh jq"}], ["jq"]), + ([{"run": "python3 .github/scripts/generate-services-tfvars.py"}], ["python-yaml"]), + ([{"run": ".github/scripts/log-to-d1.sh info x"}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh awscli"}, {"run": "aws s3 ls"}], []), + ([{"run": "echo 'use jq-style filters' # jq in a comment"}], []), + ([{"run": "jq . f\nbash .github/scripts/install-tool.sh jq"}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh jq\njq . f"}], []), + ( + [ + { + "run": "python3 .github/scripts/generate-services-tfvars.py\n" + "uv sync --frozen\n" + 'echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"' + } + ], + ["python-yaml"], + ), + ([{"run": "uv run python -m nexus_deploy s3-snapshot"}], ["cloudflared"]), + ([{"run": "ssh -o BatchMode=yes nexus true"}], ["cloudflared"]), + ( + [ + {"uses": "astral-sh/setup-uv@v3"}, + {"run": "uv run python -m nexus_deploy snapshot-prune"}, + ], + [], + ), + ([{"run": "ssh-keygen -t ed25519 -C nexus-stack"}], []), + ([{"run": "ssh-keygen -f ~/.ssh/id -C github-actions@nexus-stack"}], []), + ([{"run": 'ssh nexus "docker ps"'}], ["cloudflared"]), + ], +) +def test_the_rule_itself(steps: list[dict[str, Any]], expected: list[str]) -> None: + problems = _violations(steps) + for tool in expected: + assert any(f": {tool} (" in p for p in problems), (tool, problems) + if not expected: + assert not problems, problems + + +def test_the_ci_job_uses_the_runners_job_image() -> None: + """job-image-toolchain.yaml proves the installs in the runner's image; + that only holds while both name the same one.""" + config = yaml.safe_load( + (REPO_ROOT / "stacks" / "forgejo-runner" / "runner-config.yml").read_text() + ) + images = {label.split("docker://", 1)[1] for label in config["runner"]["labels"]} + assert len(images) == 1, images + ci = _load(WORKFLOWS / "job-image-toolchain.yaml") + assert ci["jobs"]["toolchain"]["container"]["image"] == images.pop() + # A container job defaults to `sh` on GitHub; the Forgejo runner uses + # bash, and `set -o pipefail` needs it (the first run failed on that). + assert ci["jobs"]["toolchain"]["defaults"]["run"]["shell"] == "bash" + + +def test_the_ci_job_installs_what_the_lifecycle_jobs_install() -> None: + installed = set() + for _workflow, _job, steps in _jobs(): + for step in steps: + installed |= _provides(step) + ci_steps = _load(WORKFLOWS / "job-image-toolchain.yaml")["jobs"]["toolchain"]["steps"] + ci_installed = set().union(*(_provides(s) for s in ci_steps)) + assert installed <= ci_installed, installed - ci_installed + + +def test_python_tests_run_when_the_guarded_files_change() -> None: + """These tests guard workflows, actions and scripts. If CI skipped + them for a pull request that touches only those, the guard would not + run for exactly the changes it exists for.""" + wf: dict[Any, Any] = _load(WORKFLOWS / "python-tests.yml") + triggers = wf["on"] if "on" in wf else wf[True] # PyYAML reads `on` as True + for event in ("pull_request", "push"): + paths = set(triggers[event]["paths"]) + assert {".github/workflows/**", ".github/actions/**", ".github/scripts/**"} <= paths, event