From b7985ba575bc4a2ab361ada6aaa7d7239d83c892 Mon Sep 17 00:00:00 2001 From: Stefan Koch Date: Thu, 17 Sep 2026 17:02:31 +0200 Subject: [PATCH 1/4] fix(ci): Install every lifecycle tool the job image may lack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Conductor tenant fork got past checkout and the OpenTofu install and failed at the next step on the Forgejo runner: `sudo: command not found` (#884). The job image is node:22-bookworm. It runs as root and has no sudo, jq, AWS CLI or pip. The lifecycle workflows were written for GitHub's ubuntu-latest, which has all of them. Chosen option: the workflows install what they need, as #874 did for OpenTofu. The image stays small and unchanged. - New .github/scripts/install-tool.sh . It installs a pinned version, checks it against a pinned checksum (amd64 and arm64), installs into $RUNNER_TEMP without root, reads the version back without a pipe (#883), and adds the directory to GITHUB_PATH. - jq 1.8.2: sha256sum.txt, GitHub's asset digest and the download agree. - cloudflared 2026.9.1: the release notes, GitHub's asset digest and the download agree. - AWS CLI 2.36.47, the official v2 installer: PGP signature verified against the key from the AWS install guide (fingerprint FB5D B77F ... 4672 475C), "Good signature" for both zips and "BAD signature" when the files are swapped. Installed with --install-dir and --bin-dir under $RUNNER_TEMP. - cloudflared: the four `sudo mv` installs (setup-control-plane, spin-up, teardown, nexus-bootstrap) call the script instead. The version is pinned now, where it used to be `latest`. - jq: installed right after checkout in setup-control-plane, spin-up, teardown, destroy-all and teardown-snapshot, before log-to-d1.sh first needs it. - AWS CLI: installed in setup-control-plane (R2 credential validation, bucket cleanup) and in destroy-all (bucket deletion). - PyYAML: generate-services-tfvars.py, sync-deployed-state.sh and sync-firewall-rules.sh import it with `python3`. After `uv sync --frozen --no-dev`, `.venv/bin` now goes first on GITHUB_PATH, and pyyaml is a runtime dependency. setup-control-plane had no uv and now sets it up. Guard (tests/unit/test_job_toolchain.py): each lifecycle job may use, beyond the image's baseline, only what an earlier step of the same job installed. sudo and apt-get are never allowed. It follows local composite actions and the scripts a step calls, and it counts `ssh … nexus` and `nexus_deploy run-pipeline|s3-snapshot` as needing cloudflared (the ProxyCommand). repo-secret.sh's gh is exempt: it is used only on github.com. The rule has its own parametrised tests. Reverting any install is caught, including cloudflared in nexus-bootstrap, which a first version missed because nexus_deploy uses it only through the SSH config. CI (job-image-toolchain.yaml): on pull requests that touch the installers, the actions, the runner config or the Python lock, a job runs in node:22-bookworm. It installs everything, checks that each tool answers from PATH, and renders the real services.yaml with generate-services-tfvars.py. Tests keep its image equal to the runner's label image and its installs a superset of the lifecycle jobs'. tests/unit/test_install_tool.py: pins well-formed and distinct per architecture; refusals for a wrong argument, a non-Linux runner and an unpinned architecture; a checksum mismatch for every tool and architecture (URL asserted, nothing installed, PATH untouched); no sudo and no early reader in the script. Four reverts, each caught. Docs: forgejo-runner.md lists what the job image contains and what installs the rest. CLAUDE.md has the rule under Development Guidelines. Not verified here: a whole lifecycle run on the Forgejo runner. That needs a release and a Conductor fork dispatch. Closes #884 --- .github/actions/nexus-bootstrap/action.yml | 13 +- .github/scripts/install-tool.sh | 160 ++++++++++++++ .github/workflows/destroy-all.yml | 17 +- .github/workflows/job-image-toolchain.yaml | 99 +++++++++ .github/workflows/setup-control-plane.yaml | 30 ++- .github/workflows/spin-up.yml | 18 +- .github/workflows/teardown-snapshot.yml | 5 + .github/workflows/teardown.yml | 18 +- CLAUDE.md | 12 ++ docs/stacks/forgejo-runner.md | 34 +++ tests/unit/test_install_tool.py | 178 +++++++++++++++ tests/unit/test_job_toolchain.py | 239 +++++++++++++++++++++ 12 files changed, 804 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/install-tool.sh create mode 100644 .github/workflows/job-image-toolchain.yaml create mode 100644 tests/unit/test_install_tool.py create mode 100644 tests/unit/test_job_toolchain.py diff --git a/.github/actions/nexus-bootstrap/action.yml b/.github/actions/nexus-bootstrap/action.yml index 04fa30c2..6493b173 100644 --- a/.github/actions/nexus-bootstrap/action.yml +++ b/.github/actions/nexus-bootstrap/action.yml @@ -78,16 +78,19 @@ runs: - name: Sync nexus_deploy runtime deps # --no-dev: ruff/mypy/pytest are not needed at deploy time. shell: bash - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared # Needed wherever SSH goes over Cloudflare Access, i.e. anything # calling nexus_deploy.setup.configure_ssh. + # Pinned, and without sudo: a Forgejo job image has none (#884). shell: bash - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + run: bash "$GITHUB_WORKSPACE/.github/scripts/install-tool.sh" cloudflared - name: Install SSH key shell: bash diff --git a/.github/scripts/install-tool.sh b/.github/scripts/install-tool.sh new file mode 100644 index 00000000..7e482da5 --- /dev/null +++ b/.github/scripts/install-tool.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# ============================================================================= +# install-tool.sh — install a pinned tool that a job image may not have +# ============================================================================= +# The lifecycle workflows were written for GitHub's `ubuntu-latest`, which +# ships jq, the AWS CLI and sudo. A Conductor tenant fork runs them on a +# Forgejo runner whose job image is `node:22-bookworm`, which has none of +# these (#884). Each job therefore installs what it needs with this script, +# the way install-opentofu.sh installs OpenTofu: a pinned version, a pinned +# checksum, a directory under $RUNNER_TEMP, and no sudo. It works the same on +# both runners. +# +# Usage: install-tool.sh +# +# Installs into $RUNNER_TEMP/tools/--/ and, when +# GITHUB_PATH is set, puts that directory on PATH for the following steps. +# +# Where the checksums come from (checked 2026-09-17, amd64 and arm64): +# jq 1.8.2 the release's sha256sum.txt, equal to the asset digest +# GitHub records for the release, and to the download +# cloudflared 2026.9.1 +# the checksums in the release notes, equal to GitHub's +# asset digest, and to the download +# AWS CLI 2.36.47 the zip's PGP signature verified against the key the +# AWS CLI install guide publishes (fingerprint FB5D B77F +# D5C1 18B8 0511 ADA8 A631 0ACC 4672 475C) -> "Good +# signature"; the same signature on the other +# architecture's zip -> "BAD signature" +# +# To move a tool to another version: repeat that check, then replace its +# version and both checksums below. Replace, do not add. +# ============================================================================= +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "❌ usage: install-tool.sh " >&2 + exit 1 +fi +TOOL="$1" + +if [ "$(uname -s)" != "Linux" ]; then + echo "❌ install-tool.sh supports Linux runners only (got $(uname -s))." >&2 + exit 1 +fi + +case "$(uname -m)" in + x86_64 | amd64) ARCH=amd64 ;; + aarch64 | arm64) ARCH=arm64 ;; + *) + echo "❌ No $TOOL build pinned for architecture $(uname -m)." >&2 + exit 1 + ;; +esac + +# The pins. VERSION, the download URL, its checksum per architecture, the +# command the tool installs, and the text its version output must contain. +case "$TOOL/$ARCH" in + jq/amd64 | jq/arm64) + VERSION="1.8.2" + FILE="jq-linux-$ARCH" + URL="https://github.com/jqlang/jq/releases/download/jq-$VERSION/$FILE" + case "$ARCH" in + amd64) SHA256="b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" ;; + arm64) SHA256="8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309" ;; + esac + COMMAND="jq" + EXPECT="jq-$VERSION" + ;; + cloudflared/amd64 | cloudflared/arm64) + VERSION="2026.9.1" + FILE="cloudflared-linux-$ARCH" + URL="https://github.com/cloudflare/cloudflared/releases/download/$VERSION/$FILE" + case "$ARCH" in + amd64) SHA256="03f1f25d1cc93b9ad6c60569d44060bc4f17ed97075760ed8cfca4b12dcd68cc" ;; + arm64) SHA256="3d97437c71848bd8df68041e12436b484a661d95073ea1937f01a845ce88faa3" ;; + esac + COMMAND="cloudflared" + EXPECT="cloudflared version $VERSION " + ;; + awscli/amd64 | awscli/arm64) + VERSION="2.36.47" + case "$ARCH" in + amd64) FILE="awscli-exe-linux-x86_64-$VERSION.zip" + SHA256="44ab93f1c6e90e0a34dfe5da6b798d99f2f6b41825e97022b3d409d93a65c615" ;; + arm64) FILE="awscli-exe-linux-aarch64-$VERSION.zip" + SHA256="118ef53b6f5c652471fd299af01c058901c1a53db69c6b71a7bf3ebd93590969" ;; + esac + URL="https://awscli.amazonaws.com/$FILE" + COMMAND="aws" + EXPECT="aws-cli/$VERSION " + ;; + *) + echo "❌ Unknown tool '$TOOL' (expected jq, cloudflared or awscli)." >&2 + exit 1 + ;; +esac + +DEST="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/tools/$TOOL-$VERSION-$ARCH" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +if ! curl -fsSL --retry 3 --retry-delay 2 -o "$WORK/$FILE" "$URL"; then + echo "❌ Could not download $URL" >&2 + exit 1 +fi + +# `sha256sum` prints " "; the expansion keeps only the hash +# without a pipe (#883). +if ! SUM=$(sha256sum "$WORK/$FILE"); then + echo "❌ Could not compute the checksum of $FILE." >&2 + exit 1 +fi +ACTUAL=${SUM%% *} +if [ "$ACTUAL" != "$SHA256" ]; then + echo "❌ Checksum mismatch for $FILE — refusing to install it." >&2 + echo " expected $SHA256" >&2 + echo " got $ACTUAL" >&2 + exit 1 +fi + +mkdir -p "$DEST" +case "$TOOL" in + awscli) + # The official installer, without root: it takes an install directory + # and a bin directory, both under $DEST. unzip is in both job images; + # Python's zipfile is the fallback, and python3 is in both as well. + mkdir -p "$WORK/extract" + if command -v unzip >/dev/null; then + unzip -q "$WORK/$FILE" -d "$WORK/extract" + else + python3 -m zipfile -e "$WORK/$FILE" "$WORK/extract" + chmod -R u+x "$WORK/extract/aws" + fi + "$WORK/extract/aws/install" --install-dir "$DEST/aws-cli" --bin-dir "$DEST/bin" --update >/dev/null + BIN_DIR="$DEST/bin" + ;; + *) + install -m 0755 "$WORK/$FILE" "$DEST/$COMMAND" + BIN_DIR="$DEST" + ;; +esac + +# Read back what was installed. Whole output first, no pipe (#883). +if ! INSTALLED=$("$BIN_DIR/$COMMAND" --version 2>&1); then + echo "❌ $BIN_DIR/$COMMAND --version failed: ${INSTALLED:-(no output)}" >&2 + exit 1 +fi +INSTALLED=${INSTALLED%%$'\n'*} +case "$INSTALLED" in + "$EXPECT"*) ;; + *) + echo "❌ Installed $COMMAND reports '$INSTALLED', expected it to start with '$EXPECT'." >&2 + exit 1 + ;; +esac + +if [ -n "${GITHUB_PATH:-}" ]; then + echo "$BIN_DIR" >> "$GITHUB_PATH" +fi +echo "✅ $INSTALLED installed at $BIN_DIR" diff --git a/.github/workflows/destroy-all.yml b/.github/workflows/destroy-all.yml index 5a7fcea2..9ae5fdc2 100644 --- a/.github/workflows/destroy-all.yml +++ b/.github/workflows/destroy-all.yml @@ -81,6 +81,16 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + + - name: Install AWS CLI + # Deletes the R2 buckets when delete_data=DESTROY. The official v2 + # installer, pinned and without root (#884). + run: bash .github/scripts/install-tool.sh awscli + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -119,7 +129,12 @@ jobs: - name: Sync nexus_deploy runtime deps # --no-dev: ruff/mypy/pytest are not needed here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Generate SSH key for OpenTofu run: | diff --git a/.github/workflows/job-image-toolchain.yaml b/.github/workflows/job-image-toolchain.yaml new file mode 100644 index 00000000..54ce7638 --- /dev/null +++ b/.github/workflows/job-image-toolchain.yaml @@ -0,0 +1,99 @@ +name: "CI: Job image toolchain" + +# ============================================================================= +# The lifecycle toolchain, installed inside the Forgejo runner's job image. +# ============================================================================= +# A Conductor tenant fork runs the lifecycle workflows on a Forgejo runner +# whose job image is node:22-bookworm (stacks/forgejo-runner/runner-config.yml). +# That image has no sudo, jq or AWS CLI (#884), so every lifecycle job +# installs them itself. This job runs those installs in the same image on +# GitHub, so a change that breaks them fails here, on the pull request, and +# not in a tenant's first deploy. +# +# It installs; it does not deploy. Whether a whole lifecycle run gets +# through is only shown by one, on a fork. +# ============================================================================= + +on: + pull_request: + paths: + - ".github/scripts/install-tool.sh" + - ".github/scripts/install-opentofu.sh" + - ".github/actions/**" + - ".github/workflows/job-image-toolchain.yaml" + - "stacks/forgejo-runner/runner-config.yml" + - "pyproject.toml" + - "uv.lock" + workflow_dispatch: + +concurrency: + group: job-image-toolchain-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + toolchain: + name: Toolchain in node:22-bookworm + runs-on: ubuntu-latest + # Must match the image the runner maps its labels to; + # tests/unit/test_job_toolchain.py keeps the two in step. + container: + image: node:22-bookworm + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: What the image lacks + # Informational: the reason the steps below exist. + run: | + for tool in sudo jq aws cloudflared tofu uv; do + if command -v "$tool" >/dev/null; then + echo " $tool: present in the image" + else + echo " $tool: missing from the image" + fi + done + echo " running as uid $(id -u)" + + - name: Install jq + run: bash .github/scripts/install-tool.sh jq + + - name: Install cloudflared + run: bash .github/scripts/install-tool.sh cloudflared + + - name: Install AWS CLI + run: bash .github/scripts/install-tool.sh awscli + + - name: Install OpenTofu + run: bash .github/scripts/install-opentofu.sh + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "0.11.7" + enable-cache: true + + - name: Sync nexus_deploy runtime deps + run: | + uv sync --frozen --no-dev + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + + - name: Every tool answers from PATH + run: | + set -euo pipefail + jq --version + cloudflared --version + aws --version + tofu version + uv --version + python3 -c 'import yaml; print("PyYAML", yaml.__version__)' + # The script the lifecycle workflows run with python3 and PyYAML. + # Rendering the real services.yaml, not only importing the module. + export ENABLED_SERVICES="forgejo,grafana" FIREWALL_RULES="" + mkdir -p tofu/stack && : > tofu/stack/config.tfvars + python3 .github/scripts/generate-services-tfvars.py + grep -q '^services = {' tofu/stack/config.tfvars diff --git a/.github/workflows/setup-control-plane.yaml b/.github/workflows/setup-control-plane.yaml index f2e5d4dc..a5f0994b 100644 --- a/.github/workflows/setup-control-plane.yaml +++ b/.github/workflows/setup-control-plane.yaml @@ -150,6 +150,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -184,10 +189,29 @@ jobs: run: bash .github/scripts/install-opentofu.sh - name: Install cloudflared + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared + + - name: Install AWS CLI + # Validates the R2 credentials and cleans up buckets below. The + # official v2 installer, pinned and without root (#884). + run: bash .github/scripts/install-tool.sh awscli + + - name: Install uv + # sync-deployed-state.sh and sync-firewall-rules.sh import PyYAML; + # the project environment provides it (#884). + uses: astral-sh/setup-uv@v3 + with: + version: "0.11.7" + enable-cache: true + + - name: Sync nexus_deploy runtime deps run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Generate SSH key for server access id: ssh_key diff --git a/.github/workflows/spin-up.yml b/.github/workflows/spin-up.yml index d7d247e3..b330bc8f 100644 --- a/.github/workflows/spin-up.yml +++ b/.github/workflows/spin-up.yml @@ -152,6 +152,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -229,13 +234,16 @@ jobs: # match pyproject.toml (which CI's python-tests already # enforces); `--no-dev` skips ruff/mypy/pytest etc. since we # only need runtime here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared - name: Setup SSH key for server access run: | diff --git a/.github/workflows/teardown-snapshot.yml b/.github/workflows/teardown-snapshot.yml index 09c70ffc..d4ca3cc2 100644 --- a/.github/workflows/teardown-snapshot.yml +++ b/.github/workflows/teardown-snapshot.yml @@ -98,6 +98,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml index d406738f..55e5b1fd 100644 --- a/.github/workflows/teardown.yml +++ b/.github/workflows/teardown.yml @@ -47,6 +47,11 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Install jq + # The job image of a Forgejo runner has no jq (#884); GitHub's has an + # unpinned one. Pinned here, before the first step that parses JSON. + run: bash .github/scripts/install-tool.sh jq + - name: Cache the npm store # `npm ci` and every Wrangler invocation download through # ~/.npm, and on a cold runner that is the single largest cost here: @@ -98,16 +103,19 @@ jobs: - name: Sync nexus_deploy runtime deps # Pre-resolve the lockfile-pinned runtime deps. `--no-dev` # skips ruff/mypy/pytest since we only need runtime here. - run: uv sync --frozen --no-dev + run: | + uv sync --frozen --no-dev + # The project environment has PyYAML, which the tfvars and D1 sync + # scripts import with `python3`. First on PATH, it serves them on + # any job image, not only on GitHub's (#884). + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - name: Install cloudflared # Needed by the SSH-over-Cloudflare-Access setup in # `nexus_deploy.pipeline.run_snapshot` (configure_ssh writes # a Host block that uses ProxyCommand=cloudflared). - run: | - curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared - chmod +x cloudflared - sudo mv cloudflared /usr/local/bin/ + # Pinned, and without sudo: a Forgejo job image has none (#884). + run: bash .github/scripts/install-tool.sh cloudflared - name: Install SSH key from GitHub Secrets # Teardown's pre-destroy s3-snapshot step needs SSH into the diff --git a/CLAUDE.md b/CLAUDE.md index 92f34ae0..a211fcf8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -317,6 +317,18 @@ harmless, because the value is only compared against a success code; in a hook that *prints* the code or branches on it with `case`, it is not. The answer usually reveals that your case differs. +### Tools in lifecycle workflows + +The lifecycle workflows also run on a Forgejo runner whose job image is +`node:22-bookworm`: root, no `sudo`, no `jq`, no AWS CLI (#884). So a +lifecycle job installs every tool beyond that image in an earlier step: +OpenTofu with `.github/scripts/install-opentofu.sh`; jq, cloudflared and the +AWS CLI with `.github/scripts/install-tool.sh `; uv with +`astral-sh/setup-uv`. PyYAML comes from the project environment, whose +`.venv/bin` goes onto `PATH` after `uv sync`. Never `sudo`, never `apt-get`. +`tests/unit/test_job_toolchain.py` enforces it, and +`docs/stacks/forgejo-runner.md` lists what the image contains. + ### Service Account Naming Convention All service accounts MUST use the `nexus-` prefix to prevent default username guessing: diff --git a/docs/stacks/forgejo-runner.md b/docs/stacks/forgejo-runner.md index def92827..d496f0c6 100644 --- a/docs/stacks/forgejo-runner.md +++ b/docs/stacks/forgejo-runner.md @@ -146,6 +146,40 @@ That is **not** GitHub's large `ubuntu` runner image. A workflow that assumes Python, Go, Docker CLI or the AWS CLI is preinstalled will need to install them first. +### What the job image contains + +Probed on 2026-09-17 inside the stack's own daemon: + +| Present | Missing | +|---|---| +| `bash`, `curl`, `wget`, `git`, `ssh`, `ssh-keygen`, `python3`, `unzip`, `tar`, `node`, `npm` | `sudo`, `jq`, `aws`, `gh`, `pip` | + +Jobs run as root, so nothing needs `sudo`, and nothing may use it. + +Nexus-Stack's own lifecycle workflows (Initial Setup, Setup Control +Plane, Spin Up, Teardown, Destroy All and the snapshot pair) install what +they need beyond that list themselves, so they run unchanged on GitHub +and on this runner (#884): + +| Tool | Installed by | +|---|---| +| OpenTofu | `.github/scripts/install-opentofu.sh` | +| jq, cloudflared, AWS CLI v2 | `.github/scripts/install-tool.sh `, pinned versions and checksums, into `$RUNNER_TEMP`, without root | +| uv | `astral-sh/setup-uv` | +| PyYAML | the project environment: after `uv sync`, `.venv/bin` goes first on `PATH`, so `python3` finds it | + +Two checks keep it that way: + +- `tests/unit/test_job_toolchain.py` fails when a lifecycle job uses one + of these tools before a step of the same job installed it, or uses + `sudo` or `apt-get` at all. It follows local actions and the scripts a + step calls. +- `.github/workflows/job-image-toolchain.yaml` runs the installs in + `node:22-bookworm` on every pull request that touches them. + +Neither proves that a whole lifecycle run passes on this runner. Only a +real run on a fork shows that. + Other known differences from GitHub Actions: - `permissions:` and `continue-on-error:` on a job are ignored. diff --git a/tests/unit/test_install_tool.py b/tests/unit/test_install_tool.py new file mode 100644 index 00000000..a6d743da --- /dev/null +++ b/tests/unit/test_install_tool.py @@ -0,0 +1,178 @@ +"""install-tool.sh installs pinned jq, cloudflared and the AWS CLI (#884). + +The refusals are run under bash with `curl` and `uname` replaced. The +success path cannot be faked, because the checksums are pinned to the real +release files. It runs for real in `.github/workflows/job-image-toolchain.yaml`, +inside the Forgejo runner's job image. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / ".github" / "scripts" / "install-tool.sh" +TOOLS = ("jq", "cloudflared", "awscli") + +_FAKE_CURL = """#!/usr/bin/env bash +out="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) out="$2"; shift 2 ;; + http*) echo "$1" >> "$FAKE_LOG"; shift ;; + *) shift ;; + esac +done +printf 'not the real file' > "$out" +""" + +_FAKE_UNAME = """#!/usr/bin/env bash +case "$1" in + -s) echo "$FAKE_UNAME_S" ;; + -m) echo "$FAKE_UNAME_M" ;; + *) echo "$FAKE_UNAME_S" ;; +esac +""" + +# A real sha256, so the mismatch is computed rather than faked; macOS has no +# `sha256sum`, CI does, and both must reach the same comparison. +_SHA256SUM = f"""#!{sys.executable} +import hashlib, sys +for path in sys.argv[1:]: + print(hashlib.sha256(open(path, "rb").read()).hexdigest() + " " + path) +""" + + +def _run( + tmp_path: Path, *args: str, uname_s: str = "Linux", uname_m: str = "x86_64" +) -> tuple[int, str, str, list[str]]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + for name, body in (("curl", _FAKE_CURL), ("uname", _FAKE_UNAME), ("sha256sum", _SHA256SUM)): + (bin_dir / name).write_text(body) + (bin_dir / name).chmod(0o755) + gh_path = tmp_path / "github_path" + gh_path.write_text("") + log = tmp_path / "curl.log" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "FAKE_LOG": str(log), + "FAKE_UNAME_S": uname_s, + "FAKE_UNAME_M": uname_m, + "GITHUB_PATH": str(gh_path), + "RUNNER_TEMP": str(tmp_path / "runner-temp"), + } + done = subprocess.run( + ["bash", str(SCRIPT), *args], capture_output=True, text=True, env=env, check=False + ) + urls = log.read_text().splitlines() if log.exists() else [] + return done.returncode, done.stderr, gh_path.read_text(), urls + + +def _pins() -> dict[tuple[str, str], tuple[str, str]]: + """(tool, arch) -> (version, sha256), read from the script.""" + text = SCRIPT.read_text() + out = {} + for tool in TOOLS: + block = re.search(rf"^ {tool}/amd64 \| {tool}/arm64\)(.*?)^ ;;", text, re.M | re.S) + assert block, tool + body = block.group(1) + version = re.search(r'VERSION="([^"]+)"', body) + assert version, tool + for arch in ("amd64", "arm64"): + sha = re.search(rf'{arch}\).*?SHA256="([0-9a-f]+)"', body, re.S) + assert sha, (tool, arch) + out[(tool, arch)] = (version.group(1), sha.group(1)) + return out + + +def test_every_tool_has_a_version_and_two_well_formed_checksums() -> None: + pins = _pins() + assert len(pins) == len(TOOLS) * 2 + for (tool, arch), (version, sha) in pins.items(): + assert re.fullmatch(r"[0-9]+(\.[0-9]+)+", version), (tool, version) + assert re.fullmatch(r"[0-9a-f]{64}", sha), (tool, arch) + # One checksum per file: amd64 and arm64 must differ. + for tool in TOOLS: + assert pins[(tool, "amd64")][1] != pins[(tool, "arm64")][1], tool + + +@pytest.mark.parametrize("args", [(), ("jq", "cloudflared"), ("yq",), ("../evil",)]) +def test_the_script_refuses_anything_but_one_known_tool( + tmp_path: Path, args: tuple[str, ...] +) -> None: + rc, err, gh_path, urls = _run(tmp_path, *args) + assert rc == 1 + assert "usage" in err or "Unknown tool" in err + assert urls == [] + assert gh_path == "" + + +def test_the_script_refuses_a_non_linux_runner(tmp_path: Path) -> None: + rc, err, gh_path, urls = _run(tmp_path, "jq", uname_s="Darwin") + assert rc == 1 + assert "Linux runners only" in err + assert (gh_path, urls) == ("", []) + + +def test_the_script_refuses_an_unpinned_architecture(tmp_path: Path) -> None: + rc, err, gh_path, urls = _run(tmp_path, "jq", uname_m="riscv64") + assert rc == 1 + assert "No jq build pinned" in err + assert (gh_path, urls) == ("", []) + + +_EXPECTED_URLS = { + ("jq", "amd64"): "https://github.com/jqlang/jq/releases/download/jq-{v}/jq-linux-amd64", + ("jq", "arm64"): "https://github.com/jqlang/jq/releases/download/jq-{v}/jq-linux-arm64", + ( + "cloudflared", + "amd64", + ): "https://github.com/cloudflare/cloudflared/releases/download/{v}/cloudflared-linux-amd64", + ( + "cloudflared", + "arm64", + ): "https://github.com/cloudflare/cloudflared/releases/download/{v}/cloudflared-linux-arm64", + ("awscli", "amd64"): "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{v}.zip", + ("awscli", "arm64"): "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{v}.zip", +} + + +@pytest.mark.parametrize(("tool", "arch"), sorted(_EXPECTED_URLS)) +def test_a_file_that_does_not_match_its_pin_is_not_installed( + tmp_path: Path, tool: str, arch: str +) -> None: + """The refusal that protects anything. The URL is asserted, so the test + is known to have reached the comparison for this tool and architecture.""" + version, sha = _pins()[(tool, arch)] + uname_m = {"amd64": "x86_64", "arm64": "aarch64"}[arch] + rc, err, gh_path, urls = _run(tmp_path, tool, uname_m=uname_m) + assert rc == 1, err + assert "Checksum mismatch" in err + assert urls == [_EXPECTED_URLS[(tool, arch)].format(v=version)] + assert sha in err + assert hashlib.sha256(b"not the real file").hexdigest() in err + assert gh_path == "" + assert not list((tmp_path / "runner-temp").rglob("*")), "nothing may be installed" + + +def test_the_script_uses_no_sudo_and_no_early_reader() -> None: + code = "\n".join(line.split("#", 1)[0] for line in SCRIPT.read_text().splitlines()) + assert not re.search(r"(? None: + calls = [] + for path in [*(REPO_ROOT / ".github").rglob("*.yml"), *(REPO_ROOT / ".github").rglob("*.yaml")]: + calls += re.findall(r"install-tool\.sh\"?[ \t]+(\S+)", path.read_text()) + assert calls, "no workflow calls install-tool.sh" + assert set(calls) <= set(TOOLS), calls diff --git a/tests/unit/test_job_toolchain.py b/tests/unit/test_job_toolchain.py new file mode 100644 index 00000000..414be436 --- /dev/null +++ b/tests/unit/test_job_toolchain.py @@ -0,0 +1,239 @@ +"""Lifecycle jobs install every tool they use beyond the job image (#884). + +A Conductor tenant fork runs the lifecycle workflows on a Forgejo runner +whose job image is `node:22-bookworm`. Probed on 2026-09-17: it runs as +root and has curl, wget, git, ssh, ssh-keygen, python3, unzip, tar, node +and npm, but no sudo, jq, aws, gh, or pip. The first real run failed on +`sudo: command not found`; jq, the AWS CLI and PyYAML were next in line. + +The rule: a lifecycle job may only use, beyond that baseline, what an +earlier step of the same job installed. Local composite actions and the +scripts a step calls are followed, so a tool used inside +`log-to-d1.sh` counts where that script runs. + +`sudo` and `apt-get` are never allowed: the job image has no sudo, and +GitHub's runner is not root. + +`gh` is used only by `scripts/repo-secret.sh`, on github.com; on a forge +the script uses curl. The workflow `update-from-upstream.yml` refuses to +run anywhere but GitHub and is therefore not a lifecycle job here. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +LIFECYCLE = [ + "initial-setup.yaml", + "setup-control-plane.yaml", + "spin-up.yml", + "teardown.yml", + "destroy-all.yml", + "spin-up-snapshot.yml", + "teardown-snapshot.yml", +] + +# Tools outside the job image's baseline, and what makes them available. +# "python-yaml" stands for `import yaml` under the job's python3. +WATCHED = ("jq", "aws", "cloudflared", "tofu", "uv", "uvx", "gh", "yq", "pip", "pip3") +FORBIDDEN = ("sudo", "apt-get", "apt") + +# Uses that are fine without an install, with the reason. +EXEMPT = { + ("scripts/repo-secret.sh", "gh"): "used only on github.com; a forge takes the curl branch", +} + +INSTALLERS = {"install-tool.sh", "install-opentofu.sh"} + +_SCRIPT_REF = re.compile(r"(?:\.github/scripts|(? dict[str, Any]: + data = yaml.safe_load(path.read_text()) + assert isinstance(data, dict), path + return data + + +def _code(text: str) -> str: + return "\n".join(line.split("#", 1)[0] for line in text.splitlines()) + + +def _commands(text: str, names: tuple[str, ...]) -> set[str]: + code = _code(text) + return { + name + for name in names + if re.search(rf"(? set[tuple[str, str]]: + """(source, tool) pairs a referenced script needs.""" + path = REPO_ROOT / rel + # The installers name the tools they install; that is not a use. + if not path.is_file() or path.name in INSTALLERS: + return set() + text = path.read_text() + found = {(rel, t) for t in _commands(text, WATCHED + FORBIDDEN)} + if rel.endswith(".py") and re.search(r"^\s*import yaml", text, re.M): + found.add((rel, "python-yaml")) + if rel.endswith(".sh") and re.search(r"^\s*import yaml", text, re.M): + found.add((rel, "python-yaml")) + return found + + +def _provides(step: dict[str, Any]) -> set[str]: + run = _code(str(step.get("run", ""))) + uses = str(step.get("uses", "")) + out: set[str] = set() + if "install-opentofu.sh" in run: + out.add("tofu") + for tool in re.findall(r"install-tool\.sh\"?[ \t]+(\w+)", run): + out.add({"awscli": "aws"}.get(tool, tool)) + if uses.startswith("astral-sh/setup-uv"): + out |= {"uv", "uvx"} + if "uv sync" in run and ".venv/bin" in run and "GITHUB_PATH" in run: + out.add("python-yaml") + return out + + +# SSH to the server goes through Cloudflare Access: ~/.ssh/config names +# cloudflared as the ProxyCommand (nexus_deploy.setup.configure_ssh). So a +# step needs cloudflared when it runs `ssh … nexus`, or a nexus_deploy +# command that opens that connection. +_SSH_VIA_ACCESS = re.compile( + r"(? set[tuple[str, str]]: + run = str(step.get("run", "")) + found = {("step", t) for t in _commands(run, WATCHED + FORBIDDEN)} + if _SSH_VIA_ACCESS.search(_code(run)): + found.add(("ssh over Access", "cloudflared")) + for rel in set(_SCRIPT_REF.findall(_code(run))): + found |= _script_uses(rel) + return found + + +def _expanded_steps(steps: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for step in steps: + uses = str(step.get("uses", "")) + if uses.startswith("./"): + base = REPO_ROOT / uses + action = base / "action.yml" if (base / "action.yml").exists() else base / "action.yaml" + out.extend(_expanded_steps((_load(action).get("runs") or {}).get("steps") or [])) + else: + out.append(step) + return out + + +def _jobs() -> list[tuple[str, str, list[dict[str, Any]]]]: + jobs = [] + for name in LIFECYCLE: + for job_name, job in (_load(WORKFLOWS / name).get("jobs") or {}).items(): + if "steps" in job: + jobs.append((name, job_name, _expanded_steps(job["steps"]))) + return jobs + + +def _violations(steps: list[dict[str, Any]]) -> list[str]: + available: set[str] = set() + problems = [] + for index, step in enumerate(steps): + available |= _provides(step) + label = step.get("name") or step.get("uses") or f"step {index}" + for source, tool in sorted(_uses(step)): + if (source, tool) in EXEMPT: + continue + if tool in FORBIDDEN: + problems.append(f"{label}: {tool} ({source}) is never available") + elif tool not in available: + problems.append(f"{label}: {tool} ({source}) used before it is installed") + return problems + + +@pytest.mark.parametrize( + ("workflow", "job", "steps"), _jobs(), ids=lambda v: v if isinstance(v, str) else "" +) +def test_lifecycle_job_installs_what_it_uses( + workflow: str, job: str, steps: list[dict[str, Any]] +) -> None: + problems = _violations(steps) + assert not problems, f"{workflow}:{job}\n " + "\n ".join(problems) + + +def test_the_rule_sees_the_tools_it_is_about() -> None: + """Guards the guard: if the detection stopped matching, every job + would pass on nothing.""" + seen: dict[str, set[str]] = {} + for workflow, _job, steps in _jobs(): + for step in steps: + seen.setdefault(workflow, set()).update(tool for _, tool in _uses(step)) + assert {"jq", "aws", "python-yaml", "tofu"} <= seen["setup-control-plane.yaml"] + assert {"jq", "cloudflared", "python-yaml", "uv"} <= seen["spin-up.yml"] + assert {"jq", "aws", "python-yaml"} <= seen["destroy-all.yml"] + + +@pytest.mark.parametrize( + ("steps", "expected"), + [ + ([{"run": "sudo mv x /usr/local/bin/"}], ["sudo"]), + ([{"run": "echo '{}' | jq ."}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh jq"}, {"run": "jq . f"}], []), + ([{"run": "jq . f"}, {"run": "bash .github/scripts/install-tool.sh jq"}], ["jq"]), + ([{"run": "python3 .github/scripts/generate-services-tfvars.py"}], ["python-yaml"]), + ([{"run": ".github/scripts/log-to-d1.sh info x"}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh awscli"}, {"run": "aws s3 ls"}], []), + ([{"run": "echo 'use jq-style filters' # jq in a comment"}], []), + ([{"run": "uv run python -m nexus_deploy s3-snapshot"}], ["cloudflared"]), + ([{"run": "ssh -o BatchMode=yes nexus true"}], ["cloudflared"]), + ( + [ + {"uses": "astral-sh/setup-uv@v3"}, + {"run": "uv run python -m nexus_deploy snapshot-prune"}, + ], + [], + ), + ([{"run": "ssh-keygen -t ed25519 -C nexus-stack"}], []), + ([{"run": "ssh-keygen -f ~/.ssh/id -C github-actions@nexus-stack"}], []), + ([{"run": 'ssh nexus "docker ps"'}], ["cloudflared"]), + ], +) +def test_the_rule_itself(steps: list[dict[str, Any]], expected: list[str]) -> None: + problems = _violations(steps) + for tool in expected: + assert any(f": {tool} (" in p for p in problems), (tool, problems) + if not expected: + assert not problems, problems + + +def test_the_ci_job_uses_the_runners_job_image() -> None: + """job-image-toolchain.yaml proves the installs in the runner's image; + that only holds while both name the same one.""" + config = yaml.safe_load( + (REPO_ROOT / "stacks" / "forgejo-runner" / "runner-config.yml").read_text() + ) + images = {label.split("docker://", 1)[1] for label in config["runner"]["labels"]} + assert len(images) == 1, images + ci = _load(WORKFLOWS / "job-image-toolchain.yaml") + assert ci["jobs"]["toolchain"]["container"]["image"] == images.pop() + + +def test_the_ci_job_installs_what_the_lifecycle_jobs_install() -> None: + installed = set() + for _workflow, _job, steps in _jobs(): + for step in steps: + installed |= _provides(step) + ci_steps = _load(WORKFLOWS / "job-image-toolchain.yaml")["jobs"]["toolchain"]["steps"] + ci_installed = set().union(*(_provides(s) for s in ci_steps)) + assert installed <= ci_installed, installed - ci_installed From f88dd3f5e11e1d6b508d4aef69039830ea062a3d Mon Sep 17 00:00:00 2001 From: Stefan Koch Date: Thu, 17 Sep 2026 17:09:35 +0200 Subject: [PATCH 2/4] test(ci): Check tool order within a step, and address the local review Local CodeRabbit round on 4b222edd: 3 findings, 2 themes. Fixed (minor, test_job_toolchain.py): the guard merged a step's installs before checking its uses, so a step that used jq on one line and installed it on a later one passed. It now walks each step line by line. An installer line only provides; its arguments are not uses. New cases cover both orders in one step and a python3 call before the `uv sync` that provides PyYAML. The six workflow reverts are still caught. Dismissed (major, twice): pin astral-sh/setup-uv and actions/checkout to commit SHAs in setup-control-plane.yaml and job-image-toolchain.yaml. Every other workflow uses the same tags, and pinning actions across the repository is #652, which should decide it once rather than per file. --- tests/unit/test_job_toolchain.py | 42 ++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_job_toolchain.py b/tests/unit/test_job_toolchain.py index 414be436..5162c656 100644 --- a/tests/unit/test_job_toolchain.py +++ b/tests/unit/test_job_toolchain.py @@ -146,19 +146,33 @@ def _jobs() -> list[tuple[str, str, list[dict[str, Any]]]]: return jobs +def _is_installer_line(line: str) -> bool: + return any(name in line for name in INSTALLERS) + + def _violations(steps: list[dict[str, Any]]) -> list[str]: + """Walk each step line by line, so a tool used before the line that + installs it is caught within one step too. An installer line only + provides; its arguments are not uses.""" available: set[str] = set() problems = [] for index, step in enumerate(steps): - available |= _provides(step) label = step.get("name") or step.get("uses") or f"step {index}" - for source, tool in sorted(_uses(step)): - if (source, tool) in EXEMPT: - continue - if tool in FORBIDDEN: - problems.append(f"{label}: {tool} ({source}) is never available") - elif tool not in available: - problems.append(f"{label}: {tool} ({source}) used before it is installed") + run = str(step.get("run", "")) + # A step-level provider (setup-uv) is available to its own step. + available |= _provides({"uses": step.get("uses", "")}) + for line in run.splitlines(): + if not _is_installer_line(line): + for source, tool in sorted(_uses({"run": line})): + if (source, tool) in EXEMPT: + continue + if tool in FORBIDDEN: + problems.append(f"{label}: {tool} ({source}) is never available") + elif tool not in available: + problems.append(f"{label}: {tool} ({source}) used before it is installed") + available |= _provides({"run": line}) + if ".venv/bin" in line and "GITHUB_PATH" in line and "uv sync" in run: + available.add("python-yaml") return problems @@ -195,6 +209,18 @@ def test_the_rule_sees_the_tools_it_is_about() -> None: ([{"run": ".github/scripts/log-to-d1.sh info x"}], ["jq"]), ([{"run": "bash .github/scripts/install-tool.sh awscli"}, {"run": "aws s3 ls"}], []), ([{"run": "echo 'use jq-style filters' # jq in a comment"}], []), + ([{"run": "jq . f\nbash .github/scripts/install-tool.sh jq"}], ["jq"]), + ([{"run": "bash .github/scripts/install-tool.sh jq\njq . f"}], []), + ( + [ + { + "run": "python3 .github/scripts/generate-services-tfvars.py\n" + "uv sync --frozen\n" + 'echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"' + } + ], + ["python-yaml"], + ), ([{"run": "uv run python -m nexus_deploy s3-snapshot"}], ["cloudflared"]), ([{"run": "ssh -o BatchMode=yes nexus true"}], ["cloudflared"]), ( From 68b2436ab819d03e3a3a08991a0cfb01c2131f2f Mon Sep 17 00:00:00 2001 From: Stefan Koch Date: Thu, 17 Sep 2026 19:12:32 +0200 Subject: [PATCH 3/4] fix(ci): Address PR review comments Round 1 on #886, plus the first run of the new CI job. CI (job-image-toolchain, run 35239631508): every install succeeded inside node:22-bookworm, running as uid 0: - jq-1.8.2 - cloudflared 2026.9.1 - aws-cli/2.36.47 - OpenTofu v1.12.6 - uv, and `uv sync` The final check failed with "set: Illegal option -o pipefail": on GitHub, a container job defaults to `sh`. The Forgejo runner runs steps with bash (its error in #884 has bash's "line 4:" format), so the job now sets `defaults.run.shell: bash`. A test pins it. 4038687324 (job-image-toolchain.yaml:46) and 4038687337 (setup-control-plane.yaml:203): fixed. The local round dismissed this by pointing at #652, but .coderabbit.yaml asks for exactly this when a third-party `uses:` line is added or modified. Both added lines are now pinned to a commit: - actions/checkout@fbc6f399 (v5.1.0) - astral-sh/setup-uv@caf0cab7 (v3.2.4, the commit v3 points to) Both commits are also on data.forgejo.org, which the Forgejo runner resolves actions from (fetched by SHA; a made-up SHA is refused). The untouched `@vN` lines elsewhere stay for #652. --- .github/workflows/job-image-toolchain.yaml | 11 +++++++++-- .github/workflows/setup-control-plane.yaml | 5 +++-- tests/unit/test_job_toolchain.py | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/job-image-toolchain.yaml b/.github/workflows/job-image-toolchain.yaml index 54ce7638..b45115cc 100644 --- a/.github/workflows/job-image-toolchain.yaml +++ b/.github/workflows/job-image-toolchain.yaml @@ -41,9 +41,16 @@ jobs: # tests/unit/test_job_toolchain.py keeps the two in step. container: image: node:22-bookworm + # A container job on GitHub defaults to `sh`; the Forgejo runner runs + # these steps with bash, and so does this job. + defaults: + run: + shell: bash steps: - name: Checkout - uses: actions/checkout@v5 + # Pinned as #652 asks for added `uses:` lines. The same commit is on + # data.forgejo.org, which a Forgejo runner resolves actions from. + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false @@ -72,7 +79,7 @@ jobs: run: bash .github/scripts/install-opentofu.sh - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 with: version: "0.11.7" enable-cache: true diff --git a/.github/workflows/setup-control-plane.yaml b/.github/workflows/setup-control-plane.yaml index a5f0994b..e53eac05 100644 --- a/.github/workflows/setup-control-plane.yaml +++ b/.github/workflows/setup-control-plane.yaml @@ -199,8 +199,9 @@ jobs: - name: Install uv # sync-deployed-state.sh and sync-firewall-rules.sh import PyYAML; - # the project environment provides it (#884). - uses: astral-sh/setup-uv@v3 + # the project environment provides it (#884). Pinned as #652 asks for + # added `uses:` lines; the commit is also on data.forgejo.org. + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 with: version: "0.11.7" enable-cache: true diff --git a/tests/unit/test_job_toolchain.py b/tests/unit/test_job_toolchain.py index 5162c656..5302bd34 100644 --- a/tests/unit/test_job_toolchain.py +++ b/tests/unit/test_job_toolchain.py @@ -253,6 +253,9 @@ def test_the_ci_job_uses_the_runners_job_image() -> None: assert len(images) == 1, images ci = _load(WORKFLOWS / "job-image-toolchain.yaml") assert ci["jobs"]["toolchain"]["container"]["image"] == images.pop() + # A container job defaults to `sh` on GitHub; the Forgejo runner uses + # bash, and `set -o pipefail` needs it (the first run failed on that). + assert ci["jobs"]["toolchain"]["defaults"]["run"]["shell"] == "bash" def test_the_ci_job_installs_what_the_lifecycle_jobs_install() -> None: From 8f007a6814468161b4bf42ab22daaabaf5b634fc Mon Sep 17 00:00:00 2001 From: Stefan Koch Date: Thu, 17 Sep 2026 19:26:35 +0200 Subject: [PATCH 4/4] fix(ci): Run the Python tests when workflows, actions or scripts change Round 2 on #886. CodeRabbit, outside the diff (job-image-toolchain.yaml :1-20): a pull request that changes only a lifecycle workflow does not trigger the node:22-bookworm job. Checking that uncovered a larger gap. `CI: Python Tests` did not run for changes under .github/workflows/, .github/actions/ or .github/scripts/, except for its own file. The tests that guard exactly those files therefore never ran on a pull request that touched only them: - test_job_toolchain.py: no job uses a tool before installing it, and none uses sudo; - test_install_opentofu.py; - test_sigpipe_under_pipefail.py and test_install_tool.py. python-tests.yml now also triggers on those three directories, for pull_request and push. A test pins it (mutation-checked). The container job itself keeps its paths. It exercises the installers, which a workflow edit does not change. Whether a workflow uses a tool it does not install is decided by test_job_toolchain.py, which now runs for such an edit. --- .github/workflows/python-tests.yml | 12 ++++++++++-- tests/unit/test_job_toolchain.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 8f8d90ed..4942ebce 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -30,6 +30,10 @@ name: "CI: Python Tests" +# The tests also guard files outside src/: the lifecycle workflows and +# actions (test_job_toolchain.py, test_install_opentofu.py) and the scripts +# under .github/scripts/. A pull request that changes only those must run +# them too. on: pull_request: paths: @@ -40,7 +44,9 @@ on: - "pyproject.toml" - "uv.lock" - ".python-version" - - ".github/workflows/python-tests.yml" + - ".github/workflows/**" + - ".github/actions/**" + - ".github/scripts/**" - "scripts/*.sh" push: branches: [main] @@ -52,7 +58,9 @@ on: - "pyproject.toml" - "uv.lock" - ".python-version" - - ".github/workflows/python-tests.yml" + - ".github/workflows/**" + - ".github/actions/**" + - ".github/scripts/**" - "scripts/*.sh" # Cancel previous runs of this workflow on the same ref. The CI is fast diff --git a/tests/unit/test_job_toolchain.py b/tests/unit/test_job_toolchain.py index 5302bd34..33d91fff 100644 --- a/tests/unit/test_job_toolchain.py +++ b/tests/unit/test_job_toolchain.py @@ -266,3 +266,14 @@ def test_the_ci_job_installs_what_the_lifecycle_jobs_install() -> None: ci_steps = _load(WORKFLOWS / "job-image-toolchain.yaml")["jobs"]["toolchain"]["steps"] ci_installed = set().union(*(_provides(s) for s in ci_steps)) assert installed <= ci_installed, installed - ci_installed + + +def test_python_tests_run_when_the_guarded_files_change() -> None: + """These tests guard workflows, actions and scripts. If CI skipped + them for a pull request that touches only those, the guard would not + run for exactly the changes it exists for.""" + wf: dict[Any, Any] = _load(WORKFLOWS / "python-tests.yml") + triggers = wf["on"] if "on" in wf else wf[True] # PyYAML reads `on` as True + for event in ("pull_request", "push"): + paths = set(triggers[event]["paths"]) + assert {".github/workflows/**", ".github/actions/**", ".github/scripts/**"} <= paths, event