fix(ci): Install every lifecycle tool the job image may lack

.github/actions/nexus-bootstrap/action.yml

@@ -78,16 +78,19 @@ runs:
     - name: Sync nexus_deploy runtime deps
       # --no-dev: ruff/mypy/pytest are not needed at deploy time.
       shell: bash
-      run: uv sync --frozen --no-dev
+      run: |
+        uv sync --frozen --no-dev
+        # The project environment has PyYAML, which the tfvars and D1 sync
+        # scripts import with `python3`. First on PATH, it serves them on
+        # any job image, not only on GitHub's (#884).
+        echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
 
     - name: Install cloudflared
       # Needed wherever SSH goes over Cloudflare Access, i.e. anything
       # calling nexus_deploy.setup.configure_ssh.
+      # Pinned, and without sudo: a Forgejo job image has none (#884).
       shell: bash
-      run: |
-        curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared
-        chmod +x cloudflared
-        sudo mv cloudflared /usr/local/bin/
+      run: bash "$GITHUB_WORKSPACE/.github/scripts/install-tool.sh" cloudflared
 
     - name: Install SSH key
       shell: bash

.github/scripts/install-tool.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# =============================================================================
+# install-tool.sh — install a pinned tool that a job image may not have
+# =============================================================================
+# The lifecycle workflows were written for GitHub's `ubuntu-latest`, which
+# ships jq, the AWS CLI and sudo. A Conductor tenant fork runs them on a
+# Forgejo runner whose job image is `node:22-bookworm`, which has none of
+# these (#884). Each job therefore installs what it needs with this script,
+# the way install-opentofu.sh installs OpenTofu: a pinned version, a pinned
+# checksum, a directory under $RUNNER_TEMP, and no sudo. It works the same on
+# both runners.
+#
+# Usage: install-tool.sh <jq|cloudflared|awscli>
+#
+# Installs into $RUNNER_TEMP/tools/<tool>-<version>-<arch>/ and, when
+# GITHUB_PATH is set, puts that directory on PATH for the following steps.
+#
+# Where the checksums come from (checked 2026-09-17, amd64 and arm64):
+#   jq 1.8.2           the release's sha256sum.txt, equal to the asset digest
+#                      GitHub records for the release, and to the download
+#   cloudflared 2026.9.1
+#                      the checksums in the release notes, equal to GitHub's
+#                      asset digest, and to the download
+#   AWS CLI 2.36.47    the zip's PGP signature verified against the key the
+#                      AWS CLI install guide publishes (fingerprint FB5D B77F
+#                      D5C1 18B8 0511 ADA8 A631 0ACC 4672 475C) -> "Good
+#                      signature"; the same signature on the other
+#                      architecture's zip -> "BAD signature"
+#
+# To move a tool to another version: repeat that check, then replace its
+# version and both checksums below. Replace, do not add.
+# =============================================================================
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+  echo "❌ usage: install-tool.sh <jq|cloudflared|awscli>" >&2
+  exit 1
+fi
+TOOL="$1"
+
+if [ "$(uname -s)" != "Linux" ]; then
+  echo "❌ install-tool.sh supports Linux runners only (got $(uname -s))." >&2
+  exit 1
+fi
+
+case "$(uname -m)" in
+  x86_64 | amd64) ARCH=amd64 ;;
+  aarch64 | arm64) ARCH=arm64 ;;
+  *)
+    echo "❌ No $TOOL build pinned for architecture $(uname -m)." >&2
+    exit 1
+    ;;
+esac
+
+# The pins. VERSION, the download URL, its checksum per architecture, the
+# command the tool installs, and the text its version output must contain.
+case "$TOOL/$ARCH" in
+  jq/amd64 | jq/arm64)
+    VERSION="1.8.2"
+    FILE="jq-linux-$ARCH"
+    URL="https://github.com/jqlang/jq/releases/download/jq-$VERSION/$FILE"
+    case "$ARCH" in
+      amd64) SHA256="b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" ;;
+      arm64) SHA256="8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309" ;;
+    esac
+    COMMAND="jq"
+    EXPECT="jq-$VERSION"
+    ;;
+  cloudflared/amd64 | cloudflared/arm64)
+    VERSION="2026.9.1"
+    FILE="cloudflared-linux-$ARCH"
+    URL="https://github.com/cloudflare/cloudflared/releases/download/$VERSION/$FILE"
+    case "$ARCH" in
+      amd64) SHA256="03f1f25d1cc93b9ad6c60569d44060bc4f17ed97075760ed8cfca4b12dcd68cc" ;;
+      arm64) SHA256="3d97437c71848bd8df68041e12436b484a661d95073ea1937f01a845ce88faa3" ;;
+    esac
+    COMMAND="cloudflared"
+    EXPECT="cloudflared version $VERSION "
+    ;;
+  awscli/amd64 | awscli/arm64)
+    VERSION="2.36.47"
+    case "$ARCH" in
+      amd64) FILE="awscli-exe-linux-x86_64-$VERSION.zip"
+             SHA256="44ab93f1c6e90e0a34dfe5da6b798d99f2f6b41825e97022b3d409d93a65c615" ;;
+      arm64) FILE="awscli-exe-linux-aarch64-$VERSION.zip"
+             SHA256="118ef53b6f5c652471fd299af01c058901c1a53db69c6b71a7bf3ebd93590969" ;;
+    esac
+    URL="https://awscli.amazonaws.com/$FILE"
+    COMMAND="aws"
+    EXPECT="aws-cli/$VERSION "
+    ;;
+  *)
+    echo "❌ Unknown tool '$TOOL' (expected jq, cloudflared or awscli)." >&2
+    exit 1
+    ;;
+esac
+
+DEST="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/tools/$TOOL-$VERSION-$ARCH"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+if ! curl -fsSL --retry 3 --retry-delay 2 -o "$WORK/$FILE" "$URL"; then
+  echo "❌ Could not download $URL" >&2
+  exit 1
+fi
+
+# `sha256sum` prints "<hash>  <file>"; the expansion keeps only the hash
+# without a pipe (#883).
+if ! SUM=$(sha256sum "$WORK/$FILE"); then
+  echo "❌ Could not compute the checksum of $FILE." >&2
+  exit 1
+fi
+ACTUAL=${SUM%% *}
+if [ "$ACTUAL" != "$SHA256" ]; then
+  echo "❌ Checksum mismatch for $FILE — refusing to install it." >&2
+  echo "   expected $SHA256" >&2
+  echo "   got      $ACTUAL" >&2
+  exit 1
+fi
+
+mkdir -p "$DEST"
+case "$TOOL" in
+  awscli)
+    # The official installer, without root: it takes an install directory
+    # and a bin directory, both under $DEST. unzip is in both job images;
+    # Python's zipfile is the fallback, and python3 is in both as well.
+    mkdir -p "$WORK/extract"
+    if command -v unzip >/dev/null; then
+      unzip -q "$WORK/$FILE" -d "$WORK/extract"
+    else
+      python3 -m zipfile -e "$WORK/$FILE" "$WORK/extract"
+      chmod -R u+x "$WORK/extract/aws"
+    fi
+    "$WORK/extract/aws/install" --install-dir "$DEST/aws-cli" --bin-dir "$DEST/bin" --update >/dev/null
+    BIN_DIR="$DEST/bin"
+    ;;
+  *)
+    install -m 0755 "$WORK/$FILE" "$DEST/$COMMAND"
+    BIN_DIR="$DEST"
+    ;;
+esac
+
+# Read back what was installed. Whole output first, no pipe (#883).
+if ! INSTALLED=$("$BIN_DIR/$COMMAND" --version 2>&1); then
+  echo "❌ $BIN_DIR/$COMMAND --version failed: ${INSTALLED:-(no output)}" >&2
+  exit 1
+fi
+INSTALLED=${INSTALLED%%$'\n'*}
+case "$INSTALLED" in
+  "$EXPECT"*) ;;
+  *)
+    echo "❌ Installed $COMMAND reports '$INSTALLED', expected it to start with '$EXPECT'." >&2
+    exit 1
+    ;;
+esac
+
+if [ -n "${GITHUB_PATH:-}" ]; then
+  echo "$BIN_DIR" >> "$GITHUB_PATH"
+fi
+echo "✅ $INSTALLED installed at $BIN_DIR"

.github/workflows/destroy-all.yml

@@ -81,6 +81,16 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Install jq
+        # The job image of a Forgejo runner has no jq (#884); GitHub's has an
+        # unpinned one. Pinned here, before the first step that parses JSON.
+        run: bash .github/scripts/install-tool.sh jq
+
+      - name: Install AWS CLI
+        # Deletes the R2 buckets when delete_data=DESTROY. The official v2
+        # installer, pinned and without root (#884).
+        run: bash .github/scripts/install-tool.sh awscli
+
       - name: Cache the npm store
         # `npm ci` and every Wrangler invocation download through
         # ~/.npm, and on a cold runner that is the single largest cost here:
@@ -119,7 +129,12 @@ jobs:
 
       - name: Sync nexus_deploy runtime deps
         # --no-dev: ruff/mypy/pytest are not needed here.
-        run: uv sync --frozen --no-dev
+        run: |
+          uv sync --frozen --no-dev
+          # The project environment has PyYAML, which the tfvars and D1 sync
+          # scripts import with `python3`. First on PATH, it serves them on
+          # any job image, not only on GitHub's (#884).
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
 
       - name: Generate SSH key for OpenTofu
         run: |

.github/workflows/job-image-toolchain.yaml

@@ -0,0 +1,106 @@
+name: "CI: Job image toolchain"
+
+# =============================================================================
+# The lifecycle toolchain, installed inside the Forgejo runner's job image.
+# =============================================================================
+# A Conductor tenant fork runs the lifecycle workflows on a Forgejo runner
+# whose job image is node:22-bookworm (stacks/forgejo-runner/runner-config.yml).
+# That image has no sudo, jq or AWS CLI (#884), so every lifecycle job
+# installs them itself. This job runs those installs in the same image on
+# GitHub, so a change that breaks them fails here, on the pull request, and
+# not in a tenant's first deploy.
+#
+# It installs; it does not deploy. Whether a whole lifecycle run gets
+# through is only shown by one, on a fork.
+# =============================================================================
+
+on:
+  pull_request:
+    paths:
+      - ".github/scripts/install-tool.sh"
+      - ".github/scripts/install-opentofu.sh"
+      - ".github/actions/**"
+      - ".github/workflows/job-image-toolchain.yaml"
+      - "stacks/forgejo-runner/runner-config.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+  workflow_dispatch:
+
+concurrency:
+  group: job-image-toolchain-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  toolchain:
+    name: Toolchain in node:22-bookworm
+    runs-on: ubuntu-latest
+    # Must match the image the runner maps its labels to;
+    # tests/unit/test_job_toolchain.py keeps the two in step.
+    container:
+      image: node:22-bookworm
+    # A container job on GitHub defaults to `sh`; the Forgejo runner runs
+    # these steps with bash, and so does this job.
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout
+        # Pinned as #652 asks for added `uses:` lines. The same commit is on
+        # data.forgejo.org, which a Forgejo runner resolves actions from.
+        uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+        with:
+          persist-credentials: false
+
+      - name: What the image lacks
+        # Informational: the reason the steps below exist.
+        run: |
+          for tool in sudo jq aws cloudflared tofu uv; do
+            if command -v "$tool" >/dev/null; then
+              echo "  $tool: present in the image"
+            else
+              echo "  $tool: missing from the image"
+            fi
+          done
+          echo "  running as uid $(id -u)"
+
+      - name: Install jq
+        run: bash .github/scripts/install-tool.sh jq
+
+      - name: Install cloudflared
+        run: bash .github/scripts/install-tool.sh cloudflared
+
+      - name: Install AWS CLI
+        run: bash .github/scripts/install-tool.sh awscli
+
+      - name: Install OpenTofu
+        run: bash .github/scripts/install-opentofu.sh
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4
+        with:
+          version: "0.11.7"
+          enable-cache: true
+
+      - name: Sync nexus_deploy runtime deps
+        run: |
+          uv sync --frozen --no-dev
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
+
+      - name: Every tool answers from PATH
+        run: |
+          set -euo pipefail
+          jq --version
+          cloudflared --version
+          aws --version
+          tofu version
+          uv --version
+          python3 -c 'import yaml; print("PyYAML", yaml.__version__)'
+          # The script the lifecycle workflows run with python3 and PyYAML.
+          # Rendering the real services.yaml, not only importing the module.
+          export ENABLED_SERVICES="forgejo,grafana" FIREWALL_RULES=""
+          mkdir -p tofu/stack && : > tofu/stack/config.tfvars
+          python3 .github/scripts/generate-services-tfvars.py
+          grep -q '^services = {' tofu/stack/config.tfvars

.github/workflows/python-tests.yml

@@ -30,6 +30,10 @@
 
 name: "CI: Python Tests"
 
+# The tests also guard files outside src/: the lifecycle workflows and
+# actions (test_job_toolchain.py, test_install_opentofu.py) and the scripts
+# under .github/scripts/. A pull request that changes only those must run
+# them too.
 on:
   pull_request:
     paths:
@@ -40,7 +44,9 @@ on:
       - "pyproject.toml"
       - "uv.lock"
       - ".python-version"
-      - ".github/workflows/python-tests.yml"
+      - ".github/workflows/**"
+      - ".github/actions/**"
+      - ".github/scripts/**"
       - "scripts/*.sh"
   push:
     branches: [main]
@@ -52,7 +58,9 @@ on:
       - "pyproject.toml"
       - "uv.lock"
       - ".python-version"
-      - ".github/workflows/python-tests.yml"
+      - ".github/workflows/**"
+      - ".github/actions/**"
+      - ".github/scripts/**"
       - "scripts/*.sh"
 
 # Cancel previous runs of this workflow on the same ref. The CI is fast