From 72e5f945f5c7b8f72e8cb6402955c818887c73ce Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 1 Oct 2026 11:45:21 +0200 Subject: [PATCH] Bound the time each package install of the catalog's provisioning takes Every apt-get call of the provision-meos action and of the OpenAPI workflow runs through .github/actions/provision-meos/apt-get.sh, which bounds the command's wall time with timeout, 600 seconds an attempt, retries it, three attempts, and fails the step once every attempt is spent, with apt's own status or 124 for a timeout. The download and dpkg lock timeouts it passes bound one request each; the timeout bounds the command, which a mirror serving at a crawl keeps alive through every per-request limit. A composite action step takes no timeout-minutes, so the bound sits in the one script each call runs. Why. provision-meos is the step every consumer's CI runs before any of its own: a stalled mirror holds the job until GitHub's six-hour limit and every lane waits behind it. Measured. Two fork runs, 36731927090 and 36736954866, spent 11 and 27 minutes in their first apt-get on a slow mirror, against about 4 minutes upstream on the same commit. The helper, run against a stub apt-get, returns at once on success, fails with status 100 after three attempts of a failing command, and fails with 124 after three bounded attempts of a hanging one. Witness. This pull request's own CI runs every changed call: the pytest workflow provisions with build-libmeos, which takes the six calls of the action, and the OpenAPI workflow takes its two. --- .github/actions/provision-meos/action.yml | 12 +++--- .github/actions/provision-meos/apt-get.sh | 46 +++++++++++++++++++++++ .github/workflows/openapi-validate.yml | 4 +- 3 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 .github/actions/provision-meos/apt-get.sh diff --git a/.github/actions/provision-meos/action.yml b/.github/actions/provision-meos/action.yml index 2faa2cf..bfd3e8e 100644 --- a/.github/actions/provision-meos/action.yml +++ b/.github/actions/provision-meos/action.yml @@ -63,8 +63,8 @@ runs: - name: Install dev headers for the libclang sysroot shell: bash run: | - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update -qq - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y --no-install-recommends \ + bash "$GITHUB_ACTION_PATH/apt-get.sh" update -qq + bash "$GITHUB_ACTION_PATH/apt-get.sh" install -y --no-install-recommends \ clang libclang-dev \ libjson-c-dev libgsl-dev libproj-dev libgeos-dev \ postgresql-server-dev-16 @@ -103,14 +103,14 @@ runs: shell: bash run: | sudo service postgresql stop || true - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --purge remove postgresql* -y || true + bash "$GITHUB_ACTION_PATH/apt-get.sh" --purge remove postgresql* -y || true sudo rm -rf /var/lib/postgresql/ /etc/postgresql/ /var/log/postgresql/ || true - name: Add PostgreSQL APT repository (PGDG) if: inputs.build-libmeos == 'true' shell: bash run: | - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y curl ca-certificates gnupg + bash "$GITHUB_ACTION_PATH/apt-get.sh" install -y curl ca-certificates gnupg curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ \ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' @@ -119,8 +119,8 @@ runs: if: inputs.build-libmeos == 'true' shell: bash run: | - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update - sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y \ + bash "$GITHUB_ACTION_PATH/apt-get.sh" update + bash "$GITHUB_ACTION_PATH/apt-get.sh" install -y \ libgeos-dev libproj-dev libjson-c-dev libgsl-dev \ libh3-dev libgdal-dev libxml2-dev \ autoconf automake libtool pkg-config \ diff --git a/.github/actions/provision-meos/apt-get.sh b/.github/actions/provision-meos/apt-get.sh new file mode 100644 index 0000000..1ba5bad --- /dev/null +++ b/.github/actions/provision-meos/apt-get.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# apt-get.sh — run one `sudo apt-get` command with a bound on its wall time and a retry, so a +# stalled or unreachable package mirror fails the step in minutes rather than holding the job +# until GitHub's six-hour limit. Every apt-get call of the provision-meos action and of this +# repository's workflows goes through it; a composite action step accepts no `timeout-minutes`, +# so the bound lives here, in the one place each consumer runs. +# +# The per-request options bound one download (`Acquire::*::Timeout`) and the wait for the dpkg +# lock (`DPkg::Lock::Timeout`), not the command: a mirror serving at a crawl passes every +# per-request timeout and still takes as long as it likes. `timeout` bounds the command itself. +# An attempt that exceeds its bound or fails is retried, and the step fails once every attempt is +# spent; a runner whose mirror is slow but serving completes within the bound, which is set well +# above a normal install (four minutes for the whole step). +# +# Usage: +# apt-get.sh e.g. apt-get.sh update -qq +# +# Environment: +# APT_TIMEOUT_UPDATE seconds an `update` attempt may take (default 600) +# APT_TIMEOUT seconds any other attempt may take (default 600) +# APT_ATTEMPTS attempts before giving up (default 3) +set -uo pipefail + +attempts="${APT_ATTEMPTS:-3}" +if [[ "${1:-}" == "update" ]]; then + bound="${APT_TIMEOUT_UPDATE:-600}" +else + bound="${APT_TIMEOUT:-600}" +fi + +for ((i = 1; i <= attempts; i++)); do + timeout --kill-after=30 "$bound" sudo apt-get \ + -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 \ + -o DPkg::Lock::Timeout=300 "$@" + status=$? + if [[ $status -eq 0 ]]; then + exit 0 + fi + if [[ $status -eq 124 || $status -eq 137 ]]; then + echo "::warning::apt-get $1 exceeded ${bound}s (attempt $i of $attempts)" >&2 + else + echo "::warning::apt-get $1 failed with status $status (attempt $i of $attempts)" >&2 + fi +done +echo "::error::apt-get $* did not complete in $attempts attempts of at most ${bound}s" >&2 +exit "$status" diff --git a/.github/workflows/openapi-validate.yml b/.github/workflows/openapi-validate.yml index 29ac76d..e31637b 100644 --- a/.github/workflows/openapi-validate.yml +++ b/.github/workflows/openapi-validate.yml @@ -39,8 +39,8 @@ jobs: # two regenerate paths produce byte-identical catalogs. - name: Install dev headers for libclang sysroot run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends \ + bash .github/actions/provision-meos/apt-get.sh update -qq + bash .github/actions/provision-meos/apt-get.sh install -y --no-install-recommends \ clang libclang-dev \ libjson-c-dev libgsl-dev libproj-dev libgeos-dev \ postgresql-server-dev-16