From f84419a6bc91526962b81ee3c861468e6cc9a3ab Mon Sep 17 00:00:00 2001 From: navigator Date: Sat, 3 Oct 2026 12:30:05 +0000 Subject: [PATCH] fix: the first boot's security updates never hold it 95secupdates ran apt-get update and the upgrade with no limit, so a stalled mirror or a hung maintainer script held every hook after it. apt-get update now runs within SEC_UPDATES_UPDATE_TIMEOUT (120 s) and the whole run within SEC_UPDATES_TIMEOUT (900 s), both overridable from the inithooks conf. A run stopped or failed leaves dpkg configured (dpkg --configure -a when dpkg --audit reports anything), records nothing and writes one line to the inithooks log naming cron-apt, the daily job that installs the updates instead. No network still exits 0; a stopped or failed upgrade exits 1, which run logs before going on to the next hook. --- COVERAGE.md | 12 ++ README.rst | 6 + debian/changelog | 20 ++++ firstboot.d/95secupdates | 126 +++++++++++++++++++-- tests/test-secupdates.bats | 226 ++++++++++++++++++++++++++++++++++++- 5 files changed, 376 insertions(+), 14 deletions(-) diff --git a/COVERAGE.md b/COVERAGE.md index a09880f..f29c8aa 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -4,6 +4,18 @@ Measured on 2026-09-24 against upstream master (33c43b8), following the project decision 0003 (90 percent floor per repository, 95 percent for every file our changes touch). +## Branch fix/secupdates-never-hold-boot: shell 99.71 (2026-10-03) + +`firstboot.d/95secupdates` 110/111 (the line not run is still the TurnKey +Hub status call). `tests/test-secupdates.bats` gains 12 tests: an apt-get +update and an upgrade that hang, stopped within their limits and timed, the +run's limit applied to apt-get update, dpkg configured after a stopped +upgrade, an upgrade that fails, one that succeeds, the one line naming +cron-apt (offline too) or turnkey-install-security-updates without it, a +limit that is not a number of seconds, and three through the real `run`: +a hung or failed upgrade and no network leave the hook after it running. +379 bats; shell total 99.71. + ## Branch fix/headless-first-boot: shell 99.69, Python 99 (2026-10-03) A first boot nobody can answer, the hosts entry and the certificate's diff --git a/README.rst b/README.rst index c69421e..04b90c7 100644 --- a/README.rst +++ b/README.rst @@ -533,6 +533,12 @@ Notes: were. It was called security.updates, which is still read: the old name renders the same conf and is reported once, with the new name, so a description written before the rename keeps booting. + - The updates never hold the first boot. 95secupdates gives apt-get + update SEC_UPDATES_UPDATE_TIMEOUT seconds (120) and the whole run + SEC_UPDATES_TIMEOUT seconds (900), both read from the inithooks conf. + A run stopped at its limit, or one that failed, leaves dpkg + configured, says so in one line of the inithooks log and the boot goes + on; cron-apt installs the updates at its daily run. - A secret is a mapping with exactly one of file or generate. A secret file is read as bytes, one trailing newline is stripped, and it must diff --git a/debian/changelog b/debian/changelog index 2fcb994..242f8c7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,23 @@ +inithooks (2.3.6+keel23) trixie; urgency=medium + + * The first boot's security updates never hold it. 95secupdates ran + apt-get update and the upgrade with no limit, so a stalled mirror or a + package that hung in its maintainer script held every hook after it. + apt-get update now runs within SEC_UPDATES_UPDATE_TIMEOUT (120 s) and + the whole run within SEC_UPDATES_TIMEOUT (900 s), both read from the + inithooks conf; timeout signals apt's whole process group, with + SIGKILL 30 s after SIGTERM. A run stopped or failed leaves dpkg + configured (dpkg --configure -a, itself within 300 s, when dpkg + --audit reports anything), records nothing and writes one line to the + inithooks log, the hook's log and the journal, naming cron-apt, whose + daily install action (common's conf/turnkey.d/cronapt) installs the + updates instead, or turnkey-install-security-updates when that action + is missing. An upgrade stopped or failed exits 1, which run logs and + goes on to the next hook; no network, an apt-get update that fails or + one stopped at its limit exit 0, as before. + + -- Marcos Mendez Sat, 03 Oct 2026 20:00:00 +0000 + inithooks (2.3.6+keel22) trixie; urgency=medium * A first boot nobody can answer asks nothing. The Web 19.0-3 booted diff --git a/firstboot.d/95secupdates b/firstboot.d/95secupdates index 0ce330d..af2e182 100755 --- a/firstboot.d/95secupdates +++ b/firstboot.d/95secupdates @@ -6,6 +6,13 @@ # installed as FORCE installs them, what the preseed of a headless build # says (README.rst), and 99reboot reboots for a new kernel as it does # after FORCE. +# +# The updates never hold the boot. apt-get update and the upgrade run within +# SEC_UPDATES_TIMEOUT seconds in all (15 minutes), apt-get update within +# SEC_UPDATES_UPDATE_TIMEOUT (2 minutes), both read from the inithooks conf. +# An apt stopped at its limit, or one that failed, leaves dpkg configured +# (dpkg --configure -a), and the boot goes on with one line in the inithooks +# log: the daily job, cron-apt, installs the updates instead. # every '| tee' below must carry the exit status of what it logs set -o pipefail @@ -33,6 +40,11 @@ SEC_UPDATES_LOG="${SEC_UPDATES_LOG:-/var/log/inithooks/secupdates.log}" # common's conf/bootstrap_apt. Every other source is left out on purpose, # so the first boot installs security fixes and nothing else. SEC_UPDATES_SOURCES="${SEC_UPDATES_SOURCES:-/etc/apt/sources.list.d/security.sources}" +# cron-apt's install action, written by common's conf/turnkey.d/cronapt: +# the daily job that installs from the same source what the first boot did +# not. The images ship cron-apt, not unattended-upgrades. +SEC_UPDATES_CRONAPT="${SEC_UPDATES_CRONAPT:-/etc/cron-apt/action.d/5-install}" +INITHOOKS_LOGFILE="${INITHOOKS_LOGFILE:-/var/log/inithooks.log}" # journal ARGS: logger, which may not fail the hook under -e: journald can # be down (15regen-sslcert died of it on 2026-10-03), and the hook's own @@ -41,6 +53,29 @@ journal() { logger -t inithooks "$@" 2>/dev/null || true } +# seconds NAME DEFAULT: the limit NAME holds, in whole seconds above 0, or +# DEFAULT when it holds anything else, said +seconds() { + local name=$1 default=$2 + local value=${!name} + if [[ ! "$value" =~ ^[0-9]+$ ]] || (( 10#$value == 0 )); then + journal -p warn \ + "[95secupdates] $name=$value is not a number of seconds, $default used" + value=$default + fi + echo $((10#$value)) +} + +SEC_UPDATES_TIMEOUT="${SEC_UPDATES_TIMEOUT:-900}" +SEC_UPDATES_UPDATE_TIMEOUT="${SEC_UPDATES_UPDATE_TIMEOUT:-120}" +SEC_UPDATES_TIMEOUT=$(seconds SEC_UPDATES_TIMEOUT 900) +SEC_UPDATES_UPDATE_TIMEOUT=$(seconds SEC_UPDATES_UPDATE_TIMEOUT 120) +# What an apt stopped at its limit gets to exit on SIGTERM before SIGKILL, +# and what dpkg --configure -a gets afterwards: the boot is not held by +# the repair either. +SEC_UPDATES_KILL_AFTER="${SEC_UPDATES_KILL_AFTER:-30}" +SEC_UPDATES_REPAIR_TIMEOUT="${SEC_UPDATES_REPAIR_TIMEOUT:-300}" + record() { if ! { mkdir -p "$(dirname "$SEC_UPDATES_RECORD")" \ && echo "$1" > "$SEC_UPDATES_RECORD"; } 2>/dev/null; then @@ -59,16 +94,74 @@ modules_and_boot() { ls -la /lib/modules /boot 2>/dev/null || true } -# offline MESSAGE: no update can be fetched. Said in the system log and in -# the hook's own log, and the first boot goes on: a machine with no network -# yet is not a broken one. Nothing is recorded, since nothing was installed. -offline() { - local msg="[95secupdates] $1; security updates not installed, run turnkey-install-security-updates once the network is up" +# not_installed WHY: the updates were not installed, for WHY. One line, in +# the system log, the hook's own log and the inithooks log, naming what +# installs them instead. +not_installed() { + local after + if [[ -e "$SEC_UPDATES_CRONAPT" ]]; then + after="the boot goes on and cron-apt installs them at its daily run" + else + after="the boot goes on, no daily job installs them: run turnkey-install-security-updates" + fi + local msg="[95secupdates] security updates not installed: $1; $after" journal -p warn "$msg" echo "WARNING: $msg" >> "$LOGFILE" + { echo "WARN: $msg" >> "$INITHOOKS_LOGFILE"; } 2>/dev/null || true +} + +# offline WHY: no update can be fetched, and the first boot goes on: a +# machine with no network yet is not a broken one. Nothing is recorded, +# since nothing was installed. +offline() { + not_installed "$1" exit 0 } +# stopped WHAT STATUS: WHAT, an install step, failed or was stopped at the +# run's limit. dpkg is left configured, the hook fails (run logs it and +# goes on to the next hook), and nothing is recorded. +stopped() { + local what=$1 status=$2 + if timed_out "$status"; then + not_installed "$what did not finish in $SEC_UPDATES_TIMEOUT s, stopped" + else + not_installed "$what failed (exit $status, see $LOGFILE)" + fi + configure_dpkg + exit 1 +} + +# timed_out STATUS: STATUS is timeout's, for a command it stopped +timed_out() { + [[ "$1" -eq 124 ]] || [[ "$1" -eq 137 ]] +} + +# left: the seconds the run has left, at least 1 +left() { + local n=$((deadline - SECONDS)) + echo $((n > 0 ? n : 1)) +} + +# bounded LIMIT CMD...: CMD within LIMIT seconds, its output in the hook's +# log, its status that of CMD (124 or 137 when stopped). timeout signals +# the whole process group, so dpkg and the maintainer scripts apt runs +# stop with it. Nothing is read from the console. +bounded() { + local limit=$1 + shift + timeout --kill-after="$SEC_UPDATES_KILL_AFTER" "$limit" "$@" \ + &1 | tee -a "$LOGFILE" +} + +# configure_dpkg: dpkg --configure -a when a stopped or failed run left +# packages unpacked or half configured +configure_dpkg() { + [[ -n "$(dpkg --audit 2>/dev/null)" ]] || return 0 + bounded "$SEC_UPDATES_REPAIR_TIMEOUT" dpkg --force-confdef \ + --force-confold --configure -a || true +} + # the InRelease of the first stanza of the security source security_release_url() { local uri suite @@ -98,6 +191,8 @@ install_updates() { if ! curl -fsS --max-time 15 -o /dev/null "$release" 2>/dev/null; then offline "cannot reach ${release%/dists/*}" fi + # the run's limit counts from here, the security archive reachable + deadline=$((SECONDS + SEC_UPDATES_TIMEOUT)) OLDMD5=$(modules_and_boot | md5sum) if [[ -n "$(dpkg --audit 2>/dev/null)" ]]; then msg="[95secupdates] dpkg in an inconsistent state (see $LOGFILE)" @@ -105,18 +200,27 @@ install_updates() { echo "WARNING: $msg" >> "$LOGFILE" dpkg --audit 2>&1 | tee -a "$LOGFILE" fi - DEBIAN_FRONTEND=noninteractive dpkg --force-confdef --force-confold \ - --configure -a 2>&1 | tee -a "$LOGFILE" - if ! apt-get update 2>&1 | tee -a "$LOGFILE"; then + export DEBIAN_FRONTEND=noninteractive + local status=0 + bounded "$(left)" dpkg --force-confdef --force-confold --configure -a \ + || stopped "dpkg --configure -a" $? + local limit=$SEC_UPDATES_UPDATE_TIMEOUT + if (( $(left) < limit )); then + limit=$(left) + fi + bounded "$limit" apt-get update || status=$? + if timed_out "$status"; then + offline "apt-get update did not finish in $limit s" + elif [[ "$status" -ne 0 ]]; then offline "apt-get update failed (see $LOGFILE)" fi - DEBIAN_FRONTEND=noninteractive apt-get autoclean -y - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y \ + bounded "$(left)" apt-get autoclean -y || stopped "apt-get autoclean" $? + bounded "$(left)" apt-get dist-upgrade -y \ -o APT::Get::Show-Upgraded=true \ -o Dir::Etc::sourceparts=/dev/null \ -o Dir::Etc::sourcelist="$SEC_UPDATES_SOURCES" \ -o DPkg::Options::=--force-confdef \ - -o DPkg::Options::=--force-confold | tee -a "$LOGFILE" + -o DPkg::Options::=--force-confold || stopped "the upgrade" $? NEWMD5=$(modules_and_boot | md5sum) if [[ "$NEWMD5" != "$OLDMD5" ]]; then diff --git a/tests/test-secupdates.bats b/tests/test-secupdates.bats index c6a014c..bc03c5e 100644 --- a/tests/test-secupdates.bats +++ b/tests/test-secupdates.bats @@ -7,13 +7,18 @@ # maintainer's screenshot 034). # # apt-get, dpkg, logger and ls are stubs; secupdates-ask.py is a stub -# under INITHOOKS_PATH that exits with the status a test sets. +# under INITHOOKS_PATH that exits with the status a test sets. An apt-get +# that hangs sleeps for good: the hook must stop it within its limits, and +# the tests that use one run under an outer timeout so that a hook that +# does not stop it fails instead of hanging the suite. bats_require_minimum_version 1.5.0 load helpers REPO=$BATS_TEST_DIRNAME/.. +# the real sleep, for an apt-get that hangs where sleep is a stub +REAL_SLEEP=$(command -v sleep) setup() { setup_stubs @@ -27,6 +32,11 @@ esac' # curl answers the reachability check: exit CURL_STATUS stub curl 'exit "${CURL_STATUS:-0}"' stub dpkg 'if [[ "$1" == --audit ]]; then echo "${DPKG_AUDIT-}"; fi' + # the daily job that installs what the first boot did not: cron-apt's + # install action (common's conf/turnkey.d/cronapt) + export SEC_UPDATES_CRONAPT=$BATS_TEST_TMPDIR/cron-apt/5-install + mkdir -p "$(dirname "$SEC_UPDATES_CRONAPT")" + touch "$SEC_UPDATES_CRONAPT" # the module and boot listing before and after the upgrade: the same # unless LS_CHANGES is set, when the second call differs; LS_STATUS is # its exit status (2 where /boot does not exist, as in a container) @@ -58,7 +68,8 @@ exit "${LS_STATUS:-0}"' export INITHOOKS_UNATTENDED=no export INITHOOKS_LOGFILE=$BATS_TEST_TMPDIR/inithooks.log unset SEC_UPDATES DPKG_AUDIT LS_CHANGES LS_STATUS ASK_STATUS \ - UPDATE_STATUS UPGRADE_STATUS CURL_STATUS + UPDATE_STATUS UPGRADE_STATUS CURL_STATUS \ + SEC_UPDATES_TIMEOUT SEC_UPDATES_UPDATE_TIMEOUT } # the value the dist-upgrade call passed for one apt option @@ -240,7 +251,7 @@ apt_option() { [ -z "$(calls apt-get)" ] local said="cannot reach http://security.debian.org/debian-security" [[ "$(calls logger)" == *"$said"* ]] - [[ "$(calls logger)" == *turnkey-install-security-updates* ]] + [[ "$(calls logger)" == *"cron-apt installs them"* ]] [[ "$(cat "$SEC_UPDATES_LOG")" == *"$said"* ]] } @@ -345,3 +356,212 @@ esac' [ "$status" -eq 0 ] } + +# ------------------------------------------- the update never holds the boot +# +# The maintainer's decision (2026-10-03): an unattended first boot installs +# the security updates, and they never hold it. apt-get update and the +# upgrade are bounded, a run stopped or failed leaves dpkg configured, and +# the boot goes on with one line in the inithooks log naming the daily job +# that installs them instead. + +# hang WHEN: apt-get sleeps for good on the call matching WHEN, and +# answers every other one as the default stub does +hang() { + stub apt-get 'case "$*" in +'"$1"') echo "apt-get $1 started"; "'"$REAL_SLEEP"'" 300 ;; +*dist-upgrade*) echo "0 upgraded"; exit "${UPGRADE_STATUS:-0}" ;; +esac' +} + +# run_bounded: the hook, under an outer limit far above its own, timed +run_bounded() { + local started=$SECONDS + run timeout 60 "$REPO/firstboot.d/95secupdates" + took=$((SECONDS - started)) +} + +# the lines the hook left in the inithooks log +said() { + grep -F '[95secupdates]' "$INITHOOKS_LOGFILE" 2>/dev/null || true +} + +@test "the limits default to 15 minutes for the run and 2 for apt-get update" { + run grep -c 'SEC_UPDATES_TIMEOUT:-900}' "$REPO/firstboot.d/95secupdates" + [ "$output" = 1 ] + run grep -c 'SEC_UPDATES_UPDATE_TIMEOUT:-120}' "$REPO/firstboot.d/95secupdates" + [ "$output" = 1 ] +} + +@test "an apt-get update that hangs is stopped at its limit, and the boot goes on" { + { + echo "export SEC_UPDATES=FORCE" + echo "SEC_UPDATES_UPDATE_TIMEOUT=1" + } > "$INITHOOKS_CONF" + hang 'update*' + + run_bounded + + [ "$status" -eq 0 ] + (( took < 10 )) + [[ "$(calls apt-get)" != *dist-upgrade* ]] + [ ! -e "$SEC_UPDATES_RECORD" ] + [ "$(said | wc -l)" -eq 1 ] + [[ "$(said)" == *"apt-get update did not finish in 1 s"*cron-apt* ]] +} + +@test "an upgrade that hangs is stopped at the run's limit, and dpkg is configured" { + { + echo "export SEC_UPDATES=FORCE" + echo "SEC_UPDATES_TIMEOUT=2" + } > "$INITHOOKS_CONF" + hang '*dist-upgrade*' + # what a dpkg killed while unpacking leaves + export DPKG_AUDIT="libfoo is half configured" + + run_bounded + + [ "$status" -eq 1 ] + (( took < 10 )) + [ ! -e "$SEC_UPDATES_RECORD" ] + # once before the update, and once after the upgrade was stopped + [ "$(calls dpkg | grep -c -- '--configure -a')" -eq 2 ] + [ "$(said | wc -l)" -eq 1 ] + [[ "$(said)" == *"the upgrade did not finish in 2 s"*cron-apt* ]] + [ ! -x "$INITHOOKS_PATH/firstboot.d/99reboot" ] +} + +@test "the run's limit holds for apt-get update too" { + # the run may be shorter than the update's own limit + { + echo "export SEC_UPDATES=FORCE" + echo "SEC_UPDATES_TIMEOUT=1" + } > "$INITHOOKS_CONF" + hang 'update*' + + run_bounded + + [ "$status" -eq 0 ] + (( took < 10 )) + [[ "$(said)" == *"apt-get update did not finish in 1 s"* ]] +} + +@test "an upgrade that fails is said in one line naming the daily job" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + export UPGRADE_STATUS=100 + + run_bounded + + [ "$status" -eq 1 ] + [ ! -e "$SEC_UPDATES_RECORD" ] + # dpkg was consistent: nothing to configure after the failure + [ "$(calls dpkg | grep -c -- '--configure -a')" -eq 1 ] + [ "$(said | wc -l)" -eq 1 ] + [[ "$(said)" == *"the upgrade failed (exit 100"*cron-apt* ]] + [[ "$(calls logger)" == *"the upgrade failed (exit 100"* ]] +} + +@test "an upgrade that succeeds leaves no warning in the inithooks log" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + + run_bounded + + [ "$status" -eq 0 ] + [ "$(cat "$SEC_UPDATES_RECORD")" = "force" ] + [ -z "$(said)" ] +} + +@test "offline, the one line names the daily job too" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + export CURL_STATUS=7 + + run_bounded + + [ "$status" -eq 0 ] + [ "$(said | wc -l)" -eq 1 ] + [[ "$(said)" == *"cannot reach"*cron-apt* ]] +} + +@test "without cron-apt's install action the line says how to install them" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + export UPGRADE_STATUS=100 + rm "$SEC_UPDATES_CRONAPT" + + run_bounded + + [ "$status" -eq 1 ] + [[ "$(said)" != *cron-apt* ]] + [[ "$(said)" == *"no daily job installs them"*turnkey-install-security-updates* ]] +} + +@test "a limit that is not a number of seconds is said, and the default used" { + { + echo "export SEC_UPDATES=FORCE" + echo "SEC_UPDATES_TIMEOUT=15m" + } > "$INITHOOKS_CONF" + + run_bounded + + [ "$status" -eq 0 ] + [ "$(cat "$SEC_UPDATES_RECORD")" = "force" ] + [[ "$(calls logger)" == *"SEC_UPDATES_TIMEOUT=15m is not a number of seconds, 900 used"* ]] +} + +# ------------------------------------------------ the hooks after it still run + +# run_firstboot: the real run over 95secupdates and a hook after it +run_firstboot() { + ln -s "$REPO/firstboot.d/95secupdates" "$INITHOOKS_PATH/firstboot.d/95secupdates" + printf '#!/bin/bash\necho ran > %q\n' "$BATS_TEST_TMPDIR/next" \ + > "$INITHOOKS_PATH/firstboot.d/96next" + chmod +x "$INITHOOKS_PATH/firstboot.d/96next" + stub systemctl 'echo running' + stub confconsole + stub sleep + export INITHOOKS_LOCK=$BATS_TEST_TMPDIR/inithooks.lock + export INITHOOKS_COMPLETE=$BATS_TEST_TMPDIR/inithooks-complete + { + echo "INITHOOKS_LOGFILE=$INITHOOKS_LOGFILE" + echo "RUN_FIRSTBOOT=true" + echo "REDIRECT_OUTPUT=false" + } >> "$INITHOOKS_DEFAULT" + local started=$SECONDS + run timeout 60 "$REPO/run" + took=$((SECONDS - started)) +} + +@test "an upgrade that hangs does not stop the hooks after it" { + export INITHOOKS_UNATTENDED="the console has no size" + echo "SEC_UPDATES_TIMEOUT=2" > "$INITHOOKS_CONF" + hang '*dist-upgrade*' + + run_firstboot + + [ "$status" -eq 0 ] + (( took < 15 )) + [ "$(cat "$BATS_TEST_TMPDIR/next")" = ran ] + grep -qF '[95secupdates] failed - exit code 1' "$INITHOOKS_LOGFILE" + grep -qF '[96next] successfully completed' "$INITHOOKS_LOGFILE" +} + +@test "an upgrade that fails does not stop the hooks after it" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + export UPGRADE_STATUS=100 + + run_firstboot + + [ "$status" -eq 0 ] + [ "$(cat "$BATS_TEST_TMPDIR/next")" = ran ] + grep -qF '[96next] successfully completed' "$INITHOOKS_LOGFILE" +} + +@test "offline, the hook succeeds and the hooks after it run" { + echo "export SEC_UPDATES=FORCE" > "$INITHOOKS_CONF" + export CURL_STATUS=7 + + run_firstboot + + [ "$status" -eq 0 ] + [ "$(cat "$BATS_TEST_TMPDIR/next")" = ran ] + grep -qF '[95secupdates] successfully completed' "$INITHOOKS_LOGFILE" +}