The build verifies the staging archive instead of reading it unverified

COVERAGE.md

@@ -11,12 +11,13 @@ LXC as the acceptance test of a recipe, docs/org-plan.md section 1).
 | overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch |
 | overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter |
 | tests/lib/boot-test-lib.sh | tests/boot-test.bats (44 tests) | 100 percent (154/154) under kcov | argument parsing, address discovery, deadlines, the container marks, the database, module, Webmin and diff verdicts |
+| bin/keel-archive-check | tests/archive-check.bats (25 tests) | 100 percent (52/52) under kcov | the build time check: the archive copy in the build tree is the live archive, the source entry names the keyring through signed-by, nothing says trusted=yes, and the signature on the copied InRelease verifies against the staging key (tracker#7) |
 | conf.d/main | the build | integration only | build time script, 0004 pragmatic limits |
 | tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root |
 | overlay ... firstboot.d/35pgsqlpass | common | not this repository | the hook that sets the password belongs to the `common` fork and is used, not rewritten |
 
-Total over the three measured shell files: **100 percent (210/210)**,
-65 bats tests. `tests/coverage.sh` fails below `COVERAGE_THRESHOLD`, which
+Total over the four measured shell files: **100 percent (262/262)**,
+90 bats tests. `tests/coverage.sh` fails below `COVERAGE_THRESHOLD`, which
 the workflow sets to 100, the measured number. It is only ever raised
 (decision 0006).
 
@@ -25,6 +26,7 @@ the workflow sets to 100, the measured number. It is only ever raised
      100.00  41/41  postgresql.sh
      100.00  15/15  36pgsqlverify
      100.00  154/154  boot-test-lib.sh
+     100.00  52/52  keel-archive-check
 
 ## What the hook tests cover
 

Makefile

@@ -23,21 +23,65 @@ WEBMIN_FW_TCP_INCOMING = 22 5432 12320 12321
 include $(FAB_PATH)/common/mk/turnkey.mk
 
 # The project's own packages (inithooks, confconsole, keel) come from the
-# build host's APT repository during the build only. The repository is
-# copied into the bootstrap and listed as a [trusted=yes] file source,
-# because the staging distribution is unsigned; conf.d/main removes both
-# from the image and leaves the future apt.keellinux.org entry in place,
-# disabled. Same block as keel-nodebb, which is where the pattern is
-# maintained.
+# build host's APT repository during the build only. The repository is copied
+# into the bootstrap and the build verifies it there, the way an appliance
+# verifies the release archive (tracker#7): the public half of the staging key
+# is installed as a keyring, the source entry names it through signed-by,
+# nothing in the tree says trusted=yes, and apt runs with --error-on=any, so a
+# signature that cannot be checked fails the build instead of warning about it
+# and carrying on. conf.d/main removes the copy of the archive, the source
+# entry and the keyring from the image and leaves the future apt.keellinux.org
+# entry in place, disabled.
+#
+# None of the three build time files is for an installed appliance, because
+# the staging key signs whatever the build host produced. The removelist at
+# common/removelists-final/turnkey takes all three out of the image as well,
+# whatever a recipe does. Same block as keel-nodebb, which is where the pattern is maintained.
 KEEL_APT_REPO ?= /srv/keel-apt/repo
 KEEL_APT_DIST ?= trixie-staging
+# Beside the repository rather than inside it: bin/publish of keel-linux/apt
+# installs the public half of whichever key it signed a distribution with
+# here, so the key a build verifies with cannot drift from the key the archive
+# was signed with.
+KEEL_APT_KEYRING ?= /srv/keel-apt/keys/keel-staging-keyring.asc
+# Where that key goes in the build tree, and which key has to be in it: the
+# staging signing subkey (handbook decision 0011). A keyring is only a promise
+# until the key inside it is named, so bin/keel-archive-check fails the build
+# when the keyring it finds holds some other key.
+KEEL_APT_KEYRING_PATH ?= /etc/apt/keyrings/keel-staging-keyring.asc
+KEEL_APT_KEY ?= 8CFD1A4841448B2227341CEB202CACBD0E97090A
+KEEL_STAGING_LIST ?= /etc/apt/sources.list.d/keel-staging.list
+KEEL_ARCHIVE_CHECK = KEEL_ARCHIVE_KEY=$(KEEL_APT_KEY) \
+	KEEL_ARCHIVE_KEYRING=$(KEEL_APT_KEYRING_PATH) \
+	KEEL_ARCHIVE_LIST=$(KEEL_STAGING_LIST) \
+	$(CURDIR)/bin/keel-archive-check $(KEEL_APT_REPO)
 
+# The copy is made fresh and then proved: bin/keel-archive-check compares the
+# copied package index with the live one, verifies the signature on the copied
+# InRelease against the keyring, refuses any trusted=yes, and stops the build
+# when one of them is wrong.
 define _keel_bootstrap/post
 
-	mkdir -p $O/bootstrap/srv/keel-apt/repo;
+	mkdir -p $O/bootstrap/srv/keel-apt/repo $O/bootstrap$(dir $(KEEL_APT_KEYRING_PATH));
 	rm -rf $O/bootstrap/srv/keel-apt/repo/dists $O/bootstrap/srv/keel-apt/repo/pool;
 	cp -a $(KEEL_APT_REPO)/dists $(KEEL_APT_REPO)/pool $O/bootstrap/srv/keel-apt/repo/;
-	echo "deb [trusted=yes] file:///srv/keel-apt/repo $(KEEL_APT_DIST) main" > $O/bootstrap/etc/apt/sources.list.d/keel-staging.list;
-	fab-chroot $O/bootstrap "apt-get update";
+	install -m 644 $(KEEL_APT_KEYRING) $O/bootstrap$(KEEL_APT_KEYRING_PATH);
+	echo "deb [signed-by=$(KEEL_APT_KEYRING_PATH)] file:///srv/keel-apt/repo $(KEEL_APT_DIST) main" > $O/bootstrap$(KEEL_STAGING_LIST);
+	$(KEEL_ARCHIVE_CHECK) $O/bootstrap $(KEEL_APT_DIST) $(FAB_ARCH) bootstrap;
+	fab-chroot $O/bootstrap "apt-get update --error-on=any";
 endef
 bootstrap/post += $(_keel_bootstrap/post)
+
+# bootstrap is a stamped target, so a second build of the same product reuses
+# the copy the first one made and a check in bootstrap/post does not run at
+# all. On 2026-09-26 "make clean" failed on a busy deck, the stamps survived,
+# and the rebuild installed the packages the archive had held that morning
+# without a word. So the tree that is about to be configured is checked on
+# every build, whether or not this build made the bootstrap. That check
+# verifies the signature with gpgv too, which is what proves this tree at a
+# step where no apt-get update runs.
+define _keel_root.patched/pre
+
+	$(KEEL_ARCHIVE_CHECK) $O/root.patched $(KEEL_APT_DIST) $(FAB_ARCH) root.patched;
+endef
+root.patched/pre += $(_keel_root.patched/pre)

bin/keel-archive-check

@@ -0,0 +1,142 @@
+#!/bin/bash
+# The build reads the project's own packages from a copy of the project's APT
+# archive inside the build tree. This checks two things about that copy and
+# stops the build when either one is wrong:
+#
+#   1. the copy is the archive as it is right now, byte for byte;
+#   2. the build's apt can verify the copy, and is not being told not to.
+#
+# Why (1) is checked: fab stamps the bootstrap target, so a second build of
+# the same product reuses the bootstrap the first one made, copy of the
+# archive and all. On 2026-09-26 "make clean" failed on a busy deck, the
+# stamps survived, and the rebuild installed the packages the archive had held
+# that morning. The build said nothing, because every step of it succeeded.
+#
+# Why (2) is checked (tracker#7): the staging distribution was given its own
+# signing key and the build was never given the public half, while the source
+# entry said [trusted=yes]. So apt printed
+#
+#   W: OpenPGP signature verification failed: file:/srv/keel-apt/repo
+#      trixie-staging InRelease: Missing key 8CFD...090A
+#
+# and installed the project's packages unverified, which is a warning nobody
+# fails on. Now the keyring, the key in it, the signed-by option and the
+# absence of any trusted=yes are all checked here, and the copied InRelease is
+# verified with gpgv against that keyring before a package is installed from
+# it. gpgv rather than apt's word for it, so the tree that is about to be
+# configured is proved even at a step where no apt-get update runs.
+#
+#   keel-archive-check SOURCE_REPO TREE DIST ARCH [LABEL]
+#
+# SOURCE_REPO  the live archive on the build host, e.g. /srv/keel-apt/repo
+# TREE         the build tree that carries the copy under srv/keel-apt/repo
+# DIST         the distribution, e.g. trixie-staging
+# ARCH         the Debian architecture, e.g. amd64
+# LABEL        name of the build step, for the messages (default: TREE)
+#
+# Read from the environment, so the Makefile block stays one line per step:
+#
+# KEEL_ARCHIVE_KEY      fingerprint of the key that must sign the archive,
+#                       with no spaces. Required: a keyring is only a
+#                       promise until the key inside it is named.
+# KEEL_ARCHIVE_KEYRING  where in TREE the keyring is
+# KEEL_ARCHIVE_LIST     where in TREE the source entry is
+#
+# Exit 0 when everything holds, 1 otherwise.
+set -eu
+
+KEEL_ARCHIVE_KEY="${KEEL_ARCHIVE_KEY:-}"
+KEEL_ARCHIVE_KEYRING="${KEEL_ARCHIVE_KEYRING:-/etc/apt/keyrings/keel-staging-keyring.asc}"
+KEEL_ARCHIVE_LIST="${KEEL_ARCHIVE_LIST:-/etc/apt/sources.list.d/keel-staging.list}"
+
+usage() {
+    echo "usage: $(basename "$0") SOURCE_REPO TREE DIST ARCH [LABEL]" >&2
+    echo "environment: KEEL_ARCHIVE_KEY (required), KEEL_ARCHIVE_KEYRING, KEEL_ARCHIVE_LIST" >&2
+    exit 1
+}
+
+[ $# -ge 4 ] && [ $# -le 5 ] || usage
+source_repo=$1
+tree=$2
+dist=$3
+arch=$4
+label=${5:-$2}
+
+for value in "$source_repo" "$tree" "$dist" "$arch"; do
+    [ -n "$value" ] || usage
+done
+
+fatal() {
+    echo "FATAL [archive-check $label]: $*" >&2
+    exit 1
+}
+
+[ -n "$KEEL_ARCHIVE_KEY" ] \
+    || fatal "KEEL_ARCHIVE_KEY names no key: the archive would be verified against whatever the keyring happens to hold"
+
+index=dists/$dist/main/binary-$arch/Packages
+source_index=$source_repo/$index
+copy=$tree/srv/keel-apt/repo
+copy_index=$copy/$index
+keyring=$tree/$KEEL_ARCHIVE_KEYRING
+list=$tree/$KEEL_ARCHIVE_LIST
+inrelease=$copy/dists/$dist/InRelease
+
+# 1. The copy is the archive as it is right now.
+[ -f "$source_index" ] || fatal "the archive has no index at $source_index"
+[ -f "$copy_index" ] || fatal "the build tree has no archive index at $copy_index"
+
+if ! cmp -s "$source_index" "$copy_index"; then
+    echo "FATAL [archive-check $label]: $copy_index is not $source_index" >&2
+    diff -u "$copy_index" "$source_index" >&2 || true
+    echo "The build would install the packages the archive offered when this" >&2
+    echo "bootstrap was made, not the ones it offers now. Remove the build" >&2
+    echo "directory (or at least build/stamps/bootstrap) and build again." >&2
+    exit 1
+fi
+
+# 2. Nothing in the tree switches verification off. A single trusted=yes
+# anywhere turns every failure below into a warning, which is the defect this
+# check exists for, so it is refused wherever it is written, in either of the
+# two formats apt reads a source in.
+verification_off='trusted *= *yes|^ *Trusted: *yes'
+apt_sources="$tree/etc/apt/sources.list $tree/etc/apt/sources.list.d"
+# shellcheck disable=SC2086 # the two paths are one word each, on purpose
+untrusted=$(grep -rlE "$verification_off" $apt_sources 2>/dev/null || true)
+[ -z "$untrusted" ] \
+    || fatal "verification is switched off in: $(echo "$untrusted" | tr '\n' ' ')"
+
+# 3. The source entry names this distribution and the keyring it is verified
+# with. apt takes signed-by from the entry, so the entry is what decides
+# whether anything is verified at all.
+[ -f "$list" ] || fatal "the build tree has no source entry at $list"
+entry=$(grep -E "^[[:space:]]*deb[[:space:]]" "$list" | head -1)
+[ -n "$entry" ] || fatal "$list has no deb line"
+[[ "$entry" == *"signed-by=$KEEL_ARCHIVE_KEYRING"* ]] \
+    || fatal "$list does not name signed-by=$KEEL_ARCHIVE_KEYRING: $entry"
+# Padded with spaces on both sides so that trixie does not pass for the entry
+# of trixie-staging.
+[[ " $entry " == *" $dist "* ]] \
+    || fatal "$list does not name the distribution $dist: $entry"
+
+# 4. The keyring is there and holds the key that must have signed the archive.
+# A keyring that is present but holds the wrong key reads as a configured
+# build and fails only at apt, in a line that scrolls past.
+[ -s "$keyring" ] || fatal "$keyring is missing or empty: the build cannot verify the archive"
+gpg --batch --show-keys --with-colons "$keyring" 2>/dev/null \
+    | awk -F: -v want="$KEEL_ARCHIVE_KEY" '$1 == "fpr" && $10 == want { found = 1 } END { exit !found }' \
+    || fatal "$keyring does not hold key $KEEL_ARCHIVE_KEY"
+
+# 5. The signature on the copied index is good under that keyring. gpgv wants
+# a binary keyring, and dearmouring a public key needs no agent and no
+# network.
+[ -s "$inrelease" ] || fatal "$inrelease is missing or empty: this copy of the archive is not signed"
+binary_keyring=$(mktemp)
+trap 'rm -f "$binary_keyring"' EXIT
+if ! gpg --batch --dearmor < "$keyring" > "$binary_keyring" 2>/dev/null \
+    || ! gpgv --keyring "$binary_keyring" "$inrelease" >/dev/null 2>&1; then
+    fatal "the signature on $inrelease does not verify against $keyring"
+fi
+
+echo "[archive-check $label] $copy_index is the archive at $source_index"
+echo "[archive-check $label] $inrelease verifies against $KEEL_ARCHIVE_KEY, nothing is trusted unverified"

changelog

@@ -1,3 +1,34 @@
+turnkey-postgresql-19.0 (3) turnkey; urgency=low
+
+  * The build verifies the project's own APT archive instead of reading it
+    unverified. The staging distribution has been signed since 2026-09-27, but
+    the build environment had no copy of the public key and the source entry
+    said [trusted=yes], which switches verification off: apt printed
+    "W: OpenPGP signature verification failed ... Missing key
+    8CFD1A4841448B2227341CEB202CACBD0E97090A" and installed inithooks,
+    confconsole and keel anyway. The public half of the staging key is now
+    installed into the build tree as
+    /etc/apt/keyrings/keel-staging-keyring.asc, the source entry names it
+    through signed-by, and apt-get update runs with --error-on=any, so a
+    signature that cannot be checked fails the build instead of warning
+    (tracker#7).
+
+  * bin/keel-archive-check does the same checks itself rather than trusting
+    apt to have complained: the copied package index is the live one, the
+    source entry names the keyring and this distribution, no apt source in
+    the tree says trusted=yes, and the signature on the copied InRelease
+    verifies against the named key with gpgv. It runs where the copy is made
+    and again on the tree that is about to be configured, which is a step
+    where no apt-get update runs at all.
+
+  * conf.d/main removes the keyring with the source entry and the copy of the
+    archive: the staging key signs whatever the build host produced, so an
+    image that kept it would carry trust in a nightly. The removelist
+    common/removelists-final/turnkey takes all three out as well, whatever a
+    recipe does.
+
+ -- Keel Linux maintainers <admin@keellinux.org>  Sun, 27 Sep 2026 19:30:00 +0000
+
 turnkey-postgresql-19.0 (2) turnkey; urgency=low
 
   * Listen on both loopback addresses, named literally:

conf.d/main

@@ -113,9 +113,15 @@ keel 0.2.1
 EOF
 dpkg-query -W -f '${Package} ${Version} ${Status}\n' inithooks confconsole keel
 
-# The build time package source is not for appliances: remove it, keep the
-# documented, disabled entry for the future signed repository (overlay).
+# The build time package source is not for appliances: remove the source
+# entry, the copy of the archive it names and the keyring the build verified
+# that archive with, and keep the documented, disabled entry for the future
+# signed repository (overlay). The staging key signs whatever the build host
+# produced, so an image that kept it would carry trust in a nightly
+# (tracker#7). common/removelists-final/turnkey removes the same three paths
+# at the end of the build, whatever a recipe does.
 rm -f /etc/apt/sources.list.d/keel-staging.list
+rm -f /etc/apt/keyrings/keel-staging-keyring.asc
 rm -rf /srv/keel-apt
 rm -rf /var/lib/apt/lists/*
 grep -q '^Enabled: no' /etc/apt/sources.list.d/keel.sources