From db46968830e49be3bbbce7e017b25ab5a2d82957 Mon Sep 17 00:00:00 2001 From: navigator Date: Sun, 4 Oct 2026 05:15:39 +0000 Subject: [PATCH 1/3] fix: etcd's health on its metrics listener, and no lone member kept keel now runs etcd over the mesh with TLS and client certificates on 2379, so Monit's plain probe of /health there failed every cycle and restarted etcd forever. The check asks /health on the plain metrics listener keel renders on http://[::1]:2381. etcd-server's postinst starts etcd, which leaves a member of a cluster of one in /var/lib/etcd/default. The overlay's first installation now removes it when it was not there before (preinst notes it), so the node keeps no lone cluster before keel joins it to the mesh's. Data that was there before is never touched; keel refuses to start etcd over a member it did not start. --- packages/README.md | 19 +++++++++++++++++++ packages/etcd/debian/changelog | 16 ++++++++++++++++ packages/etcd/debian/postinst | 12 +++++++++++- packages/etcd/debian/postrm | 2 +- packages/etcd/debian/preinst | 8 ++++++++ packages/etcd/manifest.yaml | 8 ++++++-- tests/overlay-install.bats | 6 ++++++ 7 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/README.md b/packages/README.md index 1868556c..a611a4e3 100644 --- a/packages/README.md +++ b/packages/README.md @@ -21,6 +21,25 @@ Every package depends on `keel (>= 0.12.0)`, the first keel that reads `manifest_version: 1`, so apt refuses a keel that could not read the manifest (docs/manifest-v1.md, "Versioning"). +## etcd's own first start + +`etcd-server`'s `postinst` starts etcd, which writes a member of a +cluster of one to `/var/lib/etcd/default`. A node must not keep it: keel +joins the node to the mesh's cluster (handbook decision 0048), and a +member directory there would bring the lone cluster back. So the +overlay's `preinst` notes whether `/var/lib/etcd/default/member` existed +before its first installation, and its `postinst`, once it has stopped +etcd, removes `/var/lib/etcd/default` only when it did not: what +`etcd-server` made in the same transaction, never data that was there +before. When apt configures `etcd-server` before it unpacks the overlay, +the directory exists at `preinst` and is kept; keel then refuses to start +etcd over a member it never started, and removes it when the node joins. + +Monit's `etcd-health` check asks `/health` on etcd's plain metrics +listener, `http://[::1]:2381`, which keel renders +(`ETCD_LISTEN_METRICS_URLS`): the client port, 2379, wants TLS and a +client certificate signed in the mesh. + ## Building ``` diff --git a/packages/etcd/debian/changelog b/packages/etcd/debian/changelog index 7b5ef47a..be79a15d 100644 --- a/packages/etcd/debian/changelog +++ b/packages/etcd/debian/changelog @@ -1,3 +1,19 @@ +keel-overlay-etcd (0.2.0) trixie; urgency=medium + + * Monit's etcd-health check asks /health on etcd's plain metrics + listener, http://[::1]:2381, which keel renders: the client port is + TLS with client certificates now (keel's etcd over the mesh, + handbook decision 0048), and a plain probe of it failed every cycle, + so Monit restarted etcd forever. + * The first installation removes /var/lib/etcd/default when + etcd-server's own postinst made it in the same transaction: its + start leaves a lone member of no cluster, which a node must not keep + before keel joins it to the mesh's. A data directory that was there + before is never touched; keel refuses to start etcd over one it did + not start. + + -- Marcos Mendez Sun, 04 Oct 2026 12:00:00 +0000 + keel-overlay-etcd (0.1.0) trixie; urgency=medium * First release (handbook decision 0041, first implementation, step 2). diff --git a/packages/etcd/debian/postinst b/packages/etcd/debian/postinst index ba7a340a..6f6bf8e9 100755 --- a/packages/etcd/debian/postinst +++ b/packages/etcd/debian/postinst @@ -17,6 +17,8 @@ set -e UNITS="etcd.service" KEPT=/var/lib/keel-overlay-etcd/kept-units +FRESH=/var/lib/keel-overlay-etcd/no-member-before +DATA=/var/lib/etcd/default if [ "$1" = configure ] && [ -z "$2" ]; then for unit in $UNITS; do @@ -35,8 +37,16 @@ if [ "$1" = configure ] && [ -z "$2" ]; then if [ -d /run/systemd/system ]; then deb-systemd-invoke stop "$unit" fi + # the member etcd-server's own start made in this transaction is + # a lone cluster of one that no mesh formed, and etcd would come + # back as it once enabled: removed, when none was there before + if [ -f "$FRESH" ] && [ -d "$DATA/member" ]; then + echo "keel-overlay-etcd: removing $DATA, the lone member" \ + "etcd-server's own start made at its installation" >&2 + rm -rf "$DATA" + fi done - rm -f "$KEPT" + rm -f "$KEPT" "$FRESH" rmdir "${KEPT%/*}" 2>/dev/null || : fi diff --git a/packages/etcd/debian/postrm b/packages/etcd/debian/postrm index b8ccf788..8efb6b56 100755 --- a/packages/etcd/debian/postrm +++ b/packages/etcd/debian/postrm @@ -11,7 +11,7 @@ set -e KEPT=/var/lib/keel-overlay-etcd/kept-units if [ "$1" = purge ] || [ "$1" = abort-install ]; then - rm -f "$KEPT" + rm -f "$KEPT" /var/lib/keel-overlay-etcd/no-member-before rmdir "${KEPT%/*}" 2>/dev/null || : fi diff --git a/packages/etcd/debian/preinst b/packages/etcd/debian/preinst index fdf33be9..e1b862e7 100755 --- a/packages/etcd/debian/preinst +++ b/packages/etcd/debian/preinst @@ -16,10 +16,18 @@ set -e UNITS="etcd.service" KEPT=/var/lib/keel-overlay-etcd/kept-units +# etcd-server's postinst starts etcd, which makes a lone member of no +# cluster in its data directory; postinst removes it only if it was not +# there before this first installation (packages/README.md) +FRESH=/var/lib/keel-overlay-etcd/no-member-before +MEMBER=/var/lib/etcd/default/member if [ "$1" = install ] && [ -z "$2" ]; then mkdir -p "${KEPT%/*}" : > "$KEPT" + if [ ! -e "$MEMBER" ]; then + : > "$FRESH" + fi [ -d /run/systemd/system ] || UNITS="" for unit in $UNITS; do # is-enabled fails for a unit whose file is not there yet, which is diff --git a/packages/etcd/manifest.yaml b/packages/etcd/manifest.yaml index e1d68cfb..19f23685 100644 --- a/packages/etcd/manifest.yaml +++ b/packages/etcd/manifest.yaml @@ -10,8 +10,12 @@ processes: listen: - {port: 2379, protocol: tcp, expose: mesh} - {port: 2380, protocol: tcp, expose: mesh} + # /health and /metrics alone, plain, on the loopback: the client + # port wants TLS and a client certificate (keel renders + # ETCD_LISTEN_METRICS_URLS=http://[::1]:2381) + - {address: "::1", port: 2381, protocol: tcp, expose: loopback} checks: - - {name: etcd-health, process: etcd, type: http, address: mesh, - port: 2379, path: /health, expect: 200, on_failure: restart} + - {name: etcd-health, process: etcd, type: http, address: loopback, + port: 2381, path: /health, expect: 200, on_failure: restart} data: - {path: /var/lib/etcd, replication: native, backup: none} diff --git a/tests/overlay-install.bats b/tests/overlay-install.bats index 2e14fd82..48bc8c92 100644 --- a/tests/overlay-install.bats +++ b/tests/overlay-install.bats @@ -241,6 +241,12 @@ assert_simple_state() { assert_simple_state } +@test "the lone member etcd-server's own start made is gone" { + # etcd-server's postinst started etcd in the same transaction; a + # node keeps no cluster of one before keel joins it to the mesh's + [ ! -e /var/lib/etcd/default/member ] +} + @test "the preset systemd applies at first boot says disabled for each unit" { local unit for unit in "${DISABLED_UNITS[@]}"; do From 6c1a7872b0434d1d5f8bf7970307f6811eeb0544 Mon Sep 17 00:00:00 2001 From: navigator Date: Sun, 4 Oct 2026 12:10:58 +0000 Subject: [PATCH 2/3] ci: golang-1.26-go for the web job from a snapshot of trixie-backports trixie-backports replaced golang-1.26-go with golang-1.27-go, so the step 5 packages pinned in the web job, which build with 1.26, could no longer have their build dependencies installed (mk-build-deps exited 1 for libcoraza). The job takes 1.26.8-1~bpo13+1 from snapshot.debian.org as of 2026-10-01, pinned. --- .github/workflows/packages.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 98c4e6a6..ced40b5b 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -39,7 +39,10 @@ # policy-rc.d: the suite needs Nginx and Anubis running, as on a machine. # Coraza's module, its engine, the Core Rule Set and Anubis are Keel's own # packages of step 5, not published yet, so they are built here with gbp -# from the commits pinned below, with golang-1.26-go from trixie-backports. +# from the commits pinned below, with golang-1.26-go 1.26.8-1~bpo13+1 of +# trixie-backports, taken from snapshot.debian.org as of 2026-10-01: +# trixie-backports has since replaced it with golang-1.27-go, and the +# pinned commits build with 1.26. # # keel, inithooks and confconsole are not published yet (Keel-Linux/apt: # the signing subkey is being rotated), so they are built here from the @@ -143,10 +146,17 @@ jobs: libnginx-mod-http-coraza:630c68f203708d0819c70bc06427b04953f9ffdd anubis:e2512d3931cd82a994106513b8eece5034a7c7fe" KEEL_COMMIT=3143e870805c6214efe2bde1f589880fc47b0955 + GO_SNAPSHOT=https://snapshot.debian.org/archive/debian/20261001T000000Z + GO_VERSION=1.26.8-1~bpo13+1 apt-get update -qq apt-get install -y -qq --no-install-recommends \ ca-certificates git git-buildpackage pristine-tar devscripts \ equivs build-essential debhelper bats curl + echo "deb [check-valid-until=no] $GO_SNAPSHOT trixie-backports main" \ + > /etc/apt/sources.list.d/go-snapshot.list + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + "golang-1.26-go=$GO_VERSION" "golang-1.26-src=$GO_VERSION" git config --global --add safe.directory '*' mkdir -p /tmp/area /tmp/deps for pinned in $STEP5; do From 4f5e4f9ae2a5f9f3ba41b60c0c016b5a491f863a Mon Sep 17 00:00:00 2001 From: navigator Date: Sun, 4 Oct 2026 16:59:56 +0000 Subject: [PATCH 3/3] fix: mark etcd-server's own lone member for keel instead of removing it keel removes a member directory only when it can prove it is the lone member etcd-server's own start made at the overlay's installation: the overlay's postinst now marks it in /var/lib/keel-overlay-etcd/ package-member rather than removing it, and never marks a member that was there before. --- packages/README.md | 12 +++++++----- packages/etcd/debian/changelog | 12 ++++++------ packages/etcd/debian/postinst | 14 +++++++++----- packages/etcd/debian/postrm | 3 ++- tests/overlay-install.bats | 13 +++++++++---- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/packages/README.md b/packages/README.md index a611a4e3..28573c42 100644 --- a/packages/README.md +++ b/packages/README.md @@ -29,11 +29,13 @@ joins the node to the mesh's cluster (handbook decision 0048), and a member directory there would bring the lone cluster back. So the overlay's `preinst` notes whether `/var/lib/etcd/default/member` existed before its first installation, and its `postinst`, once it has stopped -etcd, removes `/var/lib/etcd/default` only when it did not: what -`etcd-server` made in the same transaction, never data that was there -before. When apt configures `etcd-server` before it unpacks the overlay, -the directory exists at `preinst` and is kept; keel then refuses to start -etcd over a member it never started, and removes it when the node joins. +etcd, marks it (`/var/lib/keel-overlay-etcd/package-member`) only when it +did not: what `etcd-server` made in the same transaction, never data +that was there before. keel removes a marked member before it starts +etcd for the mesh's cluster, and refuses to start etcd over an unmarked +one it never started, for the operator to look at. When apt configures +`etcd-server` before it unpacks the overlay, the directory exists at +`preinst` and is not marked, so keel refuses rather than guesses. Monit's `etcd-health` check asks `/health` on etcd's plain metrics listener, `http://[::1]:2381`, which keel renders diff --git a/packages/etcd/debian/changelog b/packages/etcd/debian/changelog index be79a15d..2c205e09 100644 --- a/packages/etcd/debian/changelog +++ b/packages/etcd/debian/changelog @@ -5,12 +5,12 @@ keel-overlay-etcd (0.2.0) trixie; urgency=medium TLS with client certificates now (keel's etcd over the mesh, handbook decision 0048), and a plain probe of it failed every cycle, so Monit restarted etcd forever. - * The first installation removes /var/lib/etcd/default when - etcd-server's own postinst made it in the same transaction: its - start leaves a lone member of no cluster, which a node must not keep - before keel joins it to the mesh's. A data directory that was there - before is never touched; keel refuses to start etcd over one it did - not start. + * The first installation marks /var/lib/etcd/default/member, in + /var/lib/keel-overlay-etcd/package-member, when etcd-server's own + postinst made it in the same transaction: a lone member of no + cluster, which keel removes before it starts etcd for the mesh's + cluster. A member that was there before is never marked, and keel + refuses to start etcd over one it can not prove is the package's. -- Marcos Mendez Sun, 04 Oct 2026 12:00:00 +0000 diff --git a/packages/etcd/debian/postinst b/packages/etcd/debian/postinst index 6f6bf8e9..8effb757 100755 --- a/packages/etcd/debian/postinst +++ b/packages/etcd/debian/postinst @@ -19,6 +19,8 @@ UNITS="etcd.service" KEPT=/var/lib/keel-overlay-etcd/kept-units FRESH=/var/lib/keel-overlay-etcd/no-member-before DATA=/var/lib/etcd/default +# what keel reads as proof that the member is etcd-server's own +MARK=/var/lib/keel-overlay-etcd/package-member if [ "$1" = configure ] && [ -z "$2" ]; then for unit in $UNITS; do @@ -38,12 +40,14 @@ if [ "$1" = configure ] && [ -z "$2" ]; then deb-systemd-invoke stop "$unit" fi # the member etcd-server's own start made in this transaction is - # a lone cluster of one that no mesh formed, and etcd would come - # back as it once enabled: removed, when none was there before + # a lone cluster of one that no mesh formed: marked, so keel + # removes it before it starts etcd for the mesh's cluster, and + # no other (a member that was there before is never marked) if [ -f "$FRESH" ] && [ -d "$DATA/member" ]; then - echo "keel-overlay-etcd: removing $DATA, the lone member" \ - "etcd-server's own start made at its installation" >&2 - rm -rf "$DATA" + echo "keel-overlay-etcd: $DATA holds the lone member" \ + "etcd-server's own start made at its installation; keel" \ + "removes it when it joins this node to the mesh" >&2 + : > "$MARK" fi done rm -f "$KEPT" "$FRESH" diff --git a/packages/etcd/debian/postrm b/packages/etcd/debian/postrm index 8efb6b56..0dad80a7 100755 --- a/packages/etcd/debian/postrm +++ b/packages/etcd/debian/postrm @@ -11,7 +11,8 @@ set -e KEPT=/var/lib/keel-overlay-etcd/kept-units if [ "$1" = purge ] || [ "$1" = abort-install ]; then - rm -f "$KEPT" /var/lib/keel-overlay-etcd/no-member-before + rm -f "$KEPT" /var/lib/keel-overlay-etcd/no-member-before \ + /var/lib/keel-overlay-etcd/package-member rmdir "${KEPT%/*}" 2>/dev/null || : fi diff --git a/tests/overlay-install.bats b/tests/overlay-install.bats index 48bc8c92..7a66cb5c 100644 --- a/tests/overlay-install.bats +++ b/tests/overlay-install.bats @@ -241,10 +241,15 @@ assert_simple_state() { assert_simple_state } -@test "the lone member etcd-server's own start made is gone" { - # etcd-server's postinst started etcd in the same transaction; a - # node keeps no cluster of one before keel joins it to the mesh's - [ ! -e /var/lib/etcd/default/member ] +@test "the lone member etcd-server's own start made is marked for keel" { + # etcd-server's postinst started etcd in the same transaction; the + # overlay marks that member, which keel removes before it joins this + # node to the mesh's cluster, and leaves etcd stopped + if [ -d /var/lib/etcd/default/member ]; then + [ -f /var/lib/keel-overlay-etcd/package-member ] + fi + run systemctl is-active etcd.service + [ "$output" != active ] } @test "the preset systemd applies at first boot says disabled for each unit" {