fix: etcd's health on its metrics listener, and no lone member kept

.github/workflows/packages.yml

@@ -39,7 +39,10 @@
 # policy-rc.d: the suite needs Nginx and Anubis running, as on a machine.
 # Coraza's module, its engine, the Core Rule Set and Anubis are Keel's own
 # packages of step 5, not published yet, so they are built here with gbp
-# from the commits pinned below, with golang-1.26-go from trixie-backports.
+# from the commits pinned below, with golang-1.26-go 1.26.8-1~bpo13+1 of
+# trixie-backports, taken from snapshot.debian.org as of 2026-10-01:
+# trixie-backports has since replaced it with golang-1.27-go, and the
+# pinned commits build with 1.26.
 #
 # keel, inithooks and confconsole are not published yet (Keel-Linux/apt:
 # the signing subkey is being rotated), so they are built here from the
@@ -143,10 +146,17 @@ jobs:
                libnginx-mod-http-coraza:630c68f203708d0819c70bc06427b04953f9ffdd
                anubis:e2512d3931cd82a994106513b8eece5034a7c7fe"
         KEEL_COMMIT=3143e870805c6214efe2bde1f589880fc47b0955
+        GO_SNAPSHOT=https://snapshot.debian.org/archive/debian/20261001T000000Z
+        GO_VERSION=1.26.8-1~bpo13+1
         apt-get update -qq
         apt-get install -y -qq --no-install-recommends \
           ca-certificates git git-buildpackage pristine-tar devscripts \
           equivs build-essential debhelper bats curl
+        echo "deb [check-valid-until=no] $GO_SNAPSHOT trixie-backports main" \
+          > /etc/apt/sources.list.d/go-snapshot.list
+        apt-get update -qq
+        apt-get install -y -qq --no-install-recommends \
+          "golang-1.26-go=$GO_VERSION" "golang-1.26-src=$GO_VERSION"
         git config --global --add safe.directory '*'
         mkdir -p /tmp/area /tmp/deps
         for pinned in $STEP5; do

packages/README.md

@@ -21,6 +21,27 @@ Every package depends on `keel (>= 0.12.0)`, the first keel that reads
 `manifest_version: 1`, so apt refuses a keel that could not read the
 manifest (docs/manifest-v1.md, "Versioning").
 
+## etcd's own first start
+
+`etcd-server`'s `postinst` starts etcd, which writes a member of a
+cluster of one to `/var/lib/etcd/default`. A node must not keep it: keel
+joins the node to the mesh's cluster (handbook decision 0048), and a
+member directory there would bring the lone cluster back. So the
+overlay's `preinst` notes whether `/var/lib/etcd/default/member` existed
+before its first installation, and its `postinst`, once it has stopped
+etcd, marks it (`/var/lib/keel-overlay-etcd/package-member`) only when it
+did not: what `etcd-server` made in the same transaction, never data
+that was there before. keel removes a marked member before it starts
+etcd for the mesh's cluster, and refuses to start etcd over an unmarked
+one it never started, for the operator to look at. When apt configures
+`etcd-server` before it unpacks the overlay, the directory exists at
+`preinst` and is not marked, so keel refuses rather than guesses.
+
+Monit's `etcd-health` check asks `/health` on etcd's plain metrics
+listener, `http://[::1]:2381`, which keel renders
+(`ETCD_LISTEN_METRICS_URLS`): the client port, 2379, wants TLS and a
+client certificate signed in the mesh.
+
 ## Building
 
 ```

packages/etcd/debian/changelog

@@ -1,3 +1,19 @@
+keel-overlay-etcd (0.2.0) trixie; urgency=medium
+
+  * Monit's etcd-health check asks /health on etcd's plain metrics
+    listener, http://[::1]:2381, which keel renders: the client port is
+    TLS with client certificates now (keel's etcd over the mesh,
+    handbook decision 0048), and a plain probe of it failed every cycle,
+    so Monit restarted etcd forever.
+  * The first installation marks /var/lib/etcd/default/member, in
+    /var/lib/keel-overlay-etcd/package-member, when etcd-server's own
+    postinst made it in the same transaction: a lone member of no
+    cluster, which keel removes before it starts etcd for the mesh's
+    cluster. A member that was there before is never marked, and keel
+    refuses to start etcd over one it can not prove is the package's.
+
+ -- Marcos Mendez <mendez.foto@gmail.com>  Sun, 04 Oct 2026 12:00:00 +0000
+
 keel-overlay-etcd (0.1.0) trixie; urgency=medium
 
   * First release (handbook decision 0041, first implementation, step 2).

packages/etcd/debian/postinst

@@ -17,6 +17,10 @@ set -e
 
 UNITS="etcd.service"
 KEPT=/var/lib/keel-overlay-etcd/kept-units
+FRESH=/var/lib/keel-overlay-etcd/no-member-before
+DATA=/var/lib/etcd/default
+# what keel reads as proof that the member is etcd-server's own
+MARK=/var/lib/keel-overlay-etcd/package-member
 
 if [ "$1" = configure ] && [ -z "$2" ]; then
     for unit in $UNITS; do
@@ -35,8 +39,18 @@ if [ "$1" = configure ] && [ -z "$2" ]; then
         if [ -d /run/systemd/system ]; then
             deb-systemd-invoke stop "$unit"
         fi
+        # the member etcd-server's own start made in this transaction is
+        # a lone cluster of one that no mesh formed: marked, so keel
+        # removes it before it starts etcd for the mesh's cluster, and
+        # no other (a member that was there before is never marked)
+        if [ -f "$FRESH" ] && [ -d "$DATA/member" ]; then
+            echo "keel-overlay-etcd: $DATA holds the lone member" \
+                "etcd-server's own start made at its installation; keel" \
+                "removes it when it joins this node to the mesh" >&2
+            : > "$MARK"
+        fi
     done
-    rm -f "$KEPT"
+    rm -f "$KEPT" "$FRESH"
     rmdir "${KEPT%/*}" 2>/dev/null || :
 fi
 

packages/etcd/debian/postrm

@@ -11,7 +11,8 @@ set -e
 KEPT=/var/lib/keel-overlay-etcd/kept-units
 
 if [ "$1" = purge ] || [ "$1" = abort-install ]; then
-    rm -f "$KEPT"
+    rm -f "$KEPT" /var/lib/keel-overlay-etcd/no-member-before \
+        /var/lib/keel-overlay-etcd/package-member
     rmdir "${KEPT%/*}" 2>/dev/null || :
 fi
 

packages/etcd/debian/preinst

@@ -16,10 +16,18 @@ set -e
 
 UNITS="etcd.service"
 KEPT=/var/lib/keel-overlay-etcd/kept-units
+# etcd-server's postinst starts etcd, which makes a lone member of no
+# cluster in its data directory; postinst removes it only if it was not
+# there before this first installation (packages/README.md)
+FRESH=/var/lib/keel-overlay-etcd/no-member-before
+MEMBER=/var/lib/etcd/default/member
 
 if [ "$1" = install ] && [ -z "$2" ]; then
     mkdir -p "${KEPT%/*}"
     : > "$KEPT"
+    if [ ! -e "$MEMBER" ]; then
+        : > "$FRESH"
+    fi
     [ -d /run/systemd/system ] || UNITS=""
     for unit in $UNITS; do
         # is-enabled fails for a unit whose file is not there yet, which is

packages/etcd/manifest.yaml

@@ -10,8 +10,12 @@ processes:
     listen:
       - {port: 2379, protocol: tcp, expose: mesh}
       - {port: 2380, protocol: tcp, expose: mesh}
+      # /health and /metrics alone, plain, on the loopback: the client
+      # port wants TLS and a client certificate (keel renders
+      # ETCD_LISTEN_METRICS_URLS=http://[::1]:2381)
+      - {address: "::1", port: 2381, protocol: tcp, expose: loopback}
 checks:
-  - {name: etcd-health, process: etcd, type: http, address: mesh,
-     port: 2379, path: /health, expect: 200, on_failure: restart}
+  - {name: etcd-health, process: etcd, type: http, address: loopback,
+     port: 2381, path: /health, expect: 200, on_failure: restart}
 data:
   - {path: /var/lib/etcd, replication: native, backup: none}

tests/overlay-install.bats

@@ -241,6 +241,17 @@ assert_simple_state() {
     assert_simple_state
 }
 
+@test "the lone member etcd-server's own start made is marked for keel" {
+    # etcd-server's postinst started etcd in the same transaction; the
+    # overlay marks that member, which keel removes before it joins this
+    # node to the mesh's cluster, and leaves etcd stopped
+    if [ -d /var/lib/etcd/default/member ]; then
+        [ -f /var/lib/keel-overlay-etcd/package-member ]
+    fi
+    run systemctl is-active etcd.service
+    [ "$output" != active ]
+}
+
 @test "the preset systemd applies at first boot says disabled for each unit" {
     local unit
     for unit in "${DISABLED_UNITS[@]}"; do