Preflight validation + destroy teardown (open-source lifecycle)

.github/workflows/ci.yml

@@ -13,14 +13,9 @@ jobs:
       - name: Install validators
         run: sudo apt-get update -qq && sudo apt-get install -yqq shellcheck cloud-init
 
-      - name: Shellcheck
-        run: shellcheck provision.sh setup-user.sh sync-code.sh files/remote-setup.sh files/claude-notify.tmpl files/devbox-health.tmpl
-
-      - name: Bash syntax
-        run: bash -n provision.sh setup-user.sh sync-code.sh files/remote-setup.sh files/claude-notify.tmpl files/devbox-health.tmpl
-
-      - name: No personal residue outside LICENSE
-        run: make residue
+      # Single source of truth for the file list + residue gate: the Makefile.
+      - name: Static gates (shellcheck, bash -n, residue)
+        run: make check
 
       - name: Render + validate cloud-init schema
         run: |

Makefile

@@ -1,11 +1,17 @@
-.PHONY: check residue provision setup sync all
+SCRIPTS = preflight.sh provision.sh setup-user.sh sync-code.sh destroy.sh \
+          files/remote-setup.sh files/claude-notify.tmpl files/devbox-health.tmpl
+
+.PHONY: check residue preflight provision setup sync all destroy
 
 # Static gates that run locally (CI runs these plus cloud-init schema validation).
 check: residue
-	shellcheck provision.sh setup-user.sh sync-code.sh files/remote-setup.sh files/claude-notify.tmpl files/devbox-health.tmpl
-	bash -n provision.sh setup-user.sh sync-code.sh files/remote-setup.sh files/claude-notify.tmpl files/devbox-health.tmpl
+	shellcheck $(SCRIPTS)
+	bash -n $(SCRIPTS)
 	@echo "check: OK"
 
+preflight:
+	./preflight.sh
+
 provision:
 	./provision.sh
 
@@ -29,3 +35,7 @@ residue:
 	@git grep -inE '(^|[^a-z])(m[e]rt|o[z]6un|o[z]gun)([^a-z]|$$)' -- ':(exclude)LICENSE'; \
 	rc=$$?; if [ $$rc -eq 0 ]; then echo "personal residue found ^"; exit 1; \
 	elif [ $$rc -ge 2 ]; then echo "residue grep errored"; exit 1; fi
+
+# Tear it all down (Hetzner server + local state). Tailscale node removal is manual.
+destroy:
+	./destroy.sh

README.md

@@ -57,13 +57,17 @@ ACLs** (the default policy allows it). Verify both for your tailnet (new tailnet
 ## Provision
 
 ```sh
-cp secrets.env.example secrets.env   # fill in HCLOUD_TOKEN + TS_AUTHKEY
+cp secrets.env.example secrets.env   # fill in HCLOUD_TOKEN + TS_AUTHKEY (+ tweak DEV_USER, etc.)
+make preflight                       # validate token/tailnet/authkey BEFORE spending money
 make provision                       # create server; cloud-init hardens + joins tailnet (~5-10 min)
 make setup                           # user env: fish/tmux/node/claude/hooks (idempotent, re-runnable)
 # one-time interactive auth on the box (see below), then:
 make sync                            # mirror ~/Code repos + .env files
 ```
 
+`make provision` runs `preflight` itself, so a missing prerequisite fails in
+seconds instead of a confusing 15-minute timeout after the server already exists.
+
 ## Manual steps (unavoidable — interactive auth)
 
 | Step | Where | Why it can't be automated |
@@ -105,6 +109,17 @@ generate a fresh `TS_AUTHKEY`, then run the three steps above.
 
 `setup-user.sh` is safe to re-run any time to converge config drift on a live box.
 
+## Teardown
+
+```sh
+make destroy                 # delete the Hetzner server (asks you to type the name)
+FORCE=1 make destroy         # skip the prompt;  DRY_RUN=1 make destroy  to preview
+```
+
+Deletes the server (billing stops immediately) and clears the local SSH host key.
+Removing the Tailscale node is the one manual step — the script prints the link;
+skip it and a rebuild registers as `<name>-1`, breaking MagicDNS.
+
 ## What is deliberately NOT here
 
 - **Hetzner backups** — decide per-rebuild (+20% ≈ €1.10/mo, one API call or console toggle).

destroy.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Tear the devbox down: delete the Hetzner server and clean local state. The
+# inverse of provision.sh, using the same secrets.env. Removing the Tailscale
+# node is manual (no Tailscale API key is in scope) — the exact step is printed.
+#
+#   make destroy              # confirm by typing the name
+#   FORCE=1 make destroy      # skip the prompt (CI / scripted)
+#   DRY_RUN=1 make destroy     # show what would happen, delete nothing
+set -euo pipefail
+cd "$(dirname "$0")"
+
+[ -f ./secrets.env ] || { echo "secrets.env not found." >&2; exit 1; }
+# shellcheck disable=SC1091
+source ./secrets.env
+: "${HCLOUD_TOKEN:?set HCLOUD_TOKEN in secrets.env}"
+DEVBOX_NAME="${DEVBOX_NAME:-devbox}"
+DEV_USER="${DEV_USER:-dev}"
+# Normalize the flags: only explicit truthy values enable FORCE (default = ask),
+# and ANY non-false value keeps DRY_RUN on — a "preview" flag must never delete
+# by accident (DRY_RUN=true/yes previously slipped through as "not dry").
+case "${FORCE:-0}"   in 1|true|yes|on) FORCE=1 ;; *) FORCE=0 ;; esac
+case "${DRY_RUN:-0}" in 0|false|no|"") DRY_RUN=0 ;; *) DRY_RUN=1 ;; esac
+
+# Look up the server, distinguishing "0 servers" from an API/auth error. A
+# revoked token (secrets.env.example tells you to revoke it!) returns 401 with
+# an empty .servers parse — which must NOT read as "nothing to delete" while a
+# server keeps billing.
+resp=$(curl -s -w $'\n%{http_code}' --max-time 10 \
+  -H "Authorization: Bearer $HCLOUD_TOKEN" \
+  "https://api.hetzner.cloud/v1/servers?name=$DEVBOX_NAME") \
+  || { echo "Hetzner API unreachable." >&2; exit 1; }
+code=${resp##*$'\n'}
+body=${resp%$'\n'*}
+if [ "$code" != 200 ]; then
+  echo "Hetzner API returned HTTP $code (token revoked, or wrong project?)." >&2
+  echo "Cannot verify server state — aborting rather than assume it's gone." >&2
+  exit 1
+fi
+count=$(printf '%s' "$body" | jq '.servers | length')
+if [ "$count" -gt 1 ]; then
+  echo "Multiple servers named '$DEVBOX_NAME' exist — refusing to guess which." >&2
+  echo "Delete the intended one via the Hetzner console." >&2
+  exit 1
+fi
+id=$(printf '%s' "$body" | jq -r '.servers[0].id // empty')
+
+if [ -z "$id" ]; then
+  echo "No Hetzner server named '$DEVBOX_NAME' — nothing to delete."
+else
+  echo "Found server '$DEVBOX_NAME' (id $id)."
+  if [ "$FORCE" != 1 ] && [ "$DRY_RUN" != 1 ]; then
+    printf "Type the server name to confirm PERMANENT deletion: "
+    read -r ans
+    [ "$ans" = "$DEVBOX_NAME" ] || { echo "Aborted."; exit 1; }
+  fi
+  if [ "$DRY_RUN" = 1 ]; then
+    echo "[dry-run] would log out the tailnet node, then DELETE /v1/servers/$id"
+  else
+    # Best-effort: drop the node from the tailnet before the box vanishes (the
+    # operator user can run this without sudo). Never blocks the delete.
+    ssh -o ConnectTimeout=6 "$DEV_USER@$DEVBOX_NAME" 'tailscale logout' 2>/dev/null \
+      || ssh -o ConnectTimeout=6 "$DEV_USER@$DEVBOX_NAME" 'sudo tailscale logout' 2>/dev/null \
+      || true
+    del=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 -X DELETE \
+      -H "Authorization: Bearer $HCLOUD_TOKEN" "https://api.hetzner.cloud/v1/servers/$id") \
+      || { echo "DELETE request failed to reach the API." >&2; exit 1; }
+    if [ "$del" = 200 ] || [ "$del" = 204 ]; then
+      echo "Server deleted (HTTP $del). Billing stopped."
+    else
+      echo "Delete failed (HTTP $del)." >&2
+      exit 1
+    fi
+  fi
+fi
+
+# Local cleanup: a rebuilt node presents a new SSH host key under the same name.
+if [ "$DRY_RUN" != 1 ]; then
+  ssh-keygen -R "$DEVBOX_NAME" >/dev/null 2>&1 || true
+fi
+
+echo
+echo "Manual step (no Tailscale API key in scope):"
+echo "  Remove the '$DEVBOX_NAME' node → https://login.tailscale.com/admin/machines"
+echo "  Skip this and a rebuild registers as '$DEVBOX_NAME-1', breaking MagicDNS."

docs/FOOTGUNS.md

@@ -35,6 +35,11 @@ broke something in practice; don't re-learn them.
 
 ## Tailscale
 
+- **A stopped Tailscale client (not the server) shows up as `Could not resolve
+  hostname <name>`.** The Mac dropping off the tailnet — common after sleep —
+  looks identical to a server problem. Check `tailscale status`; `tailscale up`
+  fixes it. `make preflight` now catches this before provisioning.
+
 - **`tailscale serve` needs a one-time tailnet enable** (a `login.tailscale.com/f/serve`
   approval link) — the CLI silently blocks until it's clicked.
 - **Node key expiry must be disabled per node.** Tailscale SSH is the ONLY access

preflight.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Fail fast BEFORE provision creates any billable resource. Run standalone with
+# `make preflight`, and automatically as the first step of provision.sh. Every
+# check is read-only; a ✗ tells you exactly what to fix.
+set -uo pipefail
+cd "$(dirname "$0")" || exit 1
+
+fail=0
+ok()  { echo "  ✓ $1"; }
+err() { echo "  ✗ $1" >&2; fail=1; }
+
+echo "Preflight:"
+
+if [ ! -f ./secrets.env ]; then
+  err "secrets.env missing — cp secrets.env.example secrets.env and fill it in."
+  exit 1
+fi
+# shellcheck disable=SC1091
+source ./secrets.env
+DEVBOX_NAME="${DEVBOX_NAME:-devbox}"
+
+# Local CLIs the scripts call.
+for c in curl jq ssh tailscale git; do
+  if command -v "$c" >/dev/null 2>&1; then ok "$c on PATH"; else err "$c not found on PATH"; fi
+done
+
+# Hetzner token — distinguish "wrong token" from "network down".
+if [ -z "${HCLOUD_TOKEN:-}" ]; then
+  err "HCLOUD_TOKEN empty in secrets.env"
+else
+  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 \
+    -H "Authorization: Bearer $HCLOUD_TOKEN" https://api.hetzner.cloud/v1/servers || echo 000)
+  case "$code" in
+    200) ok "Hetzner token valid" ;;
+    401) err "Hetzner token rejected (401) — check HCLOUD_TOKEN" ;;
+    *)   err "Hetzner API unreachable (HTTP $code)" ;;
+  esac
+fi
+
+# Tailscale pre-auth key format.
+case "${TS_AUTHKEY:-}" in
+  tskey-*) ok "TS_AUTHKEY format looks right" ;;
+  "")      err "TS_AUTHKEY empty — generate one (reusable off, ephemeral off, pre-approved)" ;;
+  *)       err "TS_AUTHKEY doesn't look like a tskey-… key" ;;
+esac
+
+# Tailscale up + MagicDNS: provision's wait loop resolves \$DEV_USER@\$DEVBOX_NAME
+# by name, so a stopped client or MagicDNS-off tailnet makes it time out blind.
+# (This is exactly the "Could not resolve hostname" class of failure.)
+if tailscale status >/dev/null 2>&1; then
+  ok "Tailscale is up"
+  if tailscale status --json 2>/dev/null | jq -e '.CurrentTailnet.MagicDNSEnabled == true' >/dev/null 2>&1; then
+    ok "MagicDNS enabled"
+  else
+    err "MagicDNS is off — enable it (Tailscale admin → DNS); provision resolves the box by name"
+  fi
+else
+  err "Tailscale is not running — run 'tailscale up' on this machine first"
+fi
+
+# Name must be free — otherwise provision would refuse anyway, but say so now.
+# Validate the HTTP status: an error body must not parse to a false "0 / free".
+if [ -n "${HCLOUD_TOKEN:-}" ]; then
+  resp=$(curl -s -w $'\n%{http_code}' --max-time 10 -H "Authorization: Bearer $HCLOUD_TOKEN" \
+    "https://api.hetzner.cloud/v1/servers?name=$DEVBOX_NAME" || true)
+  code=${resp##*$'\n'}
+  if [ "$code" = 200 ]; then
+    n=$(printf '%s' "${resp%$'\n'*}" | jq '.servers | length' 2>/dev/null || echo "?")
+    case "$n" in
+      0)   ok "server name '$DEVBOX_NAME' is free" ;;
+      "?") err "couldn't parse the Hetzner server list" ;;
+      *)   err "a server named '$DEVBOX_NAME' already exists — pick another DEVBOX_NAME or destroy it" ;;
+    esac
+  else
+    err "couldn't check name '$DEVBOX_NAME' (Hetzner HTTP ${code:-unreachable})"
+  fi
+fi
+
+if [ "$fail" = 0 ]; then
+  echo "Preflight OK."
+else
+  echo "Preflight FAILED — fix the ✗ items above before provisioning." >&2
+  exit 1
+fi

provision.sh

@@ -27,6 +27,10 @@ case "$DEV_USER" in
   on|off|yes|no|true|false|null|root)
     echo "DEV_USER '$DEV_USER' is reserved (YAML boolean/null or root)." >&2; exit 1 ;;
 esac
+
+# Fail fast before spending money: validates token, tailnet, authkey, and that
+# the name is free. Covers the old inline existing-server check and more.
+./preflight.sh || exit 1
 SERVER_TYPE="${SERVER_TYPE:-cx23}"
 LOCATION="${LOCATION:-fsn1}"
 IMAGE="${IMAGE:-ubuntu-24.04}"
@@ -38,14 +42,7 @@ api() {
     "https://api.hetzner.cloud/v1$path"
 }
 
-# Guard: no declarative state here — a second run must not create a twin.
-existing=$(api GET "/servers?name=$DEVBOX_NAME" | jq '.servers | length')
-if [ "$existing" != "0" ]; then
-  echo "ERROR: a server named '$DEVBOX_NAME' already exists in this Hetzner project." >&2
-  echo "This script provisions from scratch; to rebuild, delete the old server (and its" >&2
-  echo "tailnet node in the Tailscale admin console) first." >&2
-  exit 1
-fi
+# (preflight.sh already refused to continue if a server of this name exists.)
 
 # Render cloud-init. sed, not envsubst (not on stock macOS). Tailscale auth keys
 # are [A-Za-z0-9-] so they are safe inside a sed replacement.