From ef531f508bd58ded216eb02924a3ad867c2f41b6 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 25 Sep 2026 01:58:43 +0000 Subject: [PATCH] ops: install a disk alarm as part of box setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev2 runs crawlproof and nichedb on one Postgres, so disk is a SHARED failure mode: a full filesystem stops writes for every database on it and takes both sites down together. Nothing was watching it. Folded into system_setup rather than shipped as a separate script, because setup-supabase.sh is deployed as a single scp'd file and a second file is a step someone forgets. Three checks: warn at 70%, critical at 85%, and — the one that earns its keep — critical whenever the observed growth rate would fill the disk inside 48 hours. A percentage threshold is useless against a burst: this box grew 149 -> 247 GB in four hours (~22 GB/h) during nichedb's catalogue import and 15% never looked alarming. It runs from cron on the box, not from an agent session. The first version of this lived in a session monitor, died with that session, and ~100 GB of overnight growth went unnoticed as a result. Monitoring that is not on the box is not monitoring. The comments also record the estimate that nearly misled us: nichedb's "6 GB/day" was measured on Railway with a throttled disk, and the same importers ran ~40x faster on NVMe. Growth figures do not transfer between hosts. Co-Authored-By: Claude Opus 5 (1M context) --- ops/selfhost/server/setup-supabase.sh | 91 ++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/ops/selfhost/server/setup-supabase.sh b/ops/selfhost/server/setup-supabase.sh index d822e61..0bfd672 100755 --- a/ops/selfhost/server/setup-supabase.sh +++ b/ops/selfhost/server/setup-supabase.sh @@ -17,7 +17,9 @@ # to it. Running Caddy here would fight nginx for :80 and :443. # # What it does: -# 1. System: base packages, Docker log rotation, ufw (ssh + 80/443 + Postgres) +# 1. System: base packages, Docker log rotation, ufw (ssh + 80/443 + Postgres), +# and a cron disk alarm — every database here shares one Postgres, so a +# full disk stops writes for all of them # 2. Supabase: official setup.sh at a pinned self-hosted release tag # 3. .env: public URLs, SMTP, compose overrides # 4. Postgres: self-signed TLS cert, pg_hba that only lets `postgres` in from @@ -111,6 +113,93 @@ system_setup() { # Docker publishes ports through its own iptables chains, ahead of ufw, so # the rules above document intent; the real gate for 5432 is pg_hba + TLS. ufw status | sed 's/^/ /' + + install_disk_alarm +} + +# A box running more than one property on one Postgres has disk as a SHARED +# failure mode: a full filesystem stops writes for every database on it, so +# one property's import takes the other's site down too. +# +# Installed as a cron job the box runs itself. The first version of this lived +# inside an agent session's monitor, died with that session, and ~100 GB of +# overnight growth went unnoticed. Monitoring that is not on the box is not +# monitoring. +# +# The rate check is the part that earns its keep. During nichedb's catalogue +# import dev2 grew 149 -> 247 GB in four hours (~22 GB/h); a percentage +# threshold would not have moved in time. Anything that would fill the disk +# within 48h is critical regardless of how full it currently is. +# +# Cautionary note baked in below: nichedb's "6 GB/day" was measured on Railway +# with a throttled disk. On NVMe the same importers ran ~40x faster. Growth +# figures do not transfer between hosts. +install_disk_alarm() { + log "Disk alarm (cron, every 15 min)" + local dest=/usr/local/bin/dev2-disk-alarm.sh + backup "$dest" + cat > "$dest" <<'ALARM' +#!/usr/bin/env bash +# Disk alarm. Warns early, and keeps a growth record in the log that is the +# right place to read a real steady-state rate off, rather than sampling by +# hand. Writes to syslog; mails root if a mailer exists. +set -uo pipefail + +WARN=${WARN:-70} +CRIT=${CRIT:-85} +LOG=/var/log/dev2-disk.log +STATE=/var/lib/dev2-disk-alarm.state + +pct=$(df --output=pcent / | tail -1 | tr -dc 0-9) +used_kb=$(df --output=used / | tail -1 | tr -dc 0-9) +now=$(date -u +%s) +stamp=$(date -u '+%F %T') + +rate="" +if [ -f "$STATE" ]; then + read -r prev_ts prev_kb < "$STATE" 2>/dev/null || true + if [ -n "${prev_ts:-}" ] && [ "$now" -gt "$prev_ts" ]; then + rate=$(( (used_kb - prev_kb) * 3600 / (now - prev_ts) / 1048576 )) + fi +fi +echo "$now $used_kb" > "$STATE" + +echo "$stamp used=${pct}% ($(df -h --output=used / | tail -1 | tr -d ' ')) rate=${rate:-?}GB/h" >> "$LOG" +tail -n 2000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG" + +notify() { + logger -t dev2-disk -p daemon.warning "$1: $2" + command -v mail >/dev/null 2>&1 && printf '%s\n\nRecent history:\n%s\n' \ + "$2" "$(tail -12 "$LOG")" | mail -s "dev2 disk $1: ${pct}% used" root 2>/dev/null || true +} + +if [ "$pct" -ge "$CRIT" ]; then + notify CRITICAL "root is ${pct}% full (${rate:-?} GB/h). One Postgres serves every database here — a full disk stops writes for all of them." +elif [ "$pct" -ge "$WARN" ]; then + notify WARNING "root is ${pct}% full (${rate:-?} GB/h). Check whether the VG still has free extents; if not, growing means adding a disk." +fi + +# Independent of thresholds: a burst fills a disk long before a percentage +# threshold reacts to it. +if [ -n "$rate" ] && [ "$rate" -gt 0 ]; then + avail_gb=$(df --output=avail -BG / | tail -1 | tr -dc 0-9) + hours=$(( avail_gb / rate )) + [ "$hours" -lt 48 ] && notify CRITICAL "will fill in ~${hours}h at ${rate} GB/h (${avail_gb} GB free)." +fi +ALARM + chmod 755 "$dest" + + cat > /etc/cron.d/dev2-disk-alarm <