diff --git a/.env.example b/.env.example index 557f868..5a7d793 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,22 @@ MAIL_ACCOUNT="email account" MAIL_PASSWORD='email app engine password' MAIL_PERSON_LIST='["receiver email list"]' ADMIN_IDENTITY="admin_id" + +# Monitoring stack (docs/monitoring.md) +GRAFANA_ADMIN_PASSWORD="" # this exact name (not GF_SECURITY_ADMIN_PASSWORD) +# PROMETHEUS_RETENTION_SIZE="5GB" # lower it if preflight warns about disk space +# ALERT_EMAIL_TO="you@example.com" # default: MAIL_ACCOUNT +# GRAFANA_BIND="127.0.0.1" +# DOCKER_ROOT_DIR="/var/snap/docker/common/var-lib-docker" # snap Docker +# CONTAINERD_SOCKET="/run/containerd/containerd.sock" # scripts/monitoring/preflight.sh prints the right value +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml +# Snap Docker (preflight --deep says containers see a different /): node-exporter on the host: +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml:deploy/monitoring/docker-compose.host-node-exporter.yml +# Status picture for the P4 display (deploy/monitoring/docker-compose.display-export.yml): +# COMPOSE_PROFILES=display-export # exporter as a container (else the host timer) +# DISPLAY_EXPORT_DIR="/home/you/code/esphome-config/grafana-export" +# DISPLAY_TZ="America/Toronto" +# DISPLAY_EXPORT_UID=1000 # uid=/gid= of the share in /etc/fstab +# DISPLAY_EXPORT_GID=1000 +# DISPLAY_INTERVAL=300 # seconds; the display marks pictures STALE after 10 min +# DISPLAY_EXPORT_REQUIRE_FSTYPE=cifs # "" to allow a plain local folder diff --git a/CLAUDE.md b/CLAUDE.md index fa9dc37..dbf9998 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,6 +52,15 @@ pytest -q tests/test_misc_routes.py::test_notes_media_public_note_for_other_user ruff check . ``` +### Monitoring and backups +Monitoring overlay: `deploy/monitoring/docker-compose.monitoring.yml` (Prometheus, Alertmanager, Grafana, exporters). Encrypted backups: `scripts/db_backup.sh` / `scripts/db_restore.sh` (age, public-key). See `docs/monitoring.md` and `docs/operations-checklist.md`; `scripts/monitoring/preflight.sh` checks host requirements. Validate rule changes with: +```bash +docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" --entrypoint promtool prom/prometheus:v3.5.0 check config /etc/prometheus/prometheus.yml +docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" --entrypoint promtool prom/prometheus:v3.5.0 test rules /etc/prometheus/tests/network_mounts_test.yml +docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" --entrypoint promtool prom/prometheus:v3.5.0 test rules /etc/prometheus/tests/adguard_test.yml +``` +The P4 display dashboard (`grafana/dashboards/safefamily-display.json`) is generated from the overview's panels by `deploy/monitoring/display-export/make_display_dashboard.py`; edit the overview panels or the script's LAYOUT, not the JSON. User extensions go in `deploy/monitoring/prometheus/scrape.d/` and `rules.d/` (loaded by glob), not in edits to `prometheus.yml`/`alerts.yml`. + ### Run log analysis CLI ```bash python -m safe_family.cli.analyze --range last_5min diff --git a/README.md b/README.md index 7c8a882..fb33c18 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,21 @@ gunzip -c backup/full-2026-09-16.sql.gz \ the analyzer pages will simply have no history until new AdGuard logs arrive. - `backup/` is gitignored, as is `deploy/init_db/`. -### 7. Host Maintenance Notes +### 7. Monitoring and Encrypted Backups +Prometheus, Grafana and Alertmanager (disk space, database size and health, +per-container CPU/memory, nginx load, website/TLS checks, email alerts) and the +age-encrypted weekly database backup are described in +[docs/monitoring.md](docs/monitoring.md); set it up with +[docs/operations-checklist.md](docs/operations-checklist.md), starting with +`sudo scripts/monitoring/preflight.sh --deep`. Quick start: +```bash +# .env: GRAFANA_ADMIN_PASSWORD=... and +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml +docker compose up -d # Grafana on https:///grafana/ (LAN only) +sudo systemctl start safefamily-backup # after the one-time setup in the doc +``` + +### 8. Host Maintenance Notes - Snap auto-refreshes Docker several times a day (`snap refresh --time`). A refresh restarts the daemon and briefly stops containers; `restart: always` brings them back, but the timing is not under your control. Use diff --git a/deploy/monitoring/adguard.env.example b/deploy/monitoring/adguard.env.example new file mode 100644 index 0000000..0501415 --- /dev/null +++ b/deploy/monitoring/adguard.env.example @@ -0,0 +1,15 @@ +# Copy to /etc/safefamily/adguard.env and make it root-only: +# sudo install -m 600 deploy/monitoring/adguard.env.example /etc/safefamily/adguard.env +# Same values as ADGUARD_HOSTPORT / ADGUARD_USERNAME / ADGUARD_PASSWORD in the app's .env. +ADGUARD_HOSTPORT=10.0.0.1:8080 +ADGUARD_USERNAME=admin +ADGUARD_PASSWORD=change-me +# Blocked-service ids to report (the list SafeFamily switches; fast, local). +ADGUARD_WATCH_SERVICES="youtube" +# Domains to check with AdGuard's check_host (also covers filter rules), space +# separated. Slow on some routers; set to "" to turn it off. +ADGUARD_WATCH_DOMAINS="youtube.com" +# Seconds per AdGuard API call. +ADGUARD_TIMEOUT=30 +# Keep in step with NODE_EXPORTER_TEXTFILE_DIR in /etc/safefamily/backup.env. +NODE_EXPORTER_TEXTFILE_DIR=/var/lib/node_exporter/textfile diff --git a/deploy/monitoring/alertmanager/alertmanager.yml.tmpl b/deploy/monitoring/alertmanager/alertmanager.yml.tmpl new file mode 100644 index 0000000..e5cd7d1 --- /dev/null +++ b/deploy/monitoring/alertmanager/alertmanager.yml.tmpl @@ -0,0 +1,43 @@ +# Rendered by entrypoint.sh: ${NAME} placeholders come from the container env +# (see docker-compose.monitoring.yml), so no secret is stored in this file. +global: + smtp_smarthost: '${ALERT_SMTP_SMARTHOST}' + smtp_from: '${MAIL_ACCOUNT}' + smtp_auth_username: '${MAIL_ACCOUNT}' + smtp_auth_password: '${MAIL_PASSWORD}' + smtp_require_tls: true + +route: + receiver: email + group_by: [alertname] + group_wait: 1m + group_interval: 10m + repeat_interval: 12h + routes: + - matchers: [severity="critical"] + receiver: email + repeat_interval: 3h + +receivers: + - name: email + email_configs: + - to: '${ALERT_EMAIL_TO}' + send_resolved: true + headers: + Subject: '[SafeFamily {{ .Status | toUpper }}] {{ .CommonLabels.alertname }} ({{ .Alerts | len }})' + +inhibit_rules: + # When the site is down because a container is down, only mail about the container. + - source_matchers: [alertname="ContainerDown"] + target_matchers: [alertname=~"WebsiteDown|WebsiteSlow|ScrapeTargetDown"] + # One mail per broken share: the explicit watch wins over the general alerts. + - source_matchers: [alertname="WatchedMountDown"] + target_matchers: [alertname=~"NetworkMountUnreachable|NetworkMountMissing"] + equal: [mountpoint] + # An unreachable share can also drop out of the filesystem list. + - source_matchers: [alertname="NetworkMountUnreachable"] + target_matchers: [alertname="NetworkMountMissing"] + equal: [mountpoint] + - source_matchers: [alertname="DiskSpaceCritical"] + target_matchers: [alertname=~"DiskSpaceLow|DiskWillFillIn24h"] + equal: [mountpoint] diff --git a/deploy/monitoring/alertmanager/entrypoint.sh b/deploy/monitoring/alertmanager/entrypoint.sh new file mode 100755 index 0000000..abb0b98 --- /dev/null +++ b/deploy/monitoring/alertmanager/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Render alertmanager.yml.tmpl with values from the environment, then start +# Alertmanager. Values are single-quoted in the template, so a ' inside a +# value is doubled to stay valid YAML. +set -eu + +# Unset or empty ALERT_EMAIL_TO: mail the sending account itself. +: "${ALERT_EMAIL_TO:=$MAIL_ACCOUNT}" +export ALERT_EMAIL_TO + +out=/tmp/alertmanager.yml +awk '{ + while (match($0, /\$\{[A-Z_]+\}/)) { + name = substr($0, RSTART + 2, RLENGTH - 3) + value = ENVIRON[name] + gsub(/'\''/, "'\'''\''", value) + $0 = substr($0, 1, RSTART - 1) value substr($0, RSTART + RLENGTH) + } + print +}' /etc/alertmanager/alertmanager.yml.tmpl > "$out" +chmod 600 "$out" + +exec /bin/alertmanager \ + --config.file="$out" \ + --storage.path=/alertmanager \ + "$@" diff --git a/deploy/monitoring/backup.env.example b/deploy/monitoring/backup.env.example new file mode 100644 index 0000000..6f57d5a --- /dev/null +++ b/deploy/monitoring/backup.env.example @@ -0,0 +1,20 @@ +# Copy to /etc/safefamily/backup.env (root-owned, mode 600). Read by +# deploy/systemd/safefamily-backup.service and safefamily-dirsize.service. +BACKUP_DIR=/var/backups/safefamily +BACKUP_AGE_RECIPIENTS=/etc/safefamily/backup-recipients.txt +BACKUP_KEEP_WEEKLY=8 +BACKUP_KEEP_MONTHLY=6 +# Let this group read the backups (they are encrypted), so you can scp/rsync +# them without sudo. Usually your login group: run `id -gn` to see it. +# BACKUP_GROUP=youruser +NODE_EXPORTER_TEXTFILE_DIR=/var/lib/node_exporter/textfile +# Optional: the repo checkout (the folder with docker-compose.yml), measured +# for Grafana's "Directory sizes" panel. Leave unset: dir_sizes.sh finds it +# from its own location. +# SAFEFAMILY_DIR=/home/you/SafeFamily +# Snap-installed Docker: +# DOCKER_ROOT_DIR=/var/snap/docker/common/var-lib-docker +# Network shares to watch (space separated mount points). Checked every 15 +# minutes by dir_sizes.sh; this also keeps a systemd automount mounted, so +# Grafana and the share alerts can see it after a reboot. +# WATCH_MOUNTS=/mnt/nas diff --git a/deploy/monitoring/blackbox/blackbox.yml b/deploy/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..c83f619 --- /dev/null +++ b/deploy/monitoring/blackbox/blackbox.yml @@ -0,0 +1,15 @@ +modules: + # Fetch the site through nginx inside the compose network. The Host header + # and TLS server name match server_name in deploy/nginx/nginx.docker.conf, so + # the real certificate is checked (expiry, hostname, chain). + safefamily_https: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: [200] + fail_if_not_ssl: true + headers: + Host: zzuse.duckdns.org + tls_config: + server_name: zzuse.duckdns.org diff --git a/deploy/monitoring/display-export/export.sh b/deploy/monitoring/display-export/export.sh new file mode 100755 index 0000000..5f9c2cf --- /dev/null +++ b/deploy/monitoring/display-export/export.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# Render the "SafeFamily Display" Grafana dashboard to a PNG for the P4 +# display, every DISPLAY_INTERVAL seconds (--once: a single run). +# +# Each run: Grafana's /render API (backed by the grafana-renderer service) +# -> a hidden temp file in the export folder -> check it is a PNG of the +# right size -> rename over one fixed file name, so readers never see a +# partial file and old files never pile up. A failed run leaves the last good +# picture in place; the display's STALE banner then shows it is old. +# +# GRAFANA_URL http://grafana:3000/grafana (Grafana's sub path) +# GRAFANA_USER / GRAFANA_PASSWORD a Grafana login (default admin user) +# DISPLAY_DASHBOARD_UID safefamily-display +# DISPLAY_WIDTH / DISPLAY_HEIGHT 1280 / 800 +# DISPLAY_TZ time zone for the timeline, e.g. America/Toronto +# DISPLAY_INTERVAL seconds between exports (300); after a failed +# export it retries after DISPLAY_RETRY (60) instead +# DISPLAY_FILE file name in the export folder (grafana-status.png) +# DISPLAY_EXPORT_REQUIRE_FSTYPE only write when the export folder is this +# filesystem type, e.g. cifs; "" = any +# EXPORT_DIR export folder inside the container (/export) +set -u + +GRAFANA_URL="${GRAFANA_URL:-http://grafana:3000/grafana}" +UID_="${DISPLAY_DASHBOARD_UID:-safefamily-display}" +WIDTH="${DISPLAY_WIDTH:-1280}" +HEIGHT="${DISPLAY_HEIGHT:-800}" +INTERVAL="${DISPLAY_INTERVAL:-300}" +RETRY="${DISPLAY_RETRY:-60}" +FILE="${DISPLAY_FILE:-grafana-status.png}" +DIR="${EXPORT_DIR:-/export}" +REQUIRE_FSTYPE="${DISPLAY_EXPORT_REQUIRE_FSTYPE-cifs}" +TMP="$DIR/.$FILE.tmp" + +log() { echo "[display-export] $(date '+%Y-%m-%d %H:%M:%S') $*" >&2; } + +# Filesystem type of the export folder, from this process's mount table. +fstype_of() { + awk -v m="$1" '{ for (i = 1; i <= NF; i++) if ($i == "-") { if ($5 == m) t = $(i + 1); break } } END { print t }' \ + /proc/self/mountinfo +} + +# PNG signature and IHDR width/height (big-endian, bytes 17-24). +png_size() { + sig="$(head -c 8 "$1" | od -An -tx1 | tr -d ' \n')" + [ "$sig" = "89504e470d0a1a0a" ] || return 1 + dims="$(head -c 24 "$1" | tail -c 8 | od -An -tu1)" + # shellcheck disable=SC2086 # split the eight byte values on purpose + set -- $dims + echo "$(( ($1 << 24) + ($2 << 16) + ($3 << 8) + $4 ))x$(( ($5 << 24) + ($6 << 16) + ($7 << 8) + $8 ))" +} + +export_once() { + if [ -n "$REQUIRE_FSTYPE" ]; then + actual="$(fstype_of "$DIR")" + case "|$REQUIRE_FSTYPE|" in + *"|$actual|"*) ;; + *) + log "not writing: $DIR is '${actual:-not a mount}', expected $REQUIRE_FSTYPE (share not mounted when this container started? docker compose restart display-export)" + return 1 ;; + esac + fi + url="$GRAFANA_URL/render/d/$UID_?orgId=1&width=$WIDTH&height=$HEIGHT&kiosk=true&theme=dark&timeout=60" + [ -n "${DISPLAY_TZ:-}" ] && url="$url&tz=$DISPLAY_TZ" + if ! curl -fsS --max-time 120 -u "${GRAFANA_USER:-admin}:${GRAFANA_PASSWORD:-}" -o "$TMP" "$url"; then + log "render request failed" + rm -f "$TMP" + return 1 + fi + size="$(png_size "$TMP")" || { log "Grafana did not return a PNG"; rm -f "$TMP"; return 1; } + if [ "$size" != "${WIDTH}x${HEIGHT}" ]; then + log "PNG is $size, expected ${WIDTH}x${HEIGHT}" + rm -f "$TMP" + return 1 + fi + mv -f "$TMP" "$DIR/$FILE" && log "wrote $DIR/$FILE ($size, $(wc -c < "$DIR/$FILE") bytes)" +} + +# Grafana answers /api/health once it is ready (no login needed). +wait_for_grafana() { + tries=0 + until curl -fsS --max-time 5 -o /dev/null "$GRAFANA_URL/api/health" 2> /dev/null; do + tries=$((tries + 1)) + [ "$tries" -eq 1 ] && log "waiting for Grafana at $GRAFANA_URL" + [ "$tries" -ge 60 ] && { log "Grafana still not ready after 5 minutes, trying anyway"; return 0; } + sleep 5 + done +} + +if [ "${1:-}" = "--once" ]; then + export_once + exit $? +fi +wait_for_grafana +while :; do + if export_once; then + sleep "$INTERVAL" + else + # Retry soon so one failure does not make the display go STALE. + sleep "$RETRY" + fi +done diff --git a/deploy/monitoring/display-export/make_display_dashboard.py b/deploy/monitoring/display-export/make_display_dashboard.py new file mode 100755 index 0000000..661d114 --- /dev/null +++ b/deploy/monitoring/display-export/make_display_dashboard.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Build the 1280x800 display dashboard from panels of the overview dashboard. + +Copies the Status and AdGuard panels (same queries, mappings and colors) into +deploy/monitoring/grafana/dashboards/safefamily-display.json, laid out to fill +one 1280x800 screenshot in kiosk mode. Rerun after changing those panels: + + python3 deploy/monitoring/display-export/make_display_dashboard.py +""" + +import copy +import json +from pathlib import Path + +DASHBOARDS = Path(__file__).resolve().parents[1] / "grafana" / "dashboards" + +# (title in the overview dashboard, x, y, w, h) on Grafana's 24-column grid. +LAYOUT = [ + ("Website", 0, 0, 6, 5), + ("Database", 6, 0, 6, 5), + ("App containers running", 12, 0, 6, 5), + ("Lowest disk free", 18, 0, 6, 5), + ("Page load time", 0, 5, 6, 5), + ("TLS cert expires in", 6, 5, 6, 5), + ("Last DB backup", 12, 5, 6, 5), + ("Database size", 18, 5, 6, 5), + ("AdGuard", 0, 10, 4, 5), + ("Protection", 4, 10, 4, 5), + ("Blocked now", 8, 10, 8, 5), + ("Blocked queries (24h)", 16, 10, 4, 5), + ("DNS queries (24h)", 20, 10, 4, 5), + ("Blocking timeline", 0, 15, 24, 5), +] + + +def main() -> None: + """Write the display dashboard JSON.""" + overview = json.loads((DASHBOARDS / "safefamily-overview.json").read_text()) + by_title = {} + for p in overview["panels"]: + # First panel with a title wins: "Database size" is both the Status + # tile and, further down, a graph. + if p.get("type") != "row": + by_title.setdefault(p["title"], p) + panels = [] + for panel_id, (title, x, y, w, h) in enumerate(LAYOUT, start=1): + panel = copy.deepcopy(by_title[title]) + panel["id"] = panel_id + panel["gridPos"] = {"x": x, "y": y, "w": w, "h": h} + panel["description"] = "" # no info icons on the display + if panel["type"] == "state-timeline": + # Rows are labelled already; the legend would only show thresholds. + panel["options"]["legend"] = {"showLegend": False} + panels.append(panel) + dashboard = { + "uid": "safefamily-display", + "title": "SafeFamily Display", + "tags": ["safefamily", "display"], + "timezone": "browser", + "schemaVersion": overview["schemaVersion"], + "version": 1, + "editable": False, + "refresh": "", + "time": {"from": "now-24h", "to": "now"}, + # No time picker bar: the whole 1280x800 is panels. + "timepicker": {"hidden": True}, + "panels": panels, + "templating": {"list": []}, + "annotations": {"list": []}, + } + out = DASHBOARDS / "safefamily-display.json" + out.write_text(json.dumps(dashboard, indent=2) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/deploy/monitoring/docker-compose.display-export.yml b/deploy/monitoring/docker-compose.display-export.yml new file mode 100644 index 0000000..ca67ea1 --- /dev/null +++ b/deploy/monitoring/docker-compose.display-export.yml @@ -0,0 +1,51 @@ +# Export the "SafeFamily Display" dashboard as a 1280x800 PNG for the ESP32-P4 +# display, into a folder the display reads (a writable Samba share). Add it +# after the monitoring overlay(s) in COMPOSE_FILE: +# +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml:deploy/monitoring/docker-compose.host-node-exporter.yml:deploy/monitoring/docker-compose.display-export.yml +# +# and set DISPLAY_EXPORT_DIR in .env. The exporter container is in the +# "display-export" profile: add COMPOSE_PROFILES=display-export to .env to run +# it, or leave that out and run the same script on the host +# (deploy/systemd/safefamily-display-export.*). See docs/monitoring.md +# "Status picture for the P4 display". + +services: + # Grafana's official headless-Chromium renderer; Grafana calls it for + # /render URLs. Internal only. + grafana-renderer: + image: grafana/grafana-image-renderer:v4.0.17 + container_name: safefamily_grafana_renderer + restart: always + environment: + TZ: ${DISPLAY_TZ:-UTC} + + grafana: + environment: + GF_RENDERING_SERVER_URL: http://grafana-renderer:8081/render + GF_RENDERING_CALLBACK_URL: http://grafana:3000/grafana/ + + display-export: + profiles: ["display-export"] + image: curlimages/curl:8.11.1 + container_name: safefamily_display_export + restart: always + depends_on: + - grafana + - grafana-renderer + # The share is mounted with uid=/gid= options, so only that user may write. + user: "${DISPLAY_EXPORT_UID:-1000}:${DISPLAY_EXPORT_GID:-1000}" + entrypoint: ["/bin/sh", "/display-export/export.sh"] + environment: + GRAFANA_URL: http://grafana:3000/grafana + GRAFANA_USER: ${GRAFANA_ADMIN_USER:-admin} + GRAFANA_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set in .env} + DISPLAY_TZ: ${DISPLAY_TZ:-} + DISPLAY_INTERVAL: ${DISPLAY_INTERVAL:-300} + DISPLAY_RETRY: ${DISPLAY_RETRY:-60} + DISPLAY_FILE: ${DISPLAY_FILE:-grafana-status.png} + # Refuse to write into the bare folder when the share is not mounted. + DISPLAY_EXPORT_REQUIRE_FSTYPE: ${DISPLAY_EXPORT_REQUIRE_FSTYPE-cifs} + volumes: + - ./deploy/monitoring/display-export:/display-export:ro + - ${DISPLAY_EXPORT_DIR:?set DISPLAY_EXPORT_DIR in .env to the folder the P4 display reads}:/export diff --git a/deploy/monitoring/docker-compose.host-node-exporter.yml b/deploy/monitoring/docker-compose.host-node-exporter.yml new file mode 100644 index 0000000..a61fc40 --- /dev/null +++ b/deploy/monitoring/docker-compose.host-node-exporter.yml @@ -0,0 +1,21 @@ +# Use a node-exporter installed on the host instead of the container. Needed +# when containers cannot see the host's real filesystem, e.g. snap-installed +# Docker (scripts/monitoring/preflight.sh --deep detects this). Add it after +# the monitoring overlay: +# +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml:deploy/monitoring/docker-compose.host-node-exporter.yml +# +# Host setup (docs/monitoring.md "Snap Docker"): +# sudo apt install prometheus-node-exporter +# ARGS="--collector.textfile.directory=/var/lib/node_exporter/textfile" +# in /etc/default/prometheus-node-exporter, then restart it. + +services: + # Only started when someone explicitly enables this profile, i.e. never. + node-exporter: + profiles: ["container-node-exporter"] + + prometheus: + extra_hosts: + # prometheus.yml scrapes node-exporter:9100; point that name at the host. + - "node-exporter:host-gateway" diff --git a/deploy/monitoring/docker-compose.monitoring.yml b/deploy/monitoring/docker-compose.monitoring.yml new file mode 100644 index 0000000..a3d7e77 --- /dev/null +++ b/deploy/monitoring/docker-compose.monitoring.yml @@ -0,0 +1,151 @@ +# Monitoring stack for SafeFamily: Prometheus + Alertmanager + Grafana and the +# exporters that feed them. Used as an overlay on the main compose file, so +# relative paths below resolve from the repository root: +# +# docker compose -f docker-compose.yml -f deploy/monitoring/docker-compose.monitoring.yml up -d +# +# or set COMPOSE_FILE once in .env and keep using plain `docker compose ...`: +# +# COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml +# +# Setup, alerts and dashboards are described in docs/monitoring.md. + +services: + prometheus: + image: prom/prometheus:v3.5.0 + container_name: safefamily_prometheus + restart: always + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d} + - --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE:-5GB} + volumes: + - ./deploy/monitoring/prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + ports: + # Localhost only; reach it through an SSH tunnel when debugging queries. + - "127.0.0.1:9090:9090" + + alertmanager: + image: prom/alertmanager:v0.28.1 + container_name: safefamily_alertmanager + restart: always + # Alertmanager cannot read env vars from its config, so render the + # template with the mail settings from .env before starting. + entrypoint: ["/bin/sh", "/etc/alertmanager/entrypoint.sh"] + environment: + MAIL_ACCOUNT: ${MAIL_ACCOUNT:?MAIL_ACCOUNT must be set in .env for alert emails} + MAIL_PASSWORD: ${MAIL_PASSWORD:?MAIL_PASSWORD must be set in .env for alert emails} + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-} # empty: send to MAIL_ACCOUNT + ALERT_SMTP_SMARTHOST: ${ALERT_SMTP_SMARTHOST:-smtp.gmail.com:587} + volumes: + - ./deploy/monitoring/alertmanager:/etc/alertmanager:ro + - alertmanager_data:/alertmanager + + grafana: + image: grafana/grafana:12.1.1 + container_name: safefamily_grafana + restart: always + depends_on: + - prometheus + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set in .env} + # Served at https:///grafana/ through nginx (home network only), + # and at http://127.0.0.1:3000/grafana/ directly. + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-https://zzuse.duckdns.org/grafana/} + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + volumes: + - ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: + # Localhost by default. Set GRAFANA_BIND=0.0.0.0 (and firewall it) to + # open it to your LAN. + - "${GRAFANA_BIND:-127.0.0.1}:3000:3000" + + # Host CPU, memory and free space of every mounted filesystem, including the + # one holding the Postgres volume. Also exports *.prom files written by + # scripts/db_backup.sh and scripts/monitoring/dir_sizes.sh. + node-exporter: + image: prom/node-exporter:v1.9.1 + container_name: safefamily_node_exporter + restart: always + pid: host + command: + - --path.rootfs=/host + - --collector.textfile.directory=/textfile + - --collector.filesystem.mount-points-exclude=^/(dev|proc|run|sys|var/lib/docker/.+|var/snap/docker/common/var-lib-docker/.+|snap/.+)($|/) + volumes: + - /:/host:ro + - ${NODE_EXPORTER_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}:/textfile:ro + + # Per-container CPU, memory, network and "is it running". + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.54.1 + container_name: safefamily_cadvisor + restart: always + privileged: true + command: + - --docker_only=true + # Docker 29+ with the containerd image store: cadvisor asks containerd + # for container layers. Set CONTAINERD_SOCKET in .env when the socket is + # not at the default path (scripts/monitoring/preflight.sh tells you). + # It must be under /run or /var/run, which is mounted below. + - --containerd=${CONTAINERD_SOCKET:-/run/containerd/containerd.sock} + - --housekeeping_interval=30s + - --store_container_labels=false + - --disable_metrics=percpu,sched,tcp,udp,disk,diskIO,hugetlb,referenced_memory,cpu_topology,resctrl + devices: + - /dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + # Snap-installed Docker keeps its data in + # /var/snap/docker/common/var-lib-docker; set DOCKER_ROOT_DIR to that. + - ${DOCKER_ROOT_DIR:-/var/lib/docker}:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + + # Database up/down, size, connections, transaction and cache stats. + postgres-exporter: + image: prometheuscommunity/postgres-exporter:v0.17.1 + container_name: safefamily_postgres_exporter + restart: always + depends_on: + - db + environment: + DATA_SOURCE_URI: db:5432/${DB_NAME:-logdb}?sslmode=disable + DATA_SOURCE_USER: ${DB_USER:-user} + DATA_SOURCE_PASS: ${DB_PASSWORD:-pass} + + # Nginx request rate and connection counts from the internal stub_status + # endpoint in deploy/nginx/nginx.docker.conf. + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.4.2 + container_name: safefamily_nginx_exporter + restart: always + depends_on: + - proxy + command: + - --nginx.scrape-uri=http://proxy:8080/stub_status + + # Loads the website through nginx like a browser would: up/down, response + # time, HTTP status and TLS certificate expiry. + blackbox-exporter: + image: prom/blackbox-exporter:v0.27.0 + container_name: safefamily_blackbox_exporter + restart: always + command: + - --config.file=/etc/blackbox/blackbox.yml + volumes: + - ./deploy/monitoring/blackbox:/etc/blackbox:ro + +volumes: + prometheus_data: + alertmanager_data: + grafana_data: diff --git a/deploy/monitoring/grafana/dashboards/safefamily-display.json b/deploy/monitoring/grafana/dashboards/safefamily-display.json new file mode 100644 index 0000000..7250107 --- /dev/null +++ b/deploy/monitoring/grafana/dashboards/safefamily-display.json @@ -0,0 +1,975 @@ +{ + "uid": "safefamily-display", + "title": "SafeFamily Display", + "tags": [ + "safefamily", + "display" + ], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": false, + "refresh": "", + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "panels": [ + { + "type": "stat", + "title": "Website", + "id": 1, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_success{job=\"website\"}", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Database", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "pg_up", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "App containers running", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(container_last_seen{name=~\"safefamily_(app|db|proxy)\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "green", + "value": 3 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Lowest disk free", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "green", + "value": 0.15 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Page load time", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 5, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_duration_seconds{job=\"website\"}", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "TLS cert expires in", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 6, + "y": 5, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_ssl_earliest_cert_expiry{job=\"website\"} - time()", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1209600 + }, + { + "color": "green", + "value": 2592000 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Last DB backup", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 12, + "y": 5, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - safefamily_backup_last_success_timestamp_seconds", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 691200 + }, + { + "color": "red", + "value": 864000 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Database size", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 18, + "y": 5, + "w": 6, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(pg_database_size_bytes{datname!~\"template.*|postgres\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "AdGuard", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 10, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_up", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Protection", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 4, + "y": 10, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_protection_enabled", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OFF", + "color": "red" + }, + "1": { + "text": "ON", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Blocked now", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 8, + "y": 10, + "w": 8, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "label_replace(safefamily_adguard_service_blocked, \"name\", \"$1 (service)\", \"service\", \"(.+)\") or label_replace(safefamily_adguard_domain_blocked, \"name\", \"$1\", \"domain\", \"(.+)\")", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ALLOWED", + "color": "blue" + }, + "1": { + "text": "BLOCKED", + "color": "purple" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "purple", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Blocked queries (24h)", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 16, + "y": 10, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_blocked_queries / safefamily_adguard_dns_queries", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "DNS queries (24h)", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 20, + "y": 10, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_dns_queries", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "state-timeline", + "title": "Blocking timeline", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 15, + "w": 24, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "label_replace(safefamily_adguard_service_blocked, \"name\", \"$1 (service)\", \"service\", \"(.+)\") or label_replace(safefamily_adguard_domain_blocked, \"name\", \"$1\", \"domain\", \"(.+)\")", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "showValue": "never", + "mergeValues": true, + "rowHeight": 0.8, + "alignValue": "center", + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "single" + } + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ALLOWED", + "color": "blue" + }, + "1": { + "text": "BLOCKED", + "color": "purple" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "purple", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/deploy/monitoring/grafana/dashboards/safefamily-overview.json b/deploy/monitoring/grafana/dashboards/safefamily-overview.json new file mode 100644 index 0000000..cdf2ca3 --- /dev/null +++ b/deploy/monitoring/grafana/dashboards/safefamily-overview.json @@ -0,0 +1,1919 @@ +{ + "uid": "safefamily-overview", + "title": "SafeFamily Overview", + "tags": [ + "safefamily" + ], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": false, + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "panels": [ + { + "type": "row", + "title": "Status", + "id": 1, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Website", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Loads https:///auth/login-ui through nginx, checking HTTP 200 and the TLS certificate.", + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_success{job=\"website\"}", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Database", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "pg_up", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "App containers running", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "app, db and proxy: 3 means all running.", + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(container_last_seen{name=~\"safefamily_(app|db|proxy)\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "green", + "value": 3 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Page load time", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_duration_seconds{job=\"website\"}", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "TLS cert expires in", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_ssl_earliest_cert_expiry{job=\"website\"} - time()", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1209600 + }, + { + "color": "green", + "value": 2592000 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Last DB backup", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Age of the last successful encrypted backup (weekly) written by scripts/db_backup.sh.", + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - safefamily_backup_last_success_timestamp_seconds", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 691200 + }, + { + "color": "red", + "value": 864000 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Database size", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(pg_database_size_bytes{datname!~\"template.*|postgres\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Lowest disk free", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "min(node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"})", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "green", + "value": 0.15 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + } + }, + { + "type": "table", + "title": "Firing alerts", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "alertstate": true + }, + "indexByName": { + "severity": 0, + "alertname": 1 + } + } + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No alerts firing" + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Disk space", + "id": 11, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Free space by mount", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs|nsfs\"}", + "legendFormat": "{{mountpoint}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0, + "max": 1 + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Directory sizes", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Written every 15 minutes by scripts/monitoring/dir_sizes.sh (app dir, logs, backups, Docker data).", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_dir_size_bytes", + "legendFormat": "{{path}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Database", + "id": 14, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Database size", + "id": 15, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 20 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "pg_database_size_bytes{datname!~\"template.*|postgres\"}", + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Connections", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 20 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (state) (pg_stat_activity_count)", + "legendFormat": "{{state}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(pg_settings_max_connections)", + "legendFormat": "max_connections", + "refId": "B" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Transactions / s", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 20 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(pg_stat_database_xact_commit{datname!~\"template.*\"}[5m]))", + "legendFormat": "commits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(pg_stat_database_xact_rollback{datname!~\"template.*\"}[5m]))", + "legendFormat": "rollbacks", + "refId": "B" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Containers", + "id": 18, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU", + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "1.0 = one full CPU core.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 29 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name=~\"safefamily_.*\"}[5m]))", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory", + "id": 20, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 29 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{name=~\"safefamily_.*\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Network in", + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 29 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (name) (rate(container_network_receive_bytes_total{name=~\"safefamily_.*\"}[5m]))", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Website & nginx", + "id": 22, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Requests / s", + "id": 23, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(nginx_http_requests_total[5m])", + "legendFormat": "requests", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Connections", + "id": 24, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nginx_connections_active", + "legendFormat": "active", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nginx_connections_reading", + "legendFormat": "reading", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nginx_connections_writing", + "legendFormat": "writing", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nginx_connections_waiting", + "legendFormat": "waiting (keep-alive)", + "refId": "D" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Page load phases", + "id": 25, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Time spent in DNS, connect, TLS, server processing and transfer for the probe request.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "probe_http_duration_seconds{job=\"website\"}", + "legendFormat": "{{phase}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Host", + "id": 26, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU busy", + "id": 27, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 47 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", + "legendFormat": "busy", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0, + "max": 1 + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory available", + "id": 28, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 47 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", + "legendFormat": "available", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0, + "max": 1 + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Load", + "id": 29, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 47 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load1", + "legendFormat": "1m", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load5", + "legendFormat": "5m", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load15", + "legendFormat": "15m", + "refId": "C" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "AdGuard", + "id": 30, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "panels": [] + }, + { + "type": "stat", + "title": "AdGuard", + "id": 31, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The AdGuard API answered with the login in /etc/safefamily/adguard.env (scripts/monitoring/adguard_check.py, every minute).", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_up", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Protection", + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_protection_enabled", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "OFF", + "color": "red" + }, + "1": { + "text": "ON", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Blocked now", + "id": 33, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Whether each watched service (ADGUARD_WATCH_SERVICES, the list SafeFamily switches) or domain (ADGUARD_WATCH_DOMAINS) is blocked right now.", + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "label_replace(safefamily_adguard_service_blocked, \"name\", \"$1 (service)\", \"service\", \"(.+)\") or label_replace(safefamily_adguard_domain_blocked, \"name\", \"$1\", \"domain\", \"(.+)\")", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ALLOWED", + "color": "blue" + }, + "1": { + "text": "BLOCKED", + "color": "purple" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "purple", + "value": 1 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Blocked queries (24h)", + "id": 34, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_blocked_queries / safefamily_adguard_dns_queries", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "DNS queries (24h)", + "id": 35, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 56 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "safefamily_adguard_dns_queries", + "legendFormat": "", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "state-timeline", + "title": "Blocking timeline", + "id": 36, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "When each watched domain was blocked or allowed: check that the SafeFamily schedule does what you expect.", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 60 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "label_replace(safefamily_adguard_service_blocked, \"name\", \"$1 (service)\", \"service\", \"(.+)\") or label_replace(safefamily_adguard_domain_blocked, \"name\", \"$1\", \"domain\", \"(.+)\")", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "options": { + "showValue": "never", + "mergeValues": true, + "rowHeight": 0.8, + "alignValue": "center", + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ALLOWED", + "color": "blue" + }, + "1": { + "text": "BLOCKED", + "color": "purple" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "purple", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..a57b2a8 --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: SafeFamily + folder: SafeFamily + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml b/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..00f9915 --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/deploy/monitoring/prometheus/alerts.yml b/deploy/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..a3210dd --- /dev/null +++ b/deploy/monitoring/prometheus/alerts.yml @@ -0,0 +1,247 @@ +groups: + - name: disk + rules: + - alert: DiskSpaceLow + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} < 0.15 + for: 10m + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.mountpoint }} has {{ $value | humanizePercentage }} free" + description: >- + Free space is below 15%. The Postgres volume, app logs and backups + all live on host disks; see the "Directory sizes" panel in Grafana. + + - alert: DiskSpaceCritical + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} < 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "Disk {{ $labels.mountpoint }} is almost full ({{ $value | humanizePercentage }} free)" + description: Postgres stops accepting writes when its disk fills up. + + - alert: DiskWillFillIn24h + expr: | + predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"}[6h], 24 * 3600) < 0 + and node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs|nsfs"} < 0.40 + for: 1h + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.mountpoint }} will be full within 24h at the current rate" + + - name: host + rules: + - alert: HostMemoryLow + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10 + for: 10m + labels: + severity: warning + annotations: + summary: "Host has only {{ $value | humanizePercentage }} memory available" + + - alert: HostHighLoad + expr: node_load15 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 1.5 + for: 30m + labels: + severity: warning + annotations: + summary: "15-minute load is {{ $value | humanize }}x the CPU count" + + - name: containers + rules: + - alert: ContainerDown + expr: | + absent(container_last_seen{name="safefamily_app"}) + or absent(container_last_seen{name="safefamily_db"}) + or absent(container_last_seen{name="safefamily_proxy"}) + for: 2m + labels: + severity: critical + annotations: + summary: "Container {{ $labels.name }} is not running" + description: "Check with: docker compose ps && docker compose logs --tail=100 {{ $labels.name }}" + + - alert: ContainerHighMemory + expr: container_memory_working_set_bytes{name=~"safefamily_(app|db|proxy)"} > 2 * 1024 * 1024 * 1024 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.name }} uses {{ $value | humanize1024 }}B of memory" + + - alert: ScrapeTargetDown + expr: up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus cannot scrape {{ $labels.job }} ({{ $labels.instance }})" + + - alert: NginxStatusUnavailable + # The exporter itself is up but cannot read nginx's stub_status page. + expr: nginx_up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: nginx-exporter cannot read nginx's status page + description: "Usually nginx runs an old copy of deploy/nginx/nginx.docker.conf: docker compose up -d --force-recreate proxy" + + - name: database + rules: + - alert: PostgresDown + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: Postgres is not answering the exporter + + - alert: PostgresTooManyConnections + expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections) > 0.8 + for: 10m + labels: + severity: warning + annotations: + summary: "Postgres is at {{ $value | humanizePercentage }} of max_connections" + + - alert: PostgresDatabaseGrowingFast + expr: delta(pg_database_size_bytes{datname!~"template.*|postgres"}[24h]) > 1024 * 1024 * 1024 + for: 1h + labels: + severity: warning + annotations: + summary: "Database {{ $labels.datname }} grew by {{ $value | humanize1024 }}B in 24h" + description: Usually the logs or media tables. Check disk headroom. + + - name: website + rules: + - alert: WebsiteDown + expr: probe_success{job="website"} == 0 + for: 3m + labels: + severity: critical + annotations: + summary: "{{ $labels.instance }} is failing (HTTP status or TLS check)" + description: "Last HTTP status: check probe_http_status_code in Grafana; then docker compose logs proxy app." + + - alert: WebsiteSlow + expr: probe_duration_seconds{job="website"} > 3 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} takes {{ $value | humanize }}s to load" + + - alert: TLSCertificateExpiringSoon + expr: probe_ssl_earliest_cert_expiry{job="website"} - time() < 14 * 24 * 3600 + for: 1h + labels: + severity: warning + annotations: + summary: "TLS certificate expires in {{ $value | humanizeDuration }}" + + - name: backup + rules: + - alert: DatabaseBackupMissing + expr: | + time() - safefamily_backup_last_success_timestamp_seconds > 8 * 24 * 3600 + or absent(safefamily_backup_last_success_timestamp_seconds) + for: 1h + labels: + severity: critical + annotations: + summary: No successful encrypted database backup in the last 8 days + description: "Check: systemctl status safefamily-backup.service and journalctl -u safefamily-backup.service" + + - alert: DatabaseBackupFailed + expr: safefamily_backup_last_run_success == 0 + labels: + severity: critical + annotations: + summary: The last database backup run failed + description: "See: journalctl -u safefamily-backup.service" + + - alert: MetricsFileUnreadable + expr: node_textfile_scrape_error == 1 + for: 15m + labels: + severity: warning + annotations: + summary: node-exporter cannot read a .prom file in its textfile directory + description: "Backup and directory-size metrics go missing. Check: docker logs safefamily_node_exporter | grep textfile" + + # Samba/NFS shares mounted on the host (node-exporter reports them like any + # other filesystem, so DiskSpaceLow & co. cover them too). + - name: network_mounts + rules: + - alert: NetworkMountUnreachable + expr: node_filesystem_device_error{fstype=~"cifs|smb3|nfs|nfs4"} == 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Network share {{ $labels.mountpoint }} is not responding" + description: "The file server is down, unreachable or rejected the credentials. Check: ls {{ $labels.mountpoint }}; sudo dmesg | grep -iE 'cifs|smb|nfs' | tail" + + - alert: NetworkMountMissing + # Mounted at some point in the last 7 days, not mounted now. The + # 20m allows for a reboot (an automounted share comes back when + # dir_sizes.sh wakes it with WATCH_MOUNTS). + expr: | + max_over_time(node_filesystem_size_bytes{fstype=~"cifs|smb3|nfs|nfs4"}[7d]) + unless on (instance, mountpoint) node_filesystem_size_bytes{fstype=~"cifs|smb3|nfs|nfs4"} + for: 20m + labels: + severity: warning + annotations: + summary: "Network share {{ $labels.mountpoint }} is no longer mounted" + description: "Remount with: sudo mount {{ $labels.mountpoint }} (or sudo mount -a). This alert stops on its own after 7 days unmounted." + + - alert: WatchedMountDown + # Mount points listed in WATCH_MOUNTS (backup.env), checked by + # dir_sizes.sh every 15 minutes: two failed checks in a row. + expr: safefamily_mount_ok == 0 + for: 20m + labels: + severity: warning + annotations: + summary: "Watched share {{ $labels.mountpoint }} is not mounted or not answering" + description: "Check: findmnt {{ $labels.mountpoint }}; ls {{ $labels.mountpoint }}; sudo systemctl status $(systemd-escape -p --suffix=automount {{ $labels.mountpoint }})" + + # From scripts/monitoring/adguard_check.py (safefamily-adguard.timer, every minute). + - name: adguard + rules: + - alert: AdGuardDown + expr: safefamily_adguard_up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: AdGuard Home is not answering, or its login failed + description: "DNS filtering and SafeFamily's blocking may not work. Check: journalctl -u safefamily-adguard.service -n 20" + + - alert: AdGuardProtectionDisabled + expr: safefamily_adguard_protection_enabled == 0 + for: 15m + labels: + severity: warning + annotations: + summary: AdGuard protection is switched off + description: No filtering or blocked services apply, so YouTube and the other blocks are off. Re-enable it in the AdGuard web UI. + + - alert: AdGuardCheckStale + expr: time() - safefamily_adguard_last_check_timestamp_seconds > 600 + for: 5m + labels: + severity: warning + annotations: + summary: The AdGuard check has not run for over 10 minutes + description: "Check: systemctl status safefamily-adguard.timer safefamily-adguard.service" diff --git a/deploy/monitoring/prometheus/prometheus.yml b/deploy/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..b6867f8 --- /dev/null +++ b/deploy/monitoring/prometheus/prometheus.yml @@ -0,0 +1,58 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + +rule_files: + - /etc/prometheus/alerts.yml + # Your own alert rules: drop *.yml files into rules.d/ (see its README). + - /etc/prometheus/rules.d/*.yml + +# Your own scrape targets: drop *.yml files into scrape.d/ (see its README). +scrape_config_files: + - /etc/prometheus/scrape.d/*.yml + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: node + static_configs: + - targets: ["node-exporter:9100"] + + - job_name: cadvisor + static_configs: + - targets: ["cadvisor:8080"] + metric_relabel_configs: + # Keep only named containers; drops cgroup slices and other noise. + - source_labels: [name] + regex: "" + action: drop + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: nginx + static_configs: + - targets: ["nginx-exporter:9113"] + + - job_name: website + metrics_path: /probe + params: + module: [safefamily_https] + static_configs: + - targets: + - https://proxy/auth/login-ui + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 diff --git a/deploy/monitoring/prometheus/rules.d/README.md b/deploy/monitoring/prometheus/rules.d/README.md new file mode 100644 index 0000000..6762e3b --- /dev/null +++ b/deploy/monitoring/prometheus/rules.d/README.md @@ -0,0 +1,27 @@ +# Extra alert rules + +Every `*.yml` file here is loaded as a Prometheus rule file (same format as +`../alerts.yml`). Alerts go out through the same Alertmanager email. Check and +reload: + +```bash +docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" \ + --entrypoint promtool prom/prometheus:v3.5.0 check rules /etc/prometheus/rules.d/my-rules.yml +docker compose restart prometheus +``` + +Example, `rules.d/samba.yml`: alert when a specific share is under 10% free +(general network-share alerts are already in `../alerts.yml`): + +```yaml +groups: + - name: my-samba + rules: + - alert: NasShareAlmostFull + expr: node_filesystem_avail_bytes{mountpoint="/mnt/nas"} / node_filesystem_size_bytes{mountpoint="/mnt/nas"} < 0.10 + for: 30m + labels: + severity: warning + annotations: + summary: "NAS share /mnt/nas has {{ $value | humanizePercentage }} free" +``` diff --git a/deploy/monitoring/prometheus/scrape.d/README.md b/deploy/monitoring/prometheus/scrape.d/README.md new file mode 100644 index 0000000..294fff7 --- /dev/null +++ b/deploy/monitoring/prometheus/scrape.d/README.md @@ -0,0 +1,23 @@ +# Extra scrape targets + +Every `*.yml` file here is loaded by Prometheus as a list of scrape configs +(`scrape_config_files` in `../prometheus.yml`). Add a file, then reload: + +```bash +docker compose restart prometheus +docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" \ + --entrypoint promtool prom/prometheus:v3.5.0 check config /etc/prometheus/prometheus.yml +``` + +Example, `scrape.d/router.yml` for an exporter on another machine: + +```yaml +scrape_configs: + - job_name: router + static_configs: + - targets: ["10.0.0.1:9100"] +``` + +An exporter on this host but outside Docker is reachable as +`host.docker.internal:` when the prometheus service has +`extra_hosts: ["host.docker.internal:host-gateway"]`. diff --git a/deploy/monitoring/prometheus/tests/adguard_test.yml b/deploy/monitoring/prometheus/tests/adguard_test.yml new file mode 100644 index 0000000..d53149b --- /dev/null +++ b/deploy/monitoring/prometheus/tests/adguard_test.yml @@ -0,0 +1,59 @@ +# Unit tests for the adguard alerts. Run from the repository root: +# docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" --entrypoint promtool \ +# prom/prometheus:v3.5.0 test rules /etc/prometheus/tests/adguard_test.yml +rule_files: + - ../alerts.yml + +evaluation_interval: 1m + +tests: + # AdGuard stops answering at 10m: a 3-minute blip is fine, 5 minutes is not. + - interval: 1m + input_series: + - series: 'safefamily_adguard_up{instance="host:9100", job="node"}' + values: '1x9 0x3 1x5 0x10' + alert_rule_test: + - eval_time: 13m + alertname: AdGuardDown + exp_alerts: [] + - eval_time: 25m + alertname: AdGuardDown + exp_alerts: + - exp_labels: {severity: critical, instance: "host:9100", job: node} + exp_annotations: + summary: AdGuard Home is not answering, or its login failed + description: "DNS filtering and SafeFamily's blocking may not work. Check: journalctl -u safefamily-adguard.service -n 20" + + # Protection switched off for 20 minutes. + - interval: 1m + input_series: + - series: 'safefamily_adguard_protection_enabled{instance="host:9100", job="node"}' + values: '1x5 0x20' + alert_rule_test: + - eval_time: 15m + alertname: AdGuardProtectionDisabled + exp_alerts: [] + - eval_time: 22m + alertname: AdGuardProtectionDisabled + exp_alerts: + - exp_labels: {severity: warning, instance: "host:9100", job: node} + exp_annotations: + summary: AdGuard protection is switched off + description: No filtering or blocked services apply, so YouTube and the other blocks are off. Re-enable it in the AdGuard web UI. + + # The timer stops: the last check timestamp freezes at 5m (300s). + - interval: 1m + input_series: + - series: 'safefamily_adguard_last_check_timestamp_seconds{instance="host:9100", job="node"}' + values: '0+60x5 300x30' + alert_rule_test: + - eval_time: 14m + alertname: AdGuardCheckStale + exp_alerts: [] + - eval_time: 21m + alertname: AdGuardCheckStale + exp_alerts: + - exp_labels: {severity: warning, instance: "host:9100", job: node} + exp_annotations: + summary: The AdGuard check has not run for over 10 minutes + description: "Check: systemctl status safefamily-adguard.timer safefamily-adguard.service" diff --git a/deploy/monitoring/prometheus/tests/network_mounts_test.yml b/deploy/monitoring/prometheus/tests/network_mounts_test.yml new file mode 100644 index 0000000..2513428 --- /dev/null +++ b/deploy/monitoring/prometheus/tests/network_mounts_test.yml @@ -0,0 +1,93 @@ +# Unit tests for the network_mounts alerts. Run from the repository root: +# docker run --rm -v "$PWD/deploy/monitoring/prometheus:/etc/prometheus:ro" --entrypoint promtool \ +# prom/prometheus:v3.5.0 test rules /etc/prometheus/tests/network_mounts_test.yml +rule_files: + - ../alerts.yml + +evaluation_interval: 1m + +tests: + # The share is mounted for 30 minutes, then disappears. + - interval: 1m + input_series: + - series: 'node_filesystem_size_bytes{instance="host:9100", job="node", device="//nas/share", fstype="cifs", mountpoint="/mnt/nas"}' + values: '1e12x30 _x60' + - series: 'node_filesystem_size_bytes{instance="host:9100", job="node", device="/dev/sda1", fstype="ext4", mountpoint="/"}' + values: '1e11x90' + alert_rule_test: + - eval_time: 20m + alertname: NetworkMountMissing + exp_alerts: [] + # Gone for 14 minutes: still within the 20m allowance (e.g. a reboot). + - eval_time: 45m + alertname: NetworkMountMissing + exp_alerts: [] + - eval_time: 60m + alertname: NetworkMountMissing + exp_alerts: + - exp_labels: + severity: warning + instance: host:9100 + job: node + device: //nas/share + fstype: cifs + mountpoint: /mnt/nas + exp_annotations: + summary: "Network share /mnt/nas is no longer mounted" + description: "Remount with: sudo mount /mnt/nas (or sudo mount -a). This alert stops on its own after 7 days unmounted." + + # The file server stops answering: node-exporter reports a device error. + - interval: 1m + input_series: + - series: 'node_filesystem_device_error{instance="host:9100", job="node", device="//nas/share", fstype="cifs", mountpoint="/mnt/nas"}' + values: '0x10 1x20' + alert_rule_test: + - eval_time: 12m + alertname: NetworkMountUnreachable + exp_alerts: [] + - eval_time: 20m + alertname: NetworkMountUnreachable + exp_alerts: + - exp_labels: + severity: warning + instance: host:9100 + job: node + device: //nas/share + fstype: cifs + mountpoint: /mnt/nas + exp_annotations: + summary: "Network share /mnt/nas is not responding" + description: "The file server is down, unreachable or rejected the credentials. Check: ls /mnt/nas; sudo dmesg | grep -iE 'cifs|smb|nfs' | tail" + + # A local disk never triggers either alert. + - interval: 1m + input_series: + - series: 'node_filesystem_size_bytes{instance="host:9100", job="node", device="/dev/sdb1", fstype="ext4", mountpoint="/data"}' + values: '1e11x10 _x30' + alert_rule_test: + - eval_time: 30m + alertname: NetworkMountMissing + exp_alerts: [] + + # WATCH_MOUNTS check: the file is rewritten every 15m but scraped every + # 30s, so Prometheus sees a continuous value. The share fails at 30m: the + # next check (45m) is still inside the 20m allowance, the alert fires by 60m. + - interval: 1m + input_series: + - series: 'safefamily_mount_ok{instance="host:9100", job="node", mountpoint="/mnt/nas"}' + values: '1x29 0x40' + alert_rule_test: + - eval_time: 45m + alertname: WatchedMountDown + exp_alerts: [] + - eval_time: 60m + alertname: WatchedMountDown + exp_alerts: + - exp_labels: + severity: warning + instance: host:9100 + job: node + mountpoint: /mnt/nas + exp_annotations: + summary: "Watched share /mnt/nas is not mounted or not answering" + description: "Check: findmnt /mnt/nas; ls /mnt/nas; sudo systemctl status $(systemd-escape -p --suffix=automount /mnt/nas)" diff --git a/deploy/nginx/nginx.docker.conf b/deploy/nginx/nginx.docker.conf index 89904b6..cbfa45d 100644 --- a/deploy/nginx/nginx.docker.conf +++ b/deploy/nginx/nginx.docker.conf @@ -1,6 +1,12 @@ # Rate limiting limit_req_zone $binary_remote_addr zone=api_limit:10m rate=5r/s; +# WebSocket upgrade for Grafana Live (/grafana/api/live/ws). +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + # PUBLIC HTTP SERVER (Redirect to HTTPS) server { listen 80; @@ -28,4 +34,54 @@ server { proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + + # GRAFANA (deploy/monitoring/), home network only: anything else gets 403. + # Grafana serves from /grafana/ itself (GF_SERVER_SERVE_FROM_SUB_PATH), so + # the path is passed through unchanged. No rate limit: one dashboard page + # loads dozens of files at once. + location = /grafana { + return 301 /grafana/; + } + location /grafana/ { + allow 10.0.0.0/8; + allow 192.168.0.0/16; + allow 127.0.0.1; + deny all; + + # Look Grafana up per request (Docker DNS), so nginx still starts when + # the monitoring stack is not running; then this returns 502. + resolver 127.0.0.11 valid=30s ipv6=off; + set $grafana_upstream http://grafana:3000; + proxy_pass $grafana_upstream; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} + +# INTERNAL STATUS SERVER (scraped by nginx-exporter, see deploy/monitoring/). +# Port 8080 is not published in docker-compose.yml, so it is reachable only +# from containers on the compose network. +server { + listen 8080; + server_name _; + access_log off; + + location = /stub_status { + stub_status; + allow 127.0.0.1; + allow 10.0.0.0/8; + allow 172.16.0.0/12; + allow 192.168.0.0/16; + deny all; + } + + location / { + return 404; + } } diff --git a/deploy/systemd/safefamily-adguard.service b/deploy/systemd/safefamily-adguard.service new file mode 100644 index 0000000..2e9a48f --- /dev/null +++ b/deploy/systemd/safefamily-adguard.service @@ -0,0 +1,16 @@ +# AdGuard Home health and "is YouTube blocked" metrics for Grafana. Install +# (edit the path first if the checkout is not /srv/SafeFamily): +# sudo cp deploy/systemd/safefamily-adguard.* /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-adguard.timer +[Unit] +Description=SafeFamily AdGuard check +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Holds the AdGuard login: root-only (chmod 600). +EnvironmentFile=/etc/safefamily/adguard.env +ExecStart=/usr/bin/python3 /srv/SafeFamily/scripts/monitoring/adguard_check.py +TimeoutStartSec=2min +Nice=10 diff --git a/deploy/systemd/safefamily-adguard.timer b/deploy/systemd/safefamily-adguard.timer new file mode 100644 index 0000000..faebe22 --- /dev/null +++ b/deploy/systemd/safefamily-adguard.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Check AdGuard Home every minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=10s + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/safefamily-backup.service b/deploy/systemd/safefamily-backup.service new file mode 100644 index 0000000..105373c --- /dev/null +++ b/deploy/systemd/safefamily-backup.service @@ -0,0 +1,15 @@ +# Encrypted weekly database backup. Install (edit the path first if your +# checkout is not /srv/SafeFamily): +# sudo cp deploy/systemd/safefamily-backup.* /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-backup.timer +[Unit] +Description=SafeFamily encrypted database backup +Wants=docker.service snap.docker.dockerd.service +After=docker.service snap.docker.dockerd.service + +[Service] +Type=oneshot +EnvironmentFile=-/etc/safefamily/backup.env +ExecStart=/srv/SafeFamily/scripts/db_backup.sh +Nice=10 +IOSchedulingClass=idle diff --git a/deploy/systemd/safefamily-backup.timer b/deploy/systemd/safefamily-backup.timer new file mode 100644 index 0000000..d171ae8 --- /dev/null +++ b/deploy/systemd/safefamily-backup.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run the SafeFamily database backup weekly + +[Timer] +# Sundays at 03:17. Change the day/time here, then: sudo systemctl daemon-reload +OnCalendar=Sun *-*-* 03:17 +RandomizedDelaySec=15m +# Run at next boot if the machine was off at that time. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/safefamily-dirsize.service b/deploy/systemd/safefamily-dirsize.service new file mode 100644 index 0000000..9f4c602 --- /dev/null +++ b/deploy/systemd/safefamily-dirsize.service @@ -0,0 +1,15 @@ +# Directory sizes for Grafana. Install alongside safefamily-backup.* (same +# path edit applies): +# sudo cp deploy/systemd/safefamily-dirsize.* /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-dirsize.timer +[Unit] +Description=SafeFamily directory size metrics + +[Service] +Type=oneshot +EnvironmentFile=-/etc/safefamily/backup.env +ExecStart=/srv/SafeFamily/scripts/monitoring/dir_sizes.sh +# A hung network share must not keep this running forever. +TimeoutStartSec=5min +Nice=19 +IOSchedulingClass=idle diff --git a/deploy/systemd/safefamily-dirsize.timer b/deploy/systemd/safefamily-dirsize.timer new file mode 100644 index 0000000..248d5de --- /dev/null +++ b/deploy/systemd/safefamily-dirsize.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Measure SafeFamily directory sizes every 15 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/safefamily-display-export.service b/deploy/systemd/safefamily-display-export.service new file mode 100644 index 0000000..270d9dc --- /dev/null +++ b/deploy/systemd/safefamily-display-export.service @@ -0,0 +1,21 @@ +# Fallback for deploy/monitoring/docker-compose.display-export.yml when the +# export folder cannot be mounted into a container (e.g. snap Docker): run the +# same export script on the host. Grafana and grafana-renderer still run in +# Docker. Settings in /etc/safefamily/display-export.env (root-only: it holds +# the Grafana password). Edit the path if the checkout is not /srv/SafeFamily. +# sudo cp deploy/systemd/safefamily-display-export.* /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-display-export.timer +[Unit] +Description=SafeFamily status picture for the P4 display +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Write as the user the share is mounted for (uid= in /etc/fstab). +User=1000 +Group=1000 +EnvironmentFile=/etc/safefamily/display-export.env +Environment=GRAFANA_URL=http://127.0.0.1:3000/grafana +ExecStart=/bin/sh /srv/SafeFamily/deploy/monitoring/display-export/export.sh --once +TimeoutStartSec=3min diff --git a/deploy/systemd/safefamily-display-export.timer b/deploy/systemd/safefamily-display-export.timer new file mode 100644 index 0000000..31725bb --- /dev/null +++ b/deploy/systemd/safefamily-display-export.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Export the SafeFamily status picture every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min + +[Install] +WantedBy=timers.target diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..7694f54 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,473 @@ +# Monitoring and encrypted backups + +What runs, how to turn it on, what each alert means, and how to back up and +restore the database. For a step-by-step setup with checks, routine tasks and +troubleshooting, follow [operations-checklist.md](operations-checklist.md); +`scripts/monitoring/preflight.sh` verifies the host first. + +``` + ┌──────────── host ─────────────┐ + systemd │ db_backup.sh ─┐ dir_sizes.sh│──► /var/lib/node_exporter/textfile/*.prom + timers └────────────────┼──────────────┘ │ + ▼ ▼ + safefamily_db ◄── postgres-exporter ──┐ node-exporter (disk, CPU, RAM) + safefamily_proxy ◄── nginx-exporter ──┤ cadvisor (per-container) + safefamily_proxy ◄── blackbox (HTTPS)─┤ │ + ▼ ▼ + Prometheus ──► Alertmanager ──► email (Gmail SMTP) + │ + ▼ + Grafana (https:///grafana/, LAN only) +``` + +| Question | Answered by | +|---|---| +| Is a disk filling up? (repo dir, logs, Docker volumes incl. Postgres) | node-exporter filesystems + `dir_sizes.sh` | +| How big is the database, how many connections, is it up? | postgres-exporter | +| Are the app, db and proxy containers running, how much CPU/RAM? | cadvisor | +| Does the website load, how fast, when does the TLS cert expire? | blackbox-exporter | +| How many requests/connections is nginx handling? | nginx-exporter (`stub_status`) | +| Did last night's encrypted backup succeed? | `db_backup.sh` metrics | + +Postgres has no "free space" of its own: it grows until the filesystem under +the Docker volume is full. So "database free space" means the free space of +the filesystem holding Docker's data dir (usually `/`), plus the database size +trend and the `DiskWillFillIn24h` prediction. + +## 1. Turn on the monitoring stack + +Add to `.env` (the alert mails reuse the app's `MAIL_ACCOUNT` / `MAIL_PASSWORD` +Gmail app password): + +```bash +GRAFANA_ADMIN_PASSWORD= +# Optional: +# ALERT_EMAIL_TO=you@example.com # default: MAIL_ACCOUNT +# GRAFANA_BIND=0.0.0.0 # default: 127.0.0.1 (localhost only) +# DOCKER_ROOT_DIR=/var/snap/docker/common/var-lib-docker # snap-installed Docker +# CONTAINERD_SOCKET=/run/snap.docker/containerd/containerd.sock # if preflight says so +# Make plain `docker compose ...` include the monitoring services: +COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml +``` + +Then: + +```bash +sudo mkdir -p /var/lib/node_exporter/textfile +docker compose up -d # starts the new services, recreates proxy for stub_status +docker compose ps # all safefamily_* should be "Up" +``` + +Open Grafana at **** from any device on +your home network (nginx answers 403 to everything else, see "Grafana from +the LAN" below), or at on the server itself +(from elsewhere: `ssh -L 3000:127.0.0.1:3000 server`). Log in as `admin`, and open +**Dashboards → SafeFamily → SafeFamily Overview**. (`GRAFANA_ADMIN_PASSWORD` +only sets the password on Grafana's first start; change it later in the UI.) Prometheus itself is at + (`/alerts` shows every rule and its state). + +### Grafana from the LAN + +nginx (`deploy/nginx/nginx.docker.conf`, `location /grafana/`) passes +`/grafana/` to Grafana only for home-network addresses (`10.0.0.0/8`, +`192.168.0.0/16`) and answers **403** to everything else, so it stays closed +to the internet although the site is public; Grafana's own login comes on +top. After pulling this change: `docker compose up -d` (Grafana now serves +from `/grafana/`) and `docker compose up -d --force-recreate proxy` (new +nginx config). Check from a phone on mobile data that +`https://zzuse.duckdns.org/grafana/` gives 403. + +- If your LAN uses another range (e.g. `172.16.x.x`), add an `allow` line for + it in that location. Do not add `172.16.0.0/12` just to make it work: + that is also Docker's own range, so it can let outside traffic through. +- If your laptop gets 403 at home, see which address nginx saw: + `docker compose logs --tail 5 proxy` (first field of each line). The router + (`10.0.0.1`) appearing there is fine: that is its loopback NAT. +- Set `GRAFANA_ROOT_URL` in `.env` if the site name is not `zzuse.duckdns.org`. +- `GRAFANA_BIND` (direct port 3000) can stay at `127.0.0.1`. + +For deeper views, import community dashboards in Grafana (Dashboards → New → +Import → ID): **1860** Node Exporter Full, **14282** cAdvisor, **9628** +PostgreSQL, **12708** NGINX exporter. + +### Host-specific notes + +- **Self-signed TLS certificate**: the website probe verifies the certificate, + so it reports DOWN with a self-signed one. Either use a real certificate + (Let's Encrypt works with duckdns), or add `insecure_skip_verify: true` under + `tls_config` in `deploy/monitoring/blackbox/blackbox.yml`. +- **cadvisor without container + names** ("App containers running" shows No data, container panels empty, + `ContainerDown` firing although everything runs): cadvisor v0.54 asks + containerd for container details with **any** image store, classic + `overlay2` included, at `/run/containerd/containerd.sock` by default. If + `docker logs safefamily_cadvisor` says `unable to create containerd client` + (or the container panels are empty), set `CONTAINERD_SOCKET` in `.env` to the + real socket; `scripts/monitoring/preflight.sh` finds it and prints the exact + line (for example `CONTAINERD_SOCKET=/run/snap.docker/containerd/containerd.sock` + for snap Docker). Then `docker compose up -d cadvisor`. +- **Snap Docker**: snap's confinement can give containers the snap's own + small filesystem instead of the host's `/` (`preflight.sh --deep`: "containers + see a different /"), which makes every disk number and disk alert wrong, and + can refuse the textfile mount ("read-only file system"). Then run + node-exporter **on the host** instead; the dashboard, alerts and backup + metrics work unchanged: + ```bash + sudo apt install prometheus-node-exporter + # /etc/default/prometheus-node-exporter: read the same dir the scripts write to + sudo sed -i 's#^ARGS=.*#ARGS="--collector.textfile.directory=/var/lib/node_exporter/textfile"#' \ + /etc/default/prometheus-node-exporter + sudo mkdir -p /var/lib/node_exporter/textfile + sudo systemctl restart prometheus-node-exporter + curl -s localhost:9100/metrics | grep -c '^node_filesystem_size_bytes' # > 0 + + # .env: append the host overlay to COMPOSE_FILE + COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml:deploy/monitoring/docker-compose.host-node-exporter.yml + + docker rm -f safefamily_node_exporter # the old container, if it exists + docker compose up -d + ``` + The overlay disables the node-exporter container and points Prometheus's + `node-exporter` name at the host. If `ufw` is active, let containers reach + it: `sudo ufw allow from 172.16.0.0/12 to any port 9100 proto tcp` (port + 9100 needs no opening to the internet). Keep `NODE_EXPORTER_TEXTFILE_DIR` at + its default; the `/var/snap/...` textfile dir suggested for snap does not + apply in this mode. +- **Snap Docker, textfile dir only**: if `--deep` reports the real `/` but + refuses the textfile mount, either use host mode above or a directory in + snap Docker's own data area, set in **both** `.env` and + `/etc/safefamily/backup.env`: + `NODE_EXPORTER_TEXTFILE_DIR=/var/snap/docker/common/node_exporter_textfile` + (create it first), then `docker compose up -d node-exporter`. + +## 2. Alerts + +Rules live in `deploy/monitoring/prometheus/alerts.yml`; emails are sent by +Alertmanager (critical alerts repeat every 3h, warnings every 12h, and a +"resolved" mail follows each). + +| Alert | Fires when | First thing to check | +|---|---|---| +| DiskSpaceLow / DiskSpaceCritical | a filesystem is under 15% / 5% free | "Directory sizes" panel; prune per README "Prune Cache" | +| DiskWillFillIn24h | the 6h trend reaches 0 within a day | same, plus `PostgresDatabaseGrowingFast` | +| ContainerDown | app, db or proxy not running for 2 min | `docker compose ps`, `docker compose logs --tail=100 ` | +| ContainerHighMemory | a container uses > 2 GiB for 15 min | `docker stats` | +| PostgresDown | the exporter cannot query Postgres | `docker compose logs db` | +| PostgresTooManyConnections | > 80% of `max_connections` | Gunicorn workers × pool size | +| PostgresDatabaseGrowingFast | > 1 GiB growth in 24h | usually `logs` or `media` tables | +| WebsiteDown | HTTPS check of `/auth/login-ui` fails for 3 min | `docker compose logs proxy app` | +| WebsiteSlow | page load > 3s for 10 min | "Page load phases" panel | +| TLSCertificateExpiringSoon | cert expires in < 14 days | renew the certificate | +| DatabaseBackupMissing | no successful backup for 8 days (backups are weekly) | `journalctl -u safefamily-backup.service` | +| DatabaseBackupFailed | the last backup run failed | same | +| MetricsFileUnreadable | node-exporter cannot read a `.prom` file | `docker logs safefamily_node_exporter` | +| ScrapeTargetDown | an exporter is unreachable for 5 min | `docker compose ps` | +| AdGuardDown | the AdGuard API does not answer or rejects the login for 5 min | `journalctl -u safefamily-adguard.service -n 20` | +| AdGuardProtectionDisabled | AdGuard protection is off for 15 min | re-enable it in the AdGuard web UI | +| AdGuardCheckStale | the AdGuard check has not run for 10 min | `systemctl status safefamily-adguard.timer` | +| NginxStatusUnavailable | nginx-exporter cannot read nginx's status page | `docker compose up -d --force-recreate proxy` | +| HostMemoryLow / HostHighLoad | < 10% RAM available / load > 1.5× CPUs | `top`, `docker stats` | +| NetworkMountUnreachable | a Samba/NFS share stops answering for 5 min | `ls `; `sudo dmesg \| grep -iE 'cifs\|smb\|nfs' \| tail` | +| NetworkMountMissing | a share mounted in the last 7 days is gone for 20 min | `sudo mount ` or `sudo mount -a` | +| WatchedMountDown | a `WATCH_MOUNTS` share failed two checks (15 min apart) | `findmnt `; `ls ` | + +Test the mail path once after setup: + +```bash +docker exec safefamily_alertmanager amtool alert add TestAlert severity=warning \ + --annotation='summary="SafeFamily alert test"' --alertmanager.url=http://localhost:9093 +``` + +## 3. Encrypted database backups + +`scripts/db_backup.sh` makes a **full** dump (every table, including logs and +media; `scripts/db_export.py` is the sanitized, schema-oriented export) and +encrypts it with [age](https://age-encryption.org) (X25519 + +ChaCha20-Poly1305). It encrypts to a **public key**: the server can create +backups but cannot read them. Only the private key, kept off the server, can +decrypt, so a stolen server, disk, or offsite copy reveals nothing. + +Each run: `pg_dump --format=custom` inside the db container → `pg_restore +--list` to prove the dump is readable → `age` encrypt → atomic rename into +`$BACKUP_DIR/weekly/`. Runs **weekly** (Sundays 03:17, or at the next boot if +the server was off). The first backup of each month is also hard-linked into +`monthly/` at no extra disk cost. Keeps the newest **8 weekly** (two months) +and **6 monthly** (half a year) backups. Files are root-only unless +`BACKUP_GROUP` names a group allowed to read them (safe: they are +encrypted), which lets you `scp` them or copy them offsite without sudo. +Change retention with `BACKUP_KEEP_WEEKLY` / +`BACKUP_KEEP_MONTHLY` in `/etc/safefamily/backup.env`. Retention counts +backups rather than days, so if backups fail for a long time the last good +ones are never pruned. Writes metrics for the backup alerts and the "Last DB +backup" panel. To change the day or time, edit `OnCalendar=` in +`/etc/systemd/system/safefamily-backup.timer` and run `sudo systemctl daemon-reload`. + +### One-time setup + +On **your laptop or another machine that is not the server**: + +```bash +sudo apt install age # macOS: brew install age +age-keygen -o safefamily-backup-key.txt +# prints: Public key: age1... +# Optional: passphrase-protect the private key file too +age -p -o safefamily-backup-key.txt.age safefamily-backup-key.txt && shred -u safefamily-backup-key.txt +``` + +Store the private key in a password manager **and** on paper or a USB stick. +Without it the backups are unrecoverable; that is the point. + +On the **server**: + +```bash +sudo apt install age +sudo mkdir -p /etc/safefamily /var/backups/safefamily /var/lib/node_exporter/textfile +echo 'age1...your public key...' | sudo tee /etc/safefamily/backup-recipients.txt +sudo cp deploy/monitoring/backup.env.example /etc/safefamily/backup.env +sudo chmod 600 /etc/safefamily/backup.env # edit paths if needed + +# Edit ExecStart in both .service files if the checkout is not /srv/SafeFamily +sudo cp deploy/systemd/safefamily-backup.* deploy/systemd/safefamily-dirsize.* /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now safefamily-backup.timer safefamily-dirsize.timer + +sudo systemctl start safefamily-backup.service # first backup now +journalctl -u safefamily-backup.service -n 20 +ls -l /var/backups/safefamily/weekly/ +``` + +The recipients file may hold several public keys (one per line), e.g. one for +you and one for a family member; any one of them can decrypt. The script +refuses to run if the file contains a private key. + +Cron instead of systemd (Sundays 03:17): `17 3 * * 0 set -a; . /etc/safefamily/backup.env; /srv/SafeFamily/scripts/db_backup.sh` + +### Offsite copies + +Point your existing backup scripts at `/var/backups/safefamily/` (it is already +encrypted, so any cloud drive or USB disk is fine). For example: +`rclone copy /var/backups/safefamily remote:safefamily-backups`, or +`rsync -a /var/backups/safefamily/ nas:/backups/safefamily/`. + +### Restore + +Practise once now, and again every few months. A restore drill goes into a +scratch database (`safefamily_restore_test`) and never touches live data: + +```bash +# On the server, with the key temporarily copied over: +scripts/db_restore.sh -i safefamily-backup-key.txt /var/backups/safefamily/weekly/safefamily-.dump.age + +# Or keep the key on your laptop and stream the decrypted dump: +age -d -i safefamily-backup-key.txt safefamily-.dump.age \ + | ssh server 'cd /srv/SafeFamily && scripts/db_restore.sh -' +``` + +It prints the row count of every table. Drop the scratch DB afterwards with +`scripts/db_restore.sh --drop-test`. + +Real disaster recovery (replaces the live data): + +```bash +docker compose stop app +scripts/db_restore.sh -i safefamily-backup-key.txt --into-live safefamily-.dump.age +# (type the database name to confirm; when streaming over ssh use --confirm ) +docker compose start app +``` + +On a brand-new host, bring up `db` first (`docker compose up -d db`), then +restore with `--into-live`. + +## 4. Adding more monitoring sources + +Pick the row that fits; none of them needs edits to the tracked config files. + +| You want to watch | Do this | +|---|---| +| A disk, USB drive or **Samba/NFS share mounted on the host** | Nothing: node-exporter reports every mounted filesystem. See below. | +| Another website or URL | Add it to `targets` of the `website` job in `prometheus/prometheus.yml`, or a blackbox job in `scrape.d/` | +| Anything that has a Prometheus exporter (router, NAS, another server, a database) | Run the exporter, then add a scrape file in `deploy/monitoring/prometheus/scrape.d/` (README there) | +| A custom check (a script, a file's age, a share being writable) | Write a script that prints metrics into `/var/lib/node_exporter/textfile/.prom` from a timer, like `scripts/monitoring/dir_sizes.sh` does | +| An alert on any of these | Add a rule file in `deploy/monitoring/prometheus/rules.d/` (README there) | +| A Grafana panel | Build it in Grafana (dashboard → Make editable → Save as a new dashboard), or drop a dashboard JSON into `grafana/dashboards/` | + +After adding scrape or rule files: `docker compose restart prometheus`, then +check Prometheus's targets page () or rerun +`scripts/monitoring/preflight.sh`. + +### AdGuard Home and "is YouTube blocked?" + +`scripts/monitoring/adguard_check.py` runs on the host every minute +(`safefamily-adguard.timer`), logs in to the AdGuard API and writes metrics +for the **AdGuard** row in Grafana: AdGuard up, protection on, 24h query +and blocked counts, and two kinds of "blocked right now": + +- **Services** (`ADGUARD_WATCH_SERVICES`, default `youtube`): whether the id + is in AdGuard's blocked-services list, the list SafeFamily switches. Fast + and local; this is the reliable YouTube signal. +- **Domains** (`ADGUARD_WATCH_DOMAINS`, default `youtube.com`): AdGuard's + `check_host` verdict, which also covers filter rules and parental control. + On a busy router, or when Safe Browsing / Parental Control cannot reach + AdGuard's servers (for example while SafeFamily has stopped all traffic), + it can take longer than `ADGUARD_TIMEOUT` (30 s); the journal then says + `check_host ... failed: timed out`. Raise the timeout, or set + `ADGUARD_WATCH_DOMAINS=""` and rely on the service check. +The **Blocking timeline** panel shows when each domain was blocked or allowed, +which is how you check that the SafeFamily schedule does what you expect. + +```bash +sudo install -m 600 deploy/monitoring/adguard.env.example /etc/safefamily/adguard.env +sudo nano /etc/safefamily/adguard.env # same ADGUARD_* values as the app's .env +sudo cp deploy/systemd/safefamily-adguard.* /etc/systemd/system/ +sudo sed -i 's#/srv/SafeFamily#'"$PWD"'#' /etc/systemd/system/safefamily-adguard.service +sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-adguard.timer +journalctl -u safefamily-adguard.service -n 5 # "service youtube: blocked" / "allowed" +docker compose restart prometheus && docker compose up -d grafana +``` + +The password lives only in that root-only file (preflight checks its +permissions), never in Prometheus or Grafana. + +YouTube being allowed is normal at the times your schedule allows it, so +there is no built-in alert for it. To be told when it is allowed outside +those times, add a rule in `prometheus/rules.d/`, for example +`rules.d/youtube.yml` (Prometheus times are UTC: adjust the hours): + +```yaml +groups: + - name: youtube-schedule + rules: + - alert: YouTubeAllowedAtNight + # 21:00-06:00 UTC + expr: safefamily_adguard_service_blocked{service="youtube"} == 0 and on() (hour() >= 21 or hour() < 6) + for: 15m + labels: + severity: warning + annotations: + summary: YouTube is not blocked although it is night +``` + +### Samba share mounted on the host + +node-exporter (on the host, see "Snap Docker" above) measures the share like +any filesystem, so it already appears in **Free space by mount** and is +covered by `DiskSpaceLow`/`DiskSpaceCritical`. On top of that, +`NetworkMountUnreachable` and `NetworkMountMissing` (section 2) fire for any +`cifs`/`smb3`/`nfs` mount; nothing to configure. + +**Automounted shares (`x-systemd.automount` in fstab)** are not mounted at +boot: systemd only puts an `autofs` placeholder there and mounts the share +the first time something opens the folder. node-exporter ignores `autofs`, so +until then the share is invisible to Grafana and the alerts. List it in +`WATCH_MOUNTS` in `/etc/safefamily/backup.env`; `dir_sizes.sh` (every 15 +minutes, and 2 minutes after boot) then opens it, which mounts it, and +records whether it answered (`WatchedMountDown` alert): + +```bash +echo 'WATCH_MOUNTS=/mnt/nas' | sudo tee -a /etc/safefamily/backup.env # your mount point(s), space separated +sudo systemctl start safefamily-dirsize.service +findmnt /mnt/nas # now shows a cifs line, not only autofs +``` + +Check that the share is being collected (replace `/mnt/nas`): + +```bash +curl -s localhost:9100/metrics | grep 'node_filesystem_avail_bytes' | grep -E 'cifs|smb3' +grep -v '^#' /var/lib/node_exporter/textfile/safefamily_mounts.prom # 1 = OK +``` + +Do not add `x-systemd.idle-timeout=` to an automounted share you monitor: +it unmounts the share when idle, which looks like a lost share +(`NetworkMountMissing`). + +- **No line for the share**: older node-exporter versions (e.g. Ubuntu + 22.04's 1.3.1) skip some mount points by default. See the active filter + with `prometheus-node-exporter --help 2>&1 | grep -A1 'mount-points'`; if + your mount point matches it, add to `ARGS` in + `/etc/default/prometheus-node-exporter` + (1.3.x: `--collector.filesystem.ignored-mount-points='^/(dev|proc|run|sys)($|/)'`, + 1.5+: `--collector.filesystem.mount-points-exclude='^/(dev|proc|run|sys)($|/)'`) + and `sudo systemctl restart prometheus-node-exporter`. +- **Mount it so a dead server cannot hang the host**: in `/etc/fstab` use + `nofail,_netdev,x-systemd.automount,x-systemd.mount-timeout=30` (then add + it to `WATCH_MOUNTS`, see above), and a + `credentials=/root/.smbcredentials` file (mode 600) instead of a password + in fstab. +- Do not add the share to `DIR_SIZES_EXTRA` for `dir_sizes.sh`: `du` over the + network every 15 minutes is slow and loads the file server. +- To alert at a different threshold for one share, copy the example in + `prometheus/rules.d/README.md`. + +Validate rule changes, including the unit tests for the share alerts: + +```bash +P='docker run --rm -v '"$PWD"'/deploy/monitoring/prometheus:/etc/prometheus:ro --entrypoint promtool prom/prometheus:v3.5.0' +$P check config /etc/prometheus/prometheus.yml +$P test rules /etc/prometheus/tests/network_mounts_test.yml +``` + +## 5. Status picture for the P4 display + +`deploy/monitoring/docker-compose.display-export.yml` renders the +**SafeFamily Display** dashboard (the Status and AdGuard tiles plus the +blocking timeline, laid out for exactly 1280x800) to a PNG every 5 minutes and +puts it in a folder the ESP32-P4 display reads: + +- `grafana-renderer`: Grafana's own headless-Chromium renderer, behind + Grafana's `/render` URL. Internal only; about 0.5 GB of RAM while rendering. +- `display-export`: fetches the PNG, checks it is a real 1280x800 PNG, writes + it to a hidden temp file (`.grafana-status.png.tmp`) and renames it over + `grafana-status.png`, one fixed name, so the display never reads a partial + file and old files never pile up. A failed render keeps the last good + picture; the display's STALE banner (10 min) then shows it is old. +- The display dashboard is generated from the overview's panels; after + changing those, run `python3 deploy/monitoring/display-export/make_display_dashboard.py` + and `docker compose restart grafana`. + +### Set it up + +1. Mount the writable share **at boot, without `x-systemd.automount`**: a + container only sees a share that was mounted before it started. Example + `/etc/fstab` line (adjust server, share and path; `uid`/`gid` = your user): + ``` + //10.0.0.133/display /home/zzuse/code/esphome-config/grafana-export cifs credentials=/etc/cifs-display.cred,rw,uid=1000,gid=1000,iocharset=utf8,_netdev,nofail,x-systemd.mount-timeout=30 0 0 + ``` + `sudo mount /home/zzuse/code/esphome-config/grafana-export`, then + `findmnt` on that path should show `cifs`. +2. Add to `.env` (append the overlay to your existing `COMPOSE_FILE`): + ```bash + COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml:deploy/monitoring/docker-compose.host-node-exporter.yml:deploy/monitoring/docker-compose.display-export.yml + COMPOSE_PROFILES=display-export # run the exporter container + DISPLAY_EXPORT_DIR=/home/zzuse/code/esphome-config/grafana-export + DISPLAY_TZ=America/Toronto # your time zone, for the timeline + ``` +3. `docker compose up -d`, then within a minute or two: + ```bash + docker logs safefamily_display_export --tail 3 # "wrote /export/grafana-status.png (1280x800, ... bytes)" + ls -l /home/zzuse/code/esphome-config/grafana-export/ + ``` + To look at the picture without the display: + in a browser. + +The exporter refuses to write while the folder is not a `cifs` mount ("not +writing: /export is 'ext4', expected cifs"), so pictures never land hidden in +the bare folder under the mount point. After mounting the share later, run +`docker compose restart display-export`. For a plain local folder set +`DISPLAY_EXPORT_REQUIRE_FSTYPE=""`. + +It logs in to Grafana with `GRAFANA_ADMIN_USER`/`GRAFANA_ADMIN_PASSWORD` from +`.env`. If you change the admin password in the Grafana UI, update `.env` and +`docker compose up -d display-export`. + +**Snap Docker refuses the folder** (`docker compose up` fails with a mount +error for `display-export`): run the same script on the host instead. +Remove `COMPOSE_PROFILES=display-export` from `.env` (the renderer keeps +running), `docker rm -f safefamily_display_export`, then: +```bash +sudo install -m 600 /dev/null /etc/safefamily/display-export.env +printf 'GRAFANA_PASSWORD=%s\nEXPORT_DIR=%s\nDISPLAY_TZ=%s\n' 'your-grafana-password' \ + /home/zzuse/code/esphome-config/grafana-export America/Toronto | sudo tee /etc/safefamily/display-export.env +sudo cp deploy/systemd/safefamily-display-export.* /etc/systemd/system/ +sudo sed -i 's#/srv/SafeFamily#'"$PWD"'#' /etc/systemd/system/safefamily-display-export.service +sudo systemctl daemon-reload && sudo systemctl enable --now safefamily-display-export.timer +``` diff --git a/docs/operations-checklist.md b/docs/operations-checklist.md new file mode 100644 index 0000000..3e91bed --- /dev/null +++ b/docs/operations-checklist.md @@ -0,0 +1,230 @@ +# Operations checklist: monitoring and encrypted backups + +A step-by-step list to set up, verify and run the monitoring stack and the +weekly encrypted database backup on the SafeFamily server. Background and the +alert reference are in [monitoring.md](monitoring.md). + +Each step has a **check** that tells you it worked. If a check fails, stop and +fix it before going on; the preflight script prints the fix for most problems. + +Paths assume the checkout is at `/srv/SafeFamily`; substitute yours. + +--- + +## Phase 0: Requirements (15 min) + +Run the preflight first. It only reads, never changes anything. Run it from +the repository root; `sudo` lets it check root-only files, and `--deep` also +test-mounts the host folders the exporters need (it pulls the node-exporter +and cadvisor images the first time): + +```bash +cd /srv/SafeFamily && git pull +sudo scripts/monitoring/preflight.sh --deep # or: sudo bash ./scripts/monitoring/preflight.sh --deep +``` + +Before Phase 2 it reports "backups are not set up yet" as a warning, not a +failure, so you can do monitoring first. + +- [ ] **Summary shows 0 failures.** Warnings are fine if you understand them + (see table). Re-run after every fix. + +| Requirement | Why | If preflight complains | +|---|---|---| +| Linux host (not macOS/Windows Docker Desktop) | exporters must see the real disks and containers | run the stack on the Linux server | +| Docker Engine 24+ (tested on 29.9) | cadvisor, compose features | upgrade Docker | +| Docker Compose **v2** plugin, 2.20+ (tested on 5.6) | overlay file, `COMPOSE_FILE` | `sudo apt install docker-compose-plugin`; legacy `docker-compose` v1 is not supported | +| Snap-installed Docker | confinement may hide the real `/` and block mounts | `--deep` tests them; if they fail, node-exporter runs on the host (monitoring.md "Snap Docker") | +| Docker root dir not `/var/lib/docker` (snap) | cadvisor and `dir_sizes.sh` read it | set `DOCKER_ROOT_DIR=` as printed | +| containerd socket not at `/run/containerd/containerd.sock` (snap Docker, any image store) | cadvisor needs it to name containers | set `CONTAINERD_SOCKET=` as printed | +| 2 GiB+ RAM; 15%+ disk free **after** Prometheus's history (5 GB by default) | `DiskSpaceLow` fires under 15% free | preflight prints the largest `PROMETHEUS_RETENTION_SIZE` that fits, or free space first | +| Ports 3000, 9090 free on 127.0.0.1 | Grafana, Prometheus | free them or change the host port in the overlay | +| Real (not self-signed) TLS certificate | website probe verifies it | Let's Encrypt, or `insecure_skip_verify` (see monitoring.md) | +| Outbound TCP 587 to smtp.gmail.com | alert emails | open the firewall, or set `ALERT_SMTP_SMARTHOST` | +| `age` installed | backup encryption | `sudo apt install age` | +| systemd running | weekly timers | use the cron lines in monitoring.md | + +The backup-related failures (recipients file, directories, timers) are +expected on the first run; Phase 2 fixes them. + +--- + +## Phase 1: Monitoring (20 min) + +1. - [ ] **Add to `.env`** (edit it yourself; values per Phase 0). Use these + exact names: the Grafana password is `GRAFANA_ADMIN_PASSWORD`, not + Grafana's internal `GF_SECURITY_ADMIN_PASSWORD` (the compose file maps one + to the other). + ```bash + GRAFANA_ADMIN_PASSWORD= + COMPOSE_FILE=docker-compose.yml:deploy/monitoring/docker-compose.monitoring.yml + # only if preflight printed them: + # DOCKER_ROOT_DIR=... + # CONTAINERD_SOCKET=... + # PROMETHEUS_RETENTION_SIZE=1GB (if preflight warns about disk space) + # ALERT_EMAIL_TO=you@example.com (default: MAIL_ACCOUNT) + ``` + Check: `docker compose config --services` lists `prometheus` and `grafana`. + +2. - [ ] **Create the metrics directory**: + `sudo mkdir -p /var/lib/node_exporter/textfile` + +3. - [ ] **Start everything**: + ```bash + docker compose up -d + docker compose up -d --force-recreate proxy # load the new nginx status page + ``` + The nginx config is mounted as a single file, so a running `proxy` keeps + the old copy until it is recreated (a few seconds of downtime; `nginx -s + reload` is not enough). + Check: `docker compose ps` shows 11 `safefamily_*` containers `Up`. + +4. - [ ] **Wait 2 minutes, then re-run preflight**: `sudo scripts/monitoring/preflight.sh` + Check: the "Running stack" section shows every scrape target `PASS`. + +5. - [ ] **Open Grafana**: from a device on your home network, + (after `docker compose up -d --force-recreate proxy`), + user `admin`. From elsewhere: `ssh -L 3000:127.0.0.1:3000 `, then . + Check: Dashboards → SafeFamily → SafeFamily Overview shows Website UP, + Database UP, App containers 3, and the disk and container graphs have lines. + +6. - [ ] **Send a test alert email**: + ```bash + docker exec safefamily_alertmanager amtool alert add TestAlert severity=warning \ + --annotation='summary="SafeFamily alert test"' --alertmanager.url=http://localhost:9093 + ``` + Check: within about 2 minutes, an email "[SafeFamily FIRING] TestAlert" + arrives (check spam). It auto-resolves in about 5 minutes and a "RESOLVED" + mail follows. No mail: `docker logs safefamily_alertmanager | grep -i notify`. + +6b. - [ ] **Optional: AdGuard and "is YouTube blocked"**: follow "AdGuard Home" + in [monitoring.md](monitoring.md#4-adding-more-monitoring-sources). + Check: Grafana's AdGuard row shows UP / ON, and preflight's "AdGuard check" + section passes. + +6c. - [ ] **Optional: status picture for the P4 display**: follow section 5 of + [monitoring.md](monitoring.md#5-status-picture-for-the-p4-display). + Check: `docker logs safefamily_display_export --tail 1` says `wrote ... + (1280x800, ...)` and the file's time updates every 5 minutes. + +--- + +## Phase 2: Weekly encrypted backups (20 min) + +7. - [ ] **Make the key pair on your laptop, not the server**: + ```bash + age-keygen -o safefamily-backup-key.txt # prints "Public key: age1..." + ``` + Check: you saved the private key file in **two** places off the server + (password manager + USB stick/paper). Losing it makes every backup + unreadable. + +8. - [ ] **Install the public key and config on the server**: + ```bash + sudo mkdir -p /etc/safefamily /var/backups/safefamily + echo 'age1...your public key...' | sudo tee /etc/safefamily/backup-recipients.txt + sudo cp deploy/monitoring/backup.env.example /etc/safefamily/backup.env + sudo chmod 600 /etc/safefamily/backup.env + sudo nano /etc/safefamily/backup.env # SAFEFAMILY_DIR, DOCKER_ROOT_DIR if needed + # echo 'BACKUP_GROUP=' | sudo tee -a /etc/safefamily/backup.env + ``` + Defaults: weekly backups, keep 8 weekly + 6 monthly. Backups are + root-only unless you set `BACKUP_GROUP=` (see `id -gn`) + in `backup.env`; do that so you (and your offsite-copy script) can copy + them without sudo. They are encrypted, so this is safe. + +9. - [ ] **Install the timers** (fix `ExecStart=` first if the checkout is not `/srv/SafeFamily`): + ```bash + sudo cp deploy/systemd/safefamily-backup.* deploy/systemd/safefamily-dirsize.* /etc/systemd/system/ + sudo sed -i 's#/srv/SafeFamily#'"$PWD"'#' /etc/systemd/system/safefamily-{backup,dirsize}.service + sudo systemctl daemon-reload + sudo systemctl enable --now safefamily-backup.timer safefamily-dirsize.timer + ``` + Check: `systemctl list-timers 'safefamily-*'` shows the backup's next run on a Sunday. + +10. - [ ] **Run the first backup now**: `sudo systemctl start safefamily-backup.service` + Check: `journalctl -u safefamily-backup.service -n 10` ends with `done in Ns`, + and `sudo ls -l /var/backups/safefamily/weekly/` shows one `.dump.age` file. + Within a minute, Grafana's "Last DB backup" tile shows a few minutes. + +11. - [ ] **Restore drill** (proves the key and the backup both work). On your laptop + (needs `BACKUP_GROUP` from step 8; otherwise first copy the file somewhere + readable on the server: `sudo install -m 644 /var/backups/safefamily/weekly/ /tmp/`): + ```bash + ssh ls /var/backups/safefamily/weekly/ # find the newest + scp :/var/backups/safefamily/weekly/safefamily-.dump.age . + age -d -i safefamily-backup-key.txt safefamily-.dump.age \ + | ssh 'cd /srv/SafeFamily && scripts/db_restore.sh -' + ``` + Check: it prints row counts for every table, matching what you expect + (users, todo_list, notes, ...). Then clean up: + `ssh 'cd /srv/SafeFamily && scripts/db_restore.sh --drop-test'` + +12. - [ ] **Offsite copy**: add `/var/backups/safefamily/` to your existing backup + script (it is already encrypted). + Check: the newest `.dump.age` file shows up at the destination. + +13. - [ ] **Final preflight**: `sudo scripts/monitoring/preflight.sh --deep` + Check: 0 failures. + +--- + +## Routine operations + +| When | Do | Check | +|---|---|---| +| Any alert email | follow "First thing to check" in [monitoring.md](monitoring.md#2-alerts) | a RESOLVED mail arrives | +| Weekly (2 min) | glance at the Grafana dashboard | tiles green, "Last DB backup" < 8 days | +| Monthly (5 min) | confirm the offsite copy has the latest backup; `df -h` | newest file present; disks < 85% | +| Every 3 months (15 min) | restore drill (step 11) | row counts look right | +| After upgrading Docker, snap refresh, or OS | `sudo scripts/monitoring/preflight.sh --deep` | 0 failures; container panels still have data | +| After `git pull` that touches `deploy/monitoring/` | `docker compose up -d`, plus `docker compose restart prometheus alertmanager` for changed rules | preflight "Running stack" all PASS | +| After `git pull` that touches `deploy/nginx/` | `docker compose up -d --force-recreate proxy` | site loads; nginx panels have data | +| Yearly | check the private key copies still open: `age -d -i > /dev/null` | no error | +| Certificate renewal | nothing; the TLS alert warns 14 days ahead | `TLSCertificateExpiringSoon` resolves | + +Useful commands: + +```bash +docker compose ps # everything up? +docker compose logs --tail=50 # e.g. prometheus, alertmanager, cadvisor +systemctl list-timers 'safefamily-*' # next backup +journalctl -u safefamily-backup.service -n 30 # last backup log +sudo ls -lh /var/backups/safefamily/{weekly,monthly} +sudo systemctl start safefamily-backup.service # extra backup now (e.g. before an upgrade) +``` + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `docker compose` errors `... must be set in .env` | missing `MAIL_ACCOUNT`, `MAIL_PASSWORD` or `GRAFANA_ADMIN_PASSWORD` (note: not `GF_SECURITY_ADMIN_PASSWORD`) | add it to `.env` under that exact name | +| "App containers running" No data, container panels empty, `ContainerDown` firing | cadvisor cannot name containers (usually the containerd socket) | `sudo scripts/monitoring/preflight.sh` shows cadvisor's error and the fix; then `docker compose up -d cadvisor` | +| `--deep` says containers see a different `/`; Grafana disk tiles and "Last DB backup" empty | snap Docker confinement | run node-exporter on the host: "Snap Docker" in monitoring.md (apt package + `docker-compose.host-node-exporter.yml`) | +| `--deep`: only the textfile dir mount FAILs | snap Docker refuses `/var/lib/node_exporter/textfile` | host mode as above, or `/var/snap/docker/common/node_exporter_textfile` in both `.env` and `/etc/safefamily/backup.env` | +| Share under `/home`: node-exporter shows `device_error="permission denied"` | the home folder is 750; node-exporter runs as user `prometheus` | `sudo setfacl -m u:prometheus:x /home/` (pass-through only); preflight names the folder | +| Host mode: `node` scrape target down | ufw blocks containers from host port 9100 | `sudo ufw allow from 172.16.0.0/12 to any port 9100 proto tcp` | +| Requests/s and Connections panels empty, `NginxStatusUnavailable` | `proxy` still runs the old nginx config | `docker compose up -d --force-recreate proxy` | +| Website tile DOWN but the site works | self-signed cert, or the `server_name` in blackbox.yml differs from nginx | see Phase 0 table | +| No alert emails | Gmail app password, port 587 blocked | `docker logs safefamily_alertmanager \| grep -i notify` | +| "Last DB backup" tile "No data" | backup never ran, or metrics file unreadable | step 10; `MetricsFileUnreadable` alert; `ls -l /var/lib/node_exporter/textfile` | +| Backup log: `recipients file ... contains a PRIVATE key` | private key copied to the server | replace with the public `age1...` line; delete the private key from the server | +| Backup log: `container ... is not running` | db container down | `docker compose up -d db`, then step 10 | +| Grafana password forgotten | set only on first start | `docker exec -it safefamily_grafana grafana cli admin reset-admin-password ` | + +--- + +## Turning it off + +Monitoring only (the app keeps running): + +```bash +docker compose -f docker-compose.yml -f deploy/monitoring/docker-compose.monitoring.yml \ + rm -sf prometheus alertmanager grafana node-exporter cadvisor postgres-exporter nginx-exporter blackbox-exporter +# then remove the COMPOSE_FILE line from .env +``` + +Backups: `sudo systemctl disable --now safefamily-backup.timer safefamily-dirsize.timer`. +Existing backup files stay in `/var/backups/safefamily/`. diff --git a/scripts/db_backup.sh b/scripts/db_backup.sh new file mode 100755 index 0000000..6b019d5 --- /dev/null +++ b/scripts/db_backup.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Full, encrypted backup of the SafeFamily Postgres database. +# +# Dumps every table (including logs and media, unlike scripts/db_export.py) +# in pg_dump custom format, checks the dump is readable, and encrypts it with +# age to one or more PUBLIC keys. Only public keys live on this host, so a +# stolen server or backup disk cannot be used to read the backups; decrypting +# needs the private key you keep elsewhere (see docs/monitoring.md). +# +# Run on the host (it uses `docker exec`), weekly via +# deploy/systemd/safefamily-backup.timer or cron. Settings come from the +# environment, typically /etc/safefamily/backup.env: +# +# BACKUP_DIR where backups go (/var/backups/safefamily) +# BACKUP_AGE_RECIPIENTS file of age public keys (/etc/safefamily/backup-recipients.txt) +# BACKUP_KEEP_WEEKLY weekly backups to keep (8) +# BACKUP_KEEP_MONTHLY monthly backups to keep (6) +# BACKUP_GROUP optional group allowed to read the (encrypted) +# backups, e.g. your login group, so scp/rsync work +# without sudo. Unset: root only. +# BACKUP_DB_CONTAINER Postgres container (safefamily_db) +# NODE_EXPORTER_TEXTFILE_DIR metrics for Prometheus (/var/lib/node_exporter/textfile) + +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-/var/backups/safefamily}" +RECIPIENTS="${BACKUP_AGE_RECIPIENTS:-/etc/safefamily/backup-recipients.txt}" +KEEP_WEEKLY="${BACKUP_KEEP_WEEKLY:-8}" +KEEP_MONTHLY="${BACKUP_KEEP_MONTHLY:-6}" +GROUP="${BACKUP_GROUP:-}" +CONTAINER="${BACKUP_DB_CONTAINER:-safefamily_db}" +TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}" + +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +name="safefamily-${stamp}.dump.age" +container_tmp="/tmp/safefamily-backup-${stamp}.dump" +start_ts="$(date +%s)" +partial="" + +log() { echo "[db_backup] $*" >&2; } + +write_metrics() { + # $1 = 1 on success, 0 on failure. Written atomically so node_exporter + # never reads a half-written file. + [[ -d "$TEXTFILE_DIR" ]] || return 0 + local prom="$TEXTFILE_DIR/safefamily_backup.prom" + local now + now="$(date +%s)" + { + echo "# HELP safefamily_backup_last_run_success 1 if the last backup run succeeded." + echo "# TYPE safefamily_backup_last_run_success gauge" + echo "safefamily_backup_last_run_success $1" + echo "# HELP safefamily_backup_last_run_timestamp_seconds When the last backup run finished." + echo "# TYPE safefamily_backup_last_run_timestamp_seconds gauge" + echo "safefamily_backup_last_run_timestamp_seconds $now" + if [[ "$1" == 1 ]]; then + echo "# HELP safefamily_backup_last_success_timestamp_seconds When the last successful backup finished." + echo "# TYPE safefamily_backup_last_success_timestamp_seconds gauge" + echo "safefamily_backup_last_success_timestamp_seconds $now" + echo "# HELP safefamily_backup_last_size_bytes Size of the last encrypted backup." + echo "# TYPE safefamily_backup_last_size_bytes gauge" + echo "safefamily_backup_last_size_bytes $(stat -c %s "$BACKUP_DIR/weekly/$name")" + echo "# HELP safefamily_backup_last_duration_seconds How long the last backup took." + echo "# TYPE safefamily_backup_last_duration_seconds gauge" + echo "safefamily_backup_last_duration_seconds $((now - start_ts))" + elif [[ -f "$prom" ]]; then + # Keep the previous success time so the "backup too old" alert + # still measures from the last good backup. + grep -E '^safefamily_backup_last_(success_timestamp_seconds|size_bytes|duration_seconds) ' "$prom" || true + fi + } > "$prom.tmp" + # Readable by node_exporter (runs as nobody); the umask below is for backups. + chmod 644 "$prom.tmp" + mv "$prom.tmp" "$prom" +} + +cleanup() { + local status=$? + docker exec "$CONTAINER" rm -f "$container_tmp" >/dev/null 2>&1 || true + [[ -n "$partial" ]] && rm -f "$partial" + if [[ $status -ne 0 ]]; then + log "FAILED (exit $status)" + write_metrics 0 || true + fi +} +trap cleanup EXIT + +command -v age >/dev/null || { log "age is not installed (apt install age)"; exit 1; } +[[ -s "$RECIPIENTS" ]] || { log "recipients file $RECIPIENTS is missing or empty"; exit 1; } +if grep -q "AGE-SECRET-KEY" "$RECIPIENTS"; then + log "$RECIPIENTS contains a PRIVATE key; it must only hold public keys (age1...)" + exit 1 +fi + +umask 077 +mkdir -p "$BACKUP_DIR/weekly" "$BACKUP_DIR/monthly" + +# Dump inside the container using its own POSTGRES_* env, so the DB password +# never passes through this script. pg_restore --list proves the archive is +# complete and readable before it is encrypted. +log "dumping database from $CONTAINER" +docker exec "$CONTAINER" sh -c \ + 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --compress=6 --file="$1" \ + && pg_restore --list "$1" > /dev/null' \ + sh "$container_tmp" + +partial="$BACKUP_DIR/weekly/.$name.partial" +docker exec "$CONTAINER" cat "$container_tmp" | age --encrypt --recipients-file "$RECIPIENTS" --output "$partial" +mv "$partial" "$BACKUP_DIR/weekly/$name" +partial="" +log "wrote $BACKUP_DIR/weekly/$name ($(du -h "$BACKUP_DIR/weekly/$name" | cut -f1))" + +# Keep the first backup of each month for longer. A hard link costs no space. +month="${stamp:0:6}" +if ! compgen -G "$BACKUP_DIR/monthly/safefamily-${month}*.dump.age" > /dev/null; then + ln "$BACKUP_DIR/weekly/$name" "$BACKUP_DIR/monthly/$name" + log "kept as monthly backup for ${month}" +fi + +# Keep the newest N of each set, by count rather than age, so a long run of +# failed backups never prunes away the last good ones. Names sort +# chronologically. +prune() { + local dir="$1" keep="$2" old + find "$dir" -name 'safefamily-*.dump.age' -printf '%f\n' | sort -r \ + | tail -n "+$((keep + 1))" | while read -r old; do + rm -f "$dir/$old" + log "pruned $(basename "$dir") backup $old" + done +} +prune "$BACKUP_DIR/weekly" "$KEEP_WEEKLY" +prune "$BACKUP_DIR/monthly" "$KEEP_MONTHLY" + +# Backups are encrypted, so letting one group read them is safe; it lets a +# normal user copy them offsite. Applied to every file so a newly set group +# also covers older backups. A wrong group name never fails the backup. +if [[ -n "$GROUP" ]]; then + if getent group "$GROUP" > /dev/null; then + chgrp -R "$GROUP" "$BACKUP_DIR" + chmod 750 "$BACKUP_DIR" "$BACKUP_DIR/weekly" "$BACKUP_DIR/monthly" + find "$BACKUP_DIR" -name 'safefamily-*.dump.age' -exec chmod 640 {} + + else + log "WARNING: BACKUP_GROUP '$GROUP' does not exist; backups stay root-only" + fi +fi + +write_metrics 1 +log "done in $(($(date +%s) - start_ts))s" diff --git a/scripts/db_restore.sh b/scripts/db_restore.sh new file mode 100755 index 0000000..101c736 --- /dev/null +++ b/scripts/db_restore.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Restore a backup made by scripts/db_backup.sh. +# +# scripts/db_restore.sh -i ~/safefamily-backup-key.txt BACKUP.dump.age +# Restores into a scratch database (safefamily_restore_test) and prints +# row counts: a safe restore drill that leaves the live data alone. +# +# scripts/db_restore.sh --drop-test +# Deletes the scratch database again when you are done looking at it. +# +# scripts/db_restore.sh -i KEY --into-live BACKUP.dump.age +# Replaces the live database's contents after you type its name. Stop +# the app first: docker compose stop app +# +# Without -i the file must already be decrypted (a plain .dump). To avoid +# copying the private key to the server, decrypt on the machine that holds it +# and stream the result ("-" reads stdin; --confirm DBNAME replaces the typed +# confirmation, since stdin carries the backup): +# age -d -i key.txt BACKUP.dump.age | ssh server 'cd SafeFamily && scripts/db_restore.sh -' +# age -d -i key.txt BACKUP.dump.age \ +# | ssh server 'cd SafeFamily && scripts/db_restore.sh --into-live --confirm logdb -' + +set -euo pipefail + +CONTAINER="${BACKUP_DB_CONTAINER:-safefamily_db}" +TEST_DB="safefamily_restore_test" +identity="" +into_live=0 +confirm="" + +usage() { sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } + +while [[ $# -gt 0 ]]; do + case "$1" in + -i|--identity) identity="$2"; shift 2 ;; + --into-live) into_live=1; shift ;; + --confirm) confirm="$2"; shift 2 ;; + --drop-test) + # Runs as the container's own Postgres superuser (POSTGRES_USER). + docker exec "$CONTAINER" sh -c 'dropdb -U "$POSTGRES_USER" --if-exists "$1"' sh "$TEST_DB" + echo "dropped scratch database $TEST_DB (live data untouched)" >&2 + exit 0 ;; + -h|--help) usage ;; + -) break ;; + -*) echo "unknown option $1" >&2; usage ;; + *) break ;; + esac +done +[[ $# -eq 1 ]] || usage +backup="$1" + +live_db="$(docker exec "$CONTAINER" sh -c 'echo "$POSTGRES_DB"')" +if [[ $into_live -eq 1 ]]; then + target="$live_db" + if [[ -z "$confirm" ]]; then + # Ask on the terminal, not stdin: stdin may be carrying the backup. + read -r -p "This REPLACES all data in database '$target'. Type the database name to continue: " \ + confirm < /dev/tty + fi + [[ "$confirm" == "$target" ]] || { echo "aborted: confirmation does not match '$target'" >&2; exit 1; } +else + target="$TEST_DB" +fi + +stream() { + if [[ "$backup" == "-" ]]; then + cat + elif [[ -n "$identity" ]]; then + age --decrypt --identity "$identity" "$backup" + else + cat "$backup" + fi +} + +if [[ $into_live -eq 0 ]]; then + docker exec "$CONTAINER" sh -c \ + 'dropdb -U "$POSTGRES_USER" --if-exists "$1" && createdb -U "$POSTGRES_USER" "$1"' sh "$target" +fi + +echo "restoring into database '$target'..." >&2 +# --clean --if-exists makes a live restore replace existing objects; on the +# fresh scratch database it is a harmless no-op. +stream | docker exec -i "$CONTAINER" sh -c \ + 'pg_restore -U "$POSTGRES_USER" -d "$1" --clean --if-exists --no-owner --exit-on-error' sh "$target" + +echo "restored. Row counts:" >&2 +docker exec "$CONTAINER" sh -c 'psql -U "$POSTGRES_USER" -d "$1" -c " + SELECT table_name AS table, (xpath(\$\$/row/c/text()\$\$, query_to_xml( + format(\$\$SELECT count(*) AS c FROM %I.%I\$\$, table_schema, table_name), + false, true, \$\$\$\$)))[1]::text::bigint AS rows + FROM information_schema.tables + WHERE table_schema = \$\$public\$\$ AND table_type = \$\$BASE TABLE\$\$ + ORDER BY 1"' sh "$target" +if [[ $into_live -eq 0 ]]; then + echo "Live data was not touched. Delete the scratch copy when done: $0 --drop-test" >&2 +fi diff --git a/scripts/monitoring/adguard_check.py b/scripts/monitoring/adguard_check.py new file mode 100755 index 0000000..4ee86ba --- /dev/null +++ b/scripts/monitoring/adguard_check.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Write AdGuard Home health and domain-blocking metrics for node-exporter. + +Two kinds of "is it blocked": + * services: whether a service id is in AdGuard's blocked-services list, the + list SafeFamily switches (blocker.py). A fast, local lookup. + * domains: AdGuard's check_host answer for a name, which also covers filter + rules and parental control, but can be slow on a busy router or when + Safe Browsing / Parental Control cannot reach AdGuard's servers. + +Run every minute by deploy/systemd/safefamily-adguard.timer; settings come +from the environment, normally /etc/safefamily/adguard.env (root-only, it +holds the password): + + ADGUARD_HOSTPORT e.g. 10.0.0.1:8080 (same value as the app's .env) + ADGUARD_USERNAME AdGuard web login + ADGUARD_PASSWORD + ADGUARD_WATCH_SERVICES blocked-service ids, space separated, default "youtube" + ADGUARD_WATCH_DOMAINS names for check_host, default "youtube.com"; "" = off + ADGUARD_TIMEOUT seconds per API call, default 30 + NODE_EXPORTER_TEXTFILE_DIR default /var/lib/node_exporter/textfile +""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +DEFAULT_TIMEOUT_SECONDS = 30 +# check_host reasons that mean AdGuard answers with a block. Rewrites and +# safe search change the answer but do not block, so they are not listed. +BLOCKING_REASONS = { + "FilteredBlackList", + "FilteredBlockedService", + "FilteredParental", + "FilteredSafeBrowsing", +} + + +class AdGuardClient: + """Minimal authenticated client for the AdGuard Home control API.""" + + def __init__(self, hostport: str, username: str, password: str, timeout: float) -> None: + """Prepare the base URL and Basic auth header.""" + self.timeout = timeout + self.base = hostport if "://" in hostport else f"http://{hostport}" + token = base64.b64encode(f"{username}:{password}".encode()).decode() + self.auth = f"Basic {token}" + + def get(self, path: str, **params: str) -> dict: + """GET a control endpoint and return its JSON body.""" + query = f"?{urllib.parse.urlencode(params)}" if params else "" + request = urllib.request.Request( # noqa: S310 - URL built from admin config + f"{self.base.rstrip('/')}/control/{path}{query}", + headers={"Authorization": self.auth, "Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=self.timeout) as response: # noqa: S310 + return json.load(response) + + +def blocked_service_ids(client: AdGuardClient) -> set[str]: + """Return the ids in AdGuard's blocked-services list (old and new API).""" + try: + # v0.107.37+: {"ids": [...], "schedule": {...}} + return set(client.get("blocked_services/get").get("ids") or []) + except urllib.error.HTTPError as exc: + if exc.code not in (404, 405): + raise + # Older versions: a plain list of ids. + return set(client.get("blocked_services/list") or []) + + +API_ERRORS = (urllib.error.URLError, OSError, ValueError) + + +class Metrics: + """Collects Prometheus text-format lines.""" + + def __init__(self) -> None: + """Start with no lines.""" + self.lines: list[str] = [] + + def add(self, name: str, help_text: str, samples: list[tuple[str, float]]) -> None: + """Add one gauge with its HELP/TYPE header.""" + self.lines.append(f"# HELP {name} {help_text}") + self.lines.append(f"# TYPE {name} gauge") + self.lines.extend(f"{name}{labels} {value:g}" for labels, value in samples) + + +def collect_health(client: AdGuardClient, out: Metrics) -> bool: + """Record whether AdGuard answers, protection and query counts.""" + help_up = "1 if the AdGuard API answered with the configured login." + try: + status = client.get("status") + stats = client.get("stats") + except API_ERRORS as exc: + reason = getattr(exc, "code", None) or getattr(exc, "reason", exc) + print(f"[adguard_check] AdGuard API not usable: {reason}", file=sys.stderr) + out.add("safefamily_adguard_up", help_up, [("", 0)]) + return False + out.add("safefamily_adguard_up", help_up, [("", 1)]) + out.add( + "safefamily_adguard_protection_enabled", + "1 if AdGuard protection (filtering) is switched on.", + [("", 1 if status.get("protection_enabled") else 0)], + ) + out.add( + "safefamily_adguard_dns_queries", + "DNS queries in AdGuard's statistics window (default 24h).", + [("", float(stats.get("num_dns_queries", 0)))], + ) + out.add( + "safefamily_adguard_blocked_queries", + "Queries blocked by filters in AdGuard's statistics window.", + [("", float(stats.get("num_blocked_filtering", 0)))], + ) + return True + + +def collect_services(client: AdGuardClient, services: list[str], out: Metrics) -> None: + """Record whether each service id is in the blocked-services list.""" + if not services: + return + try: + ids = blocked_service_ids(client) + except API_ERRORS as exc: + print(f"[adguard_check] blocked services lookup failed: {exc}", file=sys.stderr) + return + for service in services: + state = "blocked" if service in ids else "allowed" + print(f"[adguard_check] service {service}: {state}", file=sys.stderr) + out.add( + "safefamily_adguard_service_blocked", + "1 if this service id is in AdGuard's blocked-services list.", + [(f'{{service="{service}"}}', 1 if service in ids else 0) for service in services], + ) + + +def collect_domains(client: AdGuardClient, domains: list[str], out: Metrics) -> None: + """Record AdGuard's check_host verdict for each domain.""" + blocked, check_ok = [], [] + for domain in domains: + labels = f'{{domain="{domain}"}}' + try: + result = client.get("filtering/check_host", name=domain) + except API_ERRORS as exc: + hint = "" + if "timed out" in str(exc): + hint = ( + f" (after {client.timeout:g}s; check_host is slow when Safe Browsing/Parental" + " Control cannot reach AdGuard's servers: raise ADGUARD_TIMEOUT, or rely on" + ' ADGUARD_WATCH_SERVICES and set ADGUARD_WATCH_DOMAINS="")' + ) + print(f"[adguard_check] check_host {domain} failed: {exc}{hint}", file=sys.stderr) + check_ok.append((labels, 0)) + continue + reason = result.get("reason", "") + print(f"[adguard_check] {domain}: {reason}", file=sys.stderr) + blocked.append((labels, 1 if reason in BLOCKING_REASONS else 0)) + check_ok.append((labels, 1)) + if blocked: + out.add("safefamily_adguard_domain_blocked", "1 if AdGuard would block this domain right now.", blocked) + if check_ok: + out.add("safefamily_adguard_domain_check_ok", "1 if the block check for this domain worked.", check_ok) + + +def collect(client: AdGuardClient, services: list[str], domains: list[str]) -> list[str]: + """Return Prometheus text-format lines for one check run.""" + out = Metrics() + if collect_health(client, out): + collect_services(client, services, out) + collect_domains(client, domains, out) + return out.lines + + +def main() -> int: + """Run one check and write the metrics file atomically.""" + hostport = os.environ.get("ADGUARD_HOSTPORT", "") + username = os.environ.get("ADGUARD_USERNAME", "") + password = os.environ.get("ADGUARD_PASSWORD", "") + if not hostport or not username: + print("[adguard_check] set ADGUARD_HOSTPORT, ADGUARD_USERNAME and ADGUARD_PASSWORD", file=sys.stderr) + return 2 + services = os.environ.get("ADGUARD_WATCH_SERVICES", "youtube").split() + domains = os.environ.get("ADGUARD_WATCH_DOMAINS", "youtube.com").split() + try: + timeout = float(os.environ.get("ADGUARD_TIMEOUT", DEFAULT_TIMEOUT_SECONDS)) + except ValueError: + timeout = DEFAULT_TIMEOUT_SECONDS + out_dir = Path(os.environ.get("NODE_EXPORTER_TEXTFILE_DIR", "/var/lib/node_exporter/textfile")) + + lines = collect(AdGuardClient(hostport, username, password, timeout), services, domains) + lines += [ + "# HELP safefamily_adguard_last_check_timestamp_seconds When adguard_check.py last ran.", + "# TYPE safefamily_adguard_last_check_timestamp_seconds gauge", + f"safefamily_adguard_last_check_timestamp_seconds {int(time.time())}", + ] + out_dir.mkdir(parents=True, exist_ok=True) + tmp = out_dir / "safefamily_adguard.prom.tmp" + tmp.write_text("\n".join(lines) + "\n") + tmp.chmod(0o644) # node-exporter runs as another user + tmp.replace(out_dir / "safefamily_adguard.prom") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/monitoring/dir_sizes.sh b/scripts/monitoring/dir_sizes.sh new file mode 100755 index 0000000..594d5c9 --- /dev/null +++ b/scripts/monitoring/dir_sizes.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Report the size of the directories SafeFamily grows on the host, for the +# "Directory sizes" Grafana panel. node_exporter reads the .prom file from its +# textfile directory. Run every 15 minutes by +# deploy/systemd/safefamily-dirsize.timer. +# +# SAFEFAMILY_DIR the repository checkout (default: this script's repo) +# BACKUP_DIR encrypted backups (/var/backups/safefamily) +# DOCKER_ROOT_DIR Docker data incl. the Postgres volume (/var/lib/docker; +# snap Docker: /var/snap/docker/common/var-lib-docker) +# NODE_EXPORTER_TEXTFILE_DIR output directory (/var/lib/node_exporter/textfile) +# DIR_SIZES_EXTRA more directories, space separated +# WATCH_MOUNTS network mount points to check, space separated +# (e.g. a Samba share). Each check reads the +# mount point, which also wakes a systemd +# automount, and writes safefamily_mount_ok. + +set -euo pipefail + +repo="${SAFEFAMILY_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}" +docker_root="${DOCKER_ROOT_DIR:-/var/lib/docker}" +out_dir="${NODE_EXPORTER_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}" + +dirs=( + "$repo" + "$repo/logs" + "${BACKUP_DIR:-/var/backups/safefamily}" + "$docker_root/volumes" + "$docker_root/containers" + "/var/log/journal" +) +# shellcheck disable=SC2206 # intentional word splitting +dirs+=(${DIR_SIZES_EXTRA:-}) + +mkdir -p "$out_dir" + +# Watched mounts first, so an automounted share is mounted before anything +# else looks at it. OK means: the mount point answered within 10s AND a real +# (non-autofs) filesystem is mounted exactly there, not just the empty local +# directory underneath. +if [[ -n "${WATCH_MOUNTS:-}" ]]; then + { + echo "# HELP safefamily_mount_ok 1 if the watched mount point is mounted and answered within 10s." + echo "# TYPE safefamily_mount_ok gauge" + for mnt in ${WATCH_MOUNTS}; do + ok=0 + if timeout -k 5 10 stat -f "$mnt/." > /dev/null 2>&1 \ + && findmnt -n -M "$mnt" -o FSTYPE 2>/dev/null | grep -qvx autofs; then + ok=1 + fi + echo "safefamily_mount_ok{mountpoint=\"$mnt\"} $ok" + done + } > "$out_dir/safefamily_mounts.prom.tmp" + chmod 644 "$out_dir/safefamily_mounts.prom.tmp" + mv "$out_dir/safefamily_mounts.prom.tmp" "$out_dir/safefamily_mounts.prom" +else + rm -f "$out_dir/safefamily_mounts.prom" +fi + +{ + echo "# HELP safefamily_dir_size_bytes Disk usage of a directory (du)." + echo "# TYPE safefamily_dir_size_bytes gauge" + for dir in "${dirs[@]}"; do + [[ -d "$dir" ]] || continue + bytes="$(du -sxb "$dir" 2>/dev/null | cut -f1 || true)" + [[ -n "$bytes" ]] && echo "safefamily_dir_size_bytes{path=\"$dir\"} $bytes" + done +} > "$out_dir/safefamily_dir_sizes.prom.tmp" +mv "$out_dir/safefamily_dir_sizes.prom.tmp" "$out_dir/safefamily_dir_sizes.prom" diff --git a/scripts/monitoring/preflight.sh b/scripts/monitoring/preflight.sh new file mode 100755 index 0000000..b173ed8 --- /dev/null +++ b/scripts/monitoring/preflight.sh @@ -0,0 +1,707 @@ +#!/usr/bin/env bash +# Check that this host can run the SafeFamily monitoring stack and encrypted +# backups, and print the fix for anything that is missing. Read-only: it +# changes nothing. Run from the repository root on the server: +# +# sudo scripts/monitoring/preflight.sh # quick checks +# sudo scripts/monitoring/preflight.sh --deep # also test the container +# # mounts cadvisor and +# # node-exporter need (pulls +# # their images if missing) +# +# sudo is optional but some files it checks are root-only. Exit status is 1 +# if any check FAILs. See docs/operations-checklist.md. + +set -uo pipefail + +deep=0 +case "${1:-}" in + "") ;; + --deep|-d) deep=1 ;; + -h|--help) sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown option: $1 (use --deep or --help)" >&2; exit 2 ;; +esac + +repo="$(cd "$(dirname "$0")/../.." && pwd)" +overlay="deploy/monitoring/docker-compose.monitoring.yml" +host_ne_overlay="deploy/monitoring/docker-compose.host-node-exporter.yml" +backup_env="/etc/safefamily/backup.env" +fails=0 +warns=0 + +if [[ -t 1 ]]; then + G=$'\e[32m' Y=$'\e[33m' R=$'\e[31m' B=$'\e[1m' N=$'\e[0m' +else + G="" Y="" R="" B="" N="" +fi +pass() { printf ' %sPASS%s %s\n' "$G" "$N" "$1"; } +warn() { printf ' %sWARN%s %s\n' "$Y" "$N" "$1"; [[ -n "${2:-}" ]] && printf ' fix: %s\n' "$2"; warns=$((warns + 1)); } +fail() { printf ' %sFAIL%s %s\n' "$R" "$N" "$1"; [[ -n "${2:-}" ]] && printf ' fix: %s\n' "$2"; fails=$((fails + 1)); } +info() { printf ' info %s\n' "$1"; } +section() { printf '\n%s%s%s\n' "$B" "$1" "$N"; } +# version_ge A B: true when version A >= B +version_ge() { [[ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -1)" == "$2" ]]; } + +# Settings from the backup env file, if readable, else defaults. +BACKUP_DIR=/var/backups/safefamily +BACKUP_AGE_RECIPIENTS=/etc/safefamily/backup-recipients.txt +NODE_EXPORTER_TEXTFILE_DIR=/var/lib/node_exporter/textfile +if [[ -r "$backup_env" ]]; then + # Read only the settings used here, in a subshell, so nothing else from + # the file leaks into this script's environment (and into the docker + # compose calls, where it would hide a missing .env setting). + eval "$( + # shellcheck disable=SC1090 + . "$backup_env" >/dev/null 2>&1 + for v in BACKUP_DIR BACKUP_AGE_RECIPIENTS NODE_EXPORTER_TEXTFILE_DIR WATCH_MOUNTS; do + [[ -n "${!v:-}" ]] && printf '%s=%q\n' "$v" "${!v}" + done + )" +fi + +cd "$repo" || exit 1 + +if (( EUID != 0 )); then + printf 'Note: not running as root, so root-only files may be reported missing.\n' + printf ' For the full picture: sudo %s%s\n' "$0" "$([[ $deep == 1 ]] && echo ' --deep')" +fi + +# The merged compose config (main file + monitoring overlay, with .env +# applied), parsed once and queried by later checks. Holds secrets: only ever +# piped, never printed or written to disk. +cfg_json="" +cfg_err="" +compose_ok=0 +load_compose_config() { + local err_file + err_file="$(mktemp)" + # Use the user's COMPOSE_FILE when it includes monitoring (it may also add + # the host node-exporter overlay); otherwise main file + monitoring overlay. + if cfg_json="$(docker compose config --format json 2>/dev/null)" \ + && python3 -c 'import json,sys; sys.exit("prometheus" not in json.load(sys.stdin)["services"])' <<< "$cfg_json"; then + compose_ok=1 + elif cfg_json="$(docker compose -f docker-compose.yml -f "$overlay" config --format json 2>"$err_file")"; then + compose_ok=1 + fi + # Drop compose's harmless "variable is not set" warnings for optional vars. + cfg_err="$(grep -v 'level=warning' "$err_file")" + rm -f "$err_file" +} +# cfg_value SERVICE KEY PREFIX: value of a "--flag=value" command argument +# (KEY=command) or the source of the volume mounted at PREFIX (KEY=volume). +cfg_value() { + python3 -c ' +import json, sys +svc, key, want = sys.argv[1:4] +s = json.load(sys.stdin)["services"].get(svc, {}) +if key == "command": + print(next((a.split("=", 1)[1] for a in s.get("command") or [] if a.startswith(want + "=")), "")) +else: + print(next((v.get("source", "") for v in s.get("volumes") or [] if v.get("target") == want), "")) +' "$@" <<< "$cfg_json" +} +# host_ne: 1 when node-exporter runs on the host (host overlay in use). +host_ne=0 +detect_host_ne() { + (( compose_ok )) && python3 -c 'import json,sys; sys.exit("node-exporter" in json.load(sys.stdin)["services"])' <<< "$cfg_json" \ + && host_ne=1 +} +# env_has NAME: .env defines NAME (checks the name only, never reads values). +env_has() { [[ -r .env ]] && grep -Eq "^[[:space:]]*(export[[:space:]]+)?$1=" .env; } + +section "Host" +if [[ "$(uname -s)" == "Linux" ]]; then + pass "Linux $(uname -r)" +else + fail "$(uname -s) host: node-exporter and cadvisor need Linux to see the real disks and containers" \ + "run the stack on the Linux server; Docker Desktop only shows its VM" +fi +if [[ -d /run/systemd/system ]]; then + pass "systemd is running (backup timers)" +else + warn "no systemd: the backup timers cannot be installed" "use the cron line in docs/monitoring.md" +fi +mem_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)" +if (( mem_kb >= 2 * 1024 * 1024 )); then + pass "RAM $((mem_kb / 1024)) MiB (stack needs ~600 MiB)" +else + warn "RAM $((mem_kb / 1024)) MiB: monitoring adds ~600 MiB" "lower PROMETHEUS_RETENTION or skip cadvisor" +fi + +section "Docker" +if ! command -v docker >/dev/null; then + fail "docker CLI not found" "install Docker Engine: https://docs.docker.com/engine/install/" +else + if ! docker info >/dev/null 2>&1; then + fail "cannot talk to the Docker daemon as $(id -un)" \ + "run with sudo, or: sudo usermod -aG docker $(id -un) && log out and back in" + else + server="$(docker version -f '{{.Server.Version}}' 2>/dev/null)" + if version_ge "$server" "24.0"; then + pass "Docker Engine $server" + else + warn "Docker Engine $server is old (tested on 29.x)" "upgrade Docker; if you stay, watch cadvisor logs" + fi + + docker compose version >/dev/null 2>&1 && load_compose_config && detect_host_ne + root_dir="$(docker info -f '{{.DockerRootDir}}' 2>/dev/null)" + if [[ "$root_dir" == /var/snap/* ]] || { command -v snap >/dev/null && snap list docker >/dev/null 2>&1; }; then + if (( deep )); then + info "Docker is installed from snap; the --deep mount tests below show whether its confinement allows the exporters" + else + warn "Docker is installed from snap, whose confinement can block the exporters' host mounts" \ + "rerun with --deep to test them" + fi + fi + if [[ "$root_dir" == "/var/lib/docker" ]]; then + pass "Docker root dir /var/lib/docker (default, no DOCKER_ROOT_DIR needed)" + elif (( compose_ok )); then + mounted="$(cfg_value cadvisor volume /var/lib/docker)" + if [[ "$mounted" == "$root_dir" ]]; then + pass "Docker root dir $root_dir, and DOCKER_ROOT_DIR points cadvisor to it" + else + warn "Docker root dir is $root_dir but cadvisor would mount ${mounted:-nothing}" \ + "add to .env: DOCKER_ROOT_DIR=$root_dir" + fi + elif env_has DOCKER_ROOT_DIR; then + info "Docker root dir $root_dir; DOCKER_ROOT_DIR is in .env (verified once the compose error below is fixed)" + else + warn "Docker root dir is $root_dir" "add to .env: DOCKER_ROOT_DIR=$root_dir" + fi + + driver="$(docker info -f '{{.Driver}}' 2>/dev/null)" + if docker info 2>/dev/null | grep -q "io.containerd.snapshotter"; then + info "image store: containerd snapshotter ($driver)" + else + info "image store: classic $driver" + fi + # cadvisor v0.54's Docker support always talks to containerd, whatever + # the image store; without it containers have no names in Grafana. + candidates=(/run/containerd/containerd.sock /run/docker/containerd/containerd.sock) + # Snap Docker runs its own containerd; prefer it over a system one. + if [[ "$root_dir" == /var/snap/* ]]; then + candidates=(/run/snap.docker/containerd/containerd.sock + /var/snap/docker/common/run/containerd/containerd.sock "${candidates[@]}") + fi + sock="" + for candidate in "${candidates[@]}"; do + [[ -S "$candidate" ]] && { sock="$candidate"; break; } + done + if [[ "$sock" == "/run/containerd/containerd.sock" ]]; then + pass "containerd socket at /run/containerd/containerd.sock (cadvisor finds it)" + elif [[ -n "$sock" && ( "$sock" == /run/* || "$sock" == /var/run/* ) ]]; then + if (( compose_ok )) && [[ "$(cfg_value cadvisor command --containerd)" == "$sock" ]]; then + pass "containerd socket at $sock, and CONTAINERD_SOCKET points cadvisor to it" + elif (( compose_ok )); then + # Say why a CONTAINERD_SOCKET the user believes is set has no effect. + why="" + if env_has CONTAINERD_SOCKET; then + why=" (.env sets CONTAINERD_SOCKET, but a CONTAINERD_SOCKET variable in this shell overrides it: unset CONTAINERD_SOCKET)" + elif [[ -r .env ]] && grep -Eq '^[[:space:]]*#.*CONTAINERD_SOCKET' .env; then + why=" (the CONTAINERD_SOCKET line in .env is commented out)" + elif [[ -r "$backup_env" ]] && grep -q CONTAINERD_SOCKET "$backup_env"; then + why=" (CONTAINERD_SOCKET is in $backup_env, but Docker Compose only reads $repo/.env)" + else + why=" ($repo/.env has no CONTAINERD_SOCKET line)" + fi + warn "containerd socket is at $sock, but cadvisor is set to $(cfg_value cadvisor command --containerd)$why" \ + "put CONTAINERD_SOCKET=$sock in $repo/.env, then docker compose up -d cadvisor" + else + warn "containerd socket is at $sock, not where cadvisor looks" "add to .env: CONTAINERD_SOCKET=$sock" + fi + elif [[ -n "$sock" ]]; then + warn "containerd socket is at $sock, outside /run, so cadvisor cannot see it" \ + "add under cadvisor.volumes in $overlay: - $(dirname "$sock"):/run/host-containerd:ro and to .env: CONTAINERD_SOCKET=/run/host-containerd/containerd.sock" + else + warn "containerd socket not found (run with sudo to search root-only paths)" \ + "sudo find /run /var/snap -name containerd.sock" + fi + info "cgroup v$(docker info -f '{{.CgroupVersion}}' 2>/dev/null), driver $(docker info -f '{{.CgroupDriver}}' 2>/dev/null)" + fi +fi + +section "Docker Compose" +if docker compose version >/dev/null 2>&1; then + cv="$(docker compose version --short 2>/dev/null | sed 's/^v//')" + if version_ge "$cv" "2.20"; then + pass "Compose v$cv" + else + warn "Compose v$cv is old (tested on 5.x)" "upgrade the docker-compose-plugin package" + fi + if (( compose_ok )); then + pass "docker-compose.yml + monitoring overlay parse with your .env" + else + fail "compose files do not parse with your .env:" + while IFS= read -r line; do printf ' %s\n' "$line"; done <<< "$cfg_err" + hint="set the variables named above in .env (see .env.example)" + # Grafana's own variable names go in the compose file, not in .env. + if grep -q GRAFANA_ADMIN_PASSWORD <<< "$cfg_err" && env_has GF_SECURITY_ADMIN_PASSWORD; then + hint="your .env has GF_SECURITY_ADMIN_PASSWORD; rename it to GRAFANA_ADMIN_PASSWORD (the compose file maps it)" + fi + printf ' fix: %s\n' "$hint" + fi + if (( ! compose_ok )); then + info "COMPOSE_FILE check skipped until the compose files parse" + elif docker compose config --services 2>/dev/null | grep -qx prometheus; then + pass "COMPOSE_FILE includes the monitoring overlay (plain 'docker compose' manages it)" + else + warn "plain 'docker compose' does not include monitoring" \ + "add to .env: COMPOSE_FILE=docker-compose.yml:$overlay" + fi +else + if command -v docker-compose >/dev/null; then + fail "only legacy docker-compose v1 found" "install the Compose v2 plugin: sudo apt install docker-compose-plugin" + else + fail "Docker Compose not found" "sudo apt install docker-compose-plugin" + fi +fi + +section "Monitoring prerequisites" +listening() { ss -ltnH 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$1\$"; } +for port in 3000 9090; do + owner="" + command -v docker >/dev/null && owner="$(docker ps --filter "publish=$port" --format '{{.Names}}' 2>/dev/null)" + if [[ -n "$owner" ]]; then + pass "port $port is used by $owner" + elif listening "$port"; then + fail "port $port is already taken by another program" "free it, or change the host port in $overlay" + else + pass "port $port is free" + fi +done +if [[ -d "$NODE_EXPORTER_TEXTFILE_DIR" ]]; then + pass "$NODE_EXPORTER_TEXTFILE_DIR exists (node-exporter reads backup/dir-size metrics there)" +else + fail "$NODE_EXPORTER_TEXTFILE_DIR missing (checklist Phase 1 step 2)" "sudo mkdir -p $NODE_EXPORTER_TEXTFILE_DIR" +fi +if (( host_ne )); then + info "node-exporter runs on the host ($host_ne_overlay)" + if curl -fsS --max-time 3 http://127.0.0.1:9100/metrics >/dev/null 2>&1; then + pass "host node-exporter answers on :9100" + # Which textfile dir it reads: its --collector.textfile.directory flag. + # Match the process name only (the kernel truncates it to 15 chars: + # "prometheus-node"), never other command lines that mention it. + ne_args="$(pgrep -a '^(node_exporter|prometheus-node)' | head -1)" + ne_dir="$(sed -n 's/.*--collector\.textfile\.directory[= ]\([^ ]*\).*/\1/p' <<< "$ne_args")" + if [[ "$ne_dir" == "$NODE_EXPORTER_TEXTFILE_DIR" ]]; then + pass "host node-exporter reads the textfile dir $ne_dir" + else + fail "host node-exporter reads textfile dir '${ne_dir:-its package default}', scripts write to $NODE_EXPORTER_TEXTFILE_DIR" \ + "set ARGS=\"--collector.textfile.directory=$NODE_EXPORTER_TEXTFILE_DIR\" in /etc/default/prometheus-node-exporter, then: sudo systemctl restart prometheus-node-exporter" + fi + else + fail "no node-exporter on the host at :9100" \ + "sudo apt install prometheus-node-exporter (see 'Snap Docker' in docs/monitoring.md)" + fi +elif (( compose_ok )); then + mounted_textfile="$(cfg_value node-exporter volume /textfile)" + if [[ "$mounted_textfile" != "$NODE_EXPORTER_TEXTFILE_DIR" ]]; then + fail "scripts write metrics to $NODE_EXPORTER_TEXTFILE_DIR but node-exporter reads $mounted_textfile" \ + "set the same NODE_EXPORTER_TEXTFILE_DIR in .env and in $backup_env" + fi +fi +# Prometheus's volume lives under the Docker root dir and grows to its +# retention size, so check free space as it will be once that is used up. +# DiskSpaceLow fires below 15% free. +retention="5GB" +(( compose_ok )) && retention="$(cfg_value prometheus command --storage.tsdb.retention.size)" +retention_kb="$(python3 -c ' +import re, sys +m = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?", sys.argv[1].strip(), re.I) +n, u = (float(m.group(1)), m.group(2).upper()) if m else (5.0, "G") +print(int(n * {"": 1 / 1024, "K": 1, "M": 1024, "G": 1024**2, "T": 1024**3}[u])) +' "${retention:-5GB}")" +for dir in "${root_dir:-/var/lib/docker}" "$(dirname "$BACKUP_DIR")"; do + [[ -d "$dir" ]] || continue + read -r size_kb avail_kb < <(df -Pk "$dir" 2>/dev/null | awk 'NR==2 {print $2, $4}') + pct=$((avail_kb * 100 / size_kb)) + reserve_kb=0 + [[ "$dir" == "${root_dir:-/var/lib/docker}" ]] && reserve_kb=$retention_kb + after_pct=$(((avail_kb - reserve_kb) * 100 / size_kb)) + msg="$dir: $((avail_kb / 1024 / 1024)) GiB free (${pct}%)" + if (( reserve_kb == 0 )) || (( after_pct >= 15 )); then + (( reserve_kb )) && msg+=", ${after_pct}% after Prometheus uses its ${retention}" + if (( pct >= 15 )); then + pass "$msg" + else + warn "$msg: DiskSpaceLow (<15%) will fire" "free space (README 'Prune Cache')" + fi + else + # Largest retention that still leaves 15% free, in whole GB. + fit_gb=$(((avail_kb - size_kb * 15 / 100) / 1024 / 1024)) + if (( fit_gb >= 1 )); then + fix="add to .env: PROMETHEUS_RETENTION_SIZE=${fit_gb}GB (or free space: README 'Prune Cache')" + else + fix="free space first (README 'Prune Cache'); then set PROMETHEUS_RETENTION_SIZE=1GB in .env" + fi + warn "$msg, but only ${after_pct}% once Prometheus uses its ${retention}: DiskSpaceLow (<15%) would fire" "$fix" + fi +done + +if [[ -n "${WATCH_MOUNTS:-}" ]]; then + section "Watched mounts (WATCH_MOUNTS)" + # Where node-exporter's metrics can be read: the host exporter directly, + # otherwise the container through Prometheus. + ne_metrics="" + if (( host_ne )); then + ne_metrics="$(curl -fsS --max-time 10 http://127.0.0.1:9100/metrics 2>/dev/null)" + ne_version="$(sed -n 's/^node_exporter_build_info{.*version="\([^"]*\)".*/\1/p' <<< "$ne_metrics" | head -1)" + [[ -n "$ne_version" ]] && info "host node-exporter version $ne_version" + fi + for mnt in ${WATCH_MOUNTS}; do + fstypes="$(findmnt -n -M "$mnt" -o FSTYPE 2>/dev/null | tr '\n' ' ')" + real_fs="$(tr ' ' '\n' <<< "$fstypes" | grep -vx -e autofs -e '' | head -1)" + if [[ -n "$real_fs" ]]; then + pass "$mnt is mounted ($real_fs)" + elif [[ "$fstypes" == *autofs* ]]; then + fail "$mnt is only an automount placeholder: the share itself did not mount" \ + "ls $mnt; sudo journalctl -b | grep -iE 'cifs|mount' | tail; check the server and the credentials file" + continue + else + fail "$mnt is not a mount point" "check its /etc/fstab line, then: sudo mount $mnt" + continue + fi + ok_file="$NODE_EXPORTER_TEXTFILE_DIR/safefamily_mounts.prom" + if grep -q "mountpoint=\"$mnt\"} 1" "$ok_file" 2>/dev/null; then + pass "watch check reports $mnt OK" + else + warn "watch check has no OK result for $mnt yet" "sudo systemctl start safefamily-dirsize.service, then rerun" + fi + # A sandboxed exporter service may not be allowed into /home. + if (( host_ne )) && [[ "$mnt" == /home/* || "$mnt" == /root/* ]] && [[ -d /run/systemd/system ]]; then + protect="$(systemctl show prometheus-node-exporter -p ProtectHome --value 2>/dev/null)" + if [[ -n "$protect" && "$protect" != "no" ]]; then + fail "prometheus-node-exporter runs with ProtectHome=$protect, so it cannot read $mnt" \ + "sudo systemctl edit prometheus-node-exporter, add [Service] / ProtectHome=no, then sudo systemctl restart prometheus-node-exporter" + fi + fi + if (( host_ne )); then + lines="$(grep -F "mountpoint=\"$mnt\"" <<< "$ne_metrics")" + else + lines="$(curl -fsS --max-time 5 --get http://127.0.0.1:9090/api/v1/query \ + --data-urlencode "query={__name__=~\"node_filesystem_(size_bytes|device_error)\",mountpoint=\"$mnt\"}" 2>/dev/null \ + | python3 -c ' +import json, sys +for r in json.load(sys.stdin)["data"]["result"]: + print(r["metric"]["__name__"], r["value"][1])' 2>/dev/null)" + fi + if grep -q "node_filesystem_size_bytes" <<< "$lines"; then + pass "node-exporter reports $mnt (Grafana and the share alerts can see it)" + elif grep -qE 'node_filesystem_device_error.* 1$' <<< "$lines"; then + # Usually a parent folder the exporter's user may not pass + # through, e.g. a 750 home directory. Find which ones. + ne_user="$(systemctl show prometheus-node-exporter -p User --value 2>/dev/null)" + ne_user="${ne_user:-prometheus}" + blocked=() + if (( EUID == 0 )) && id "$ne_user" > /dev/null 2>&1; then + dir="" + IFS=/ read -ra parts <<< "${mnt#/}" + for part in "${parts[@]::${#parts[@]}-1}"; do + dir+="/$part" + # Stop at the first one: everything below it looks + # blocked too until it is fixed, so rerun afterwards. + if ! runuser -u "$ne_user" -- test -x "$dir" 2> /dev/null; then + blocked+=("$dir") + break + fi + done + fi + if (( ${#blocked[@]} )); then + fail "node-exporter (user $ne_user) cannot pass through ${blocked[0]} to reach $mnt" \ + "sudo setfacl -m u:$ne_user:x ${blocked[0]} (pass-through only, it cannot list or read it; needs the acl package), wait a minute, rerun" + else + fail "node-exporter lists $mnt but cannot read it (device error)" \ + "check: sudo -u $ne_user stat -f $mnt (run this preflight with sudo to find a blocking parent folder)" + fi + else + fail "node-exporter does not report $mnt at all${ne_version:+ (version $ne_version)}" \ + "its mount-point/fs-type filter skips it: compare 'prometheus-node-exporter --help 2>&1 | grep -A1 -E \"mount-points|fs-types\"' with $mnt ($real_fs); see docs/monitoring.md 'Samba share'" + fi + done +fi + +adguard_env=/etc/safefamily/adguard.env +if [[ -e "$adguard_env" ]]; then + section "AdGuard check" + mode="$(stat -c '%a %U' "$adguard_env" 2>/dev/null)" + if [[ "$mode" == "600 root" ]]; then + pass "$adguard_env is root-only" + else + fail "$adguard_env is '$mode', but it holds the AdGuard password" "sudo chown root: $adguard_env && sudo chmod 600 $adguard_env" + fi + prom="$NODE_EXPORTER_TEXTFILE_DIR/safefamily_adguard.prom" + last="$(sed -n 's/^safefamily_adguard_last_check_timestamp_seconds //p' "$prom" 2>/dev/null)" + if [[ -z "$last" ]]; then + fail "no AdGuard metrics yet in $prom" \ + "sudo systemctl enable --now safefamily-adguard.timer; journalctl -u safefamily-adguard.service -n 20" + elif (( $(date +%s) - last > 300 )); then + fail "the AdGuard check last ran $(( ($(date +%s) - last) / 60 )) minutes ago" "systemctl status safefamily-adguard.timer" + elif grep -q '^safefamily_adguard_up 1' "$prom"; then + pass "AdGuard answers with the configured login" + while read -r domain value; do + info "$domain is $([[ "$value" == 1 ]] && echo BLOCKED || echo allowed) right now" + done < <(sed -n -e 's/^safefamily_adguard_service_blocked{service="\([^"]*\)"} \(.*\)$/service:\1 \2/p' \ + -e 's/^safefamily_adguard_domain_blocked{domain="\([^"]*\)"} \(.*\)$/\1 \2/p' "$prom") + if grep -q '^safefamily_adguard_domain_check_ok{.*} 0' "$prom"; then + warn "AdGuard's check_host timed out or failed for a watched domain" \ + "raise ADGUARD_TIMEOUT in $adguard_env, or set ADGUARD_WATCH_DOMAINS=\"\" and rely on ADGUARD_WATCH_SERVICES" + fi + else + fail "AdGuard does not answer, or rejects the login" \ + "check ADGUARD_HOSTPORT/USERNAME/PASSWORD in $adguard_env; journalctl -u safefamily-adguard.service -n 5" + fi +fi + +section "TLS certificate and outbound mail" +cert=/etc/ssl/certs/safefamily.crt +if [[ -r "$cert" ]] && command -v openssl >/dev/null; then + end="$(openssl x509 -in "$cert" -noout -enddate 2>/dev/null | cut -d= -f2)" + if openssl x509 -in "$cert" -noout -checkend $((14 * 86400)) >/dev/null 2>&1; then + pass "certificate valid until $end" + else + warn "certificate expires within 14 days ($end)" "renew it before enabling the TLS alert" + fi + subject="$(openssl x509 -in "$cert" -noout -subject 2>/dev/null | sed 's/^subject=//')" + issuer="$(openssl x509 -in "$cert" -noout -issuer 2>/dev/null | sed 's/^issuer=//')" + if [[ "$subject" == "$issuer" ]]; then + warn "certificate is self-signed: the website probe will report DOWN" \ + "use Let's Encrypt, or set insecure_skip_verify: true in deploy/monitoring/blackbox/blackbox.yml" + else + pass "certificate issued by ${issuer}" + fi +else + warn "cannot read $cert (run with sudo)" +fi +if timeout 5 bash -c 'exec 3<>/dev/tcp/smtp.gmail.com/587' 2>/dev/null; then + pass "can reach smtp.gmail.com:587 for alert mail" +else + warn "cannot reach smtp.gmail.com:587: alert emails will not arrive" \ + "check the firewall/ISP; set ALERT_SMTP_SMARTHOST in .env for another mail server" +fi + +section "Encrypted backups (checklist Phase 2)" +# Only backup-specific artifacts count: /etc/safefamily and other safefamily-* +# units also come from the AdGuard and display-export setups. +backup_started=0 +[[ -e "$BACKUP_AGE_RECIPIENTS" || -e "$backup_env" || -d "$BACKUP_DIR" ]] && backup_started=1 +if [[ -d /run/systemd/system ]] && systemctl list-unit-files 'safefamily-backup.*' 2>/dev/null | grep -q safefamily-backup; then + backup_started=1 +fi +if (( ! backup_started )); then + if command -v age >/dev/null; then + pass "age $(age --version 2>/dev/null) installed" + else + warn "age not installed" "sudo apt install age" + fi + warn "backups are not set up yet" "follow docs/operations-checklist.md Phase 2 (steps 7-13) when ready" +elif command -v age >/dev/null; then + pass "age $(age --version 2>/dev/null)" +else + fail "age not installed" "sudo apt install age" +fi +if (( backup_started )); then + if [[ -r "$BACKUP_AGE_RECIPIENTS" ]]; then + if grep -q "AGE-SECRET-KEY" "$BACKUP_AGE_RECIPIENTS"; then + fail "$BACKUP_AGE_RECIPIENTS contains a PRIVATE key" \ + "replace it with only the public key (age1...) and move the private key off this server" + elif grep -q "^age1" "$BACKUP_AGE_RECIPIENTS"; then + pass "$BACKUP_AGE_RECIPIENTS has $(grep -c '^age1' "$BACKUP_AGE_RECIPIENTS") public key(s)" + else + fail "$BACKUP_AGE_RECIPIENTS has no age1... public key" "see 'One-time setup' in docs/monitoring.md" + fi + elif [[ -e "$BACKUP_AGE_RECIPIENTS" ]]; then + warn "$BACKUP_AGE_RECIPIENTS exists but is not readable (run with sudo)" + else + fail "$BACKUP_AGE_RECIPIENTS missing" "create it with your age public key (docs/monitoring.md)" + fi + if [[ -r "$backup_env" ]]; then + pass "$backup_env present" + else + warn "$backup_env not found or not readable (defaults apply)" "sudo cp deploy/monitoring/backup.env.example $backup_env" + fi + if [[ -d "$BACKUP_DIR" ]]; then pass "$BACKUP_DIR exists"; else fail "$BACKUP_DIR missing" "sudo mkdir -p $BACKUP_DIR"; fi + if [[ -d /run/systemd/system ]]; then + for unit in safefamily-backup.timer safefamily-dirsize.timer; do + if [[ "$(systemctl is-enabled "$unit" 2>/dev/null)" == "enabled" ]]; then + next="$(systemctl show "$unit" -p NextElapseUSecRealtime --value 2>/dev/null)" + pass "$unit enabled${next:+ (next run: $next)}" + else + fail "$unit not enabled" "sudo systemctl enable --now $unit (after copying deploy/systemd/${unit%.timer}.*)" + fi + svc="/etc/systemd/system/${unit%.timer}.service" + if [[ -r "$svc" ]]; then + exe="$(sed -n 's/^ExecStart=//p' "$svc" | awk '{print $1}')" + if [[ -x "$exe" ]]; then + pass "$(basename "$svc") runs $exe" + else + fail "$(basename "$svc") runs $exe, which does not exist" \ + "edit ExecStart to $repo/scripts/..., then sudo systemctl daemon-reload" + fi + fi + done + fi +fi +if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx safefamily_db; then + pass "safefamily_db is running (backups can dump it)" +else + warn "safefamily_db is not running: backups will fail until it is" "docker compose up -d db" +fi + +if (( deep )); then + section "Container mounts (--deep)" + ne_image=prom/node-exporter:v1.9.1 + # docker run's errors end with a "Run 'docker run --help'" footer; keep + # the lines that say what actually went wrong. + show_error() { + grep -v -e "^Run 'docker run --help'" -e '^[[:space:]]*$' <<< "$1" | tail -n 3 \ + | sed -e 's/^docker: //' -e 's/^/ error: /' + } + # try_run DESCRIPTION FIX DOCKER-RUN-ARGS...: PASS, or FAIL with the real error. + try_run() { + local what="$1" fix="$2" + shift 2 + if out="$(docker run --rm "$@" 2>&1)"; then + pass "$what" + return 0 + fi + fail "$what" + show_error "$out" + printf ' fix: %s\n' "$fix" + return 1 + } + snap_note="Docker from snap is confined; see 'Host-specific notes' in docs/monitoring.md" + host_mode_fix="run node-exporter on the host: 'Snap Docker' in docs/monitoring.md (apt install prometheus-node-exporter + add $host_ne_overlay to COMPOSE_FILE)" + root_ok=1 + + # 1. node-exporter measures disks through a read-only mount of /. Under + # some confinement setups / inside containers is not the host's root, and + # every disk number (and alert) for / would be wrong. + if (( host_ne )); then + info "node-exporter runs on the host, so its container mount tests are skipped" + elif host_root_kb="$(df -Pk / | awk 'NR==2 {print $2}')" \ + && try_run "node-exporter can mount / read-only" "$snap_note" \ + -v /:/host:ro --entrypoint /bin/ls "$ne_image" /host/etc/hostname; then + seen_kb="$(docker run --rm -v /:/host:ro --entrypoint /bin/df "$ne_image" -Pk /host 2>/dev/null \ + | awk 'NR==2 {print $2}')" + if [[ -n "$seen_kb" ]] && (( seen_kb * 100 / host_root_kb >= 99 && seen_kb * 100 / host_root_kb <= 101 )); then + pass "containers see the host's real / ($((host_root_kb / 1024 / 1024)) GiB), so disk metrics are right" + else + root_ok=0 + fail "containers see a different / (${seen_kb:-?} KiB) than the host ($host_root_kb KiB): disk metrics for / would be wrong" \ + "$host_mode_fix" + fi + fi + + # 2. The textfile directory: the scripts write metrics there and + # node-exporter reads them. Test the directory compose really mounts. + textfile_src="$NODE_EXPORTER_TEXTFILE_DIR" + (( compose_ok )) && textfile_src="$(cfg_value node-exporter volume /textfile)" + snap_textfile=/var/snap/docker/common/node_exporter_textfile + if (( host_ne )); then + : + elif (( ! root_ok )); then + info "textfile mount test skipped: running node-exporter on the host (fix above) also solves it" + elif [[ ! -d "$textfile_src" ]]; then + warn "textfile dir $textfile_src does not exist yet, so it cannot be test-mounted" \ + "sudo mkdir -p $textfile_src, then rerun" + elif ! try_run "node-exporter can read the textfile dir $textfile_src" \ + "see the suggestion below" \ + -v "$textfile_src:/textfile:ro" --entrypoint /bin/ls "$ne_image" /textfile; then + # Snap Docker can always use its own data area under /var/snap/docker/common. + if [[ -d /var/snap/docker/common ]] && docker run --rm -v /var/snap/docker/common:/probe:ro \ + --entrypoint /bin/ls "$ne_image" /probe >/dev/null 2>&1; then + info "Docker can mount /var/snap/docker/common, so use a textfile dir there:" + info " sudo mkdir -p $snap_textfile" + info " add to .env: NODE_EXPORTER_TEXTFILE_DIR=$snap_textfile" + info " add to /etc/safefamily/backup.env: NODE_EXPORTER_TEXTFILE_DIR=$snap_textfile" + info " then: docker compose up -d node-exporter, and rerun this check" + else + info "$snap_note" + fi + fi + + # 3. cadvisor's host mounts. + try_run "cadvisor can mount /sys, /var/run and the Docker root" "$snap_note" \ + --privileged -v /sys:/sys:ro -v /var/run:/var/run:ro -v "${root_dir:-/var/lib/docker}:/var/lib/docker:ro" \ + --entrypoint /bin/ls gcr.io/cadvisor/cadvisor:v0.54.1 /var/lib/docker +fi + +section "Running stack (skipped when monitoring is not up yet)" +if curl -fsS --max-time 3 http://127.0.0.1:9090/-/ready >/dev/null 2>&1; then + if targets="$(curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/targets | python3 -c ' +import json, sys +for t in json.load(sys.stdin)["data"]["activeTargets"]: + state = "PASS" if t["health"] == "up" else "FAIL" + print(" %s scrape %s: %s %s" % (state, t["labels"]["job"], t["health"], t["lastError"][:120])) +')"; then + echo "$targets" + if grep -q "FAIL" <<< "$targets"; then fails=$((fails + 1)); fi + else + fail "could not read scrape targets from Prometheus" + fi + # nginx-exporter is "up" even when it cannot reach nginx's status page; + # nginx_up says whether it actually got data. + nginx_up="$(curl -fsS --max-time 5 --get http://127.0.0.1:9090/api/v1/query \ + --data-urlencode 'query=nginx_up' 2>/dev/null \ + | python3 -c 'import json,sys; r=json.load(sys.stdin)["data"]["result"]; print(r[0]["value"][1] if r else "")' 2>/dev/null)" + if [[ "$nginx_up" == "1" ]]; then + pass "nginx status page answers (Requests/s and Connections panels have data)" + else + # The config is a single-file bind mount: after git pull the container + # keeps the old file until it is recreated (a reload is not enough). + host_has="$(grep -c 'stub_status' deploy/nginx/nginx.docker.conf 2>/dev/null)" + cont_has="$(docker exec safefamily_proxy grep -c 'stub_status' /etc/nginx/conf.d/default.conf 2>/dev/null)" + if [[ "${host_has:-0}" != "0" && "${cont_has:-0}" == "0" ]]; then + fail "nginx is running an old copy of deploy/nginx/nginx.docker.conf (no status page): nginx panels empty" \ + "docker compose up -d --force-recreate proxy (a reload does not pick up the new file)" + else + fail "nginx-exporter gets no data from http://proxy:8080/stub_status" \ + "docker compose logs --tail=20 nginx-exporter proxy" + fi + fi + + # cadvisor can be up yet unable to tell which container is which (its + # Docker integration failed); then the container panels and the + # ContainerDown alert have nothing to go on. + named="$(curl -fsS --max-time 5 --get http://127.0.0.1:9090/api/v1/query \ + --data-urlencode 'query=count(container_last_seen{name=~"safefamily_(app|db|proxy)"})' 2>/dev/null \ + | python3 -c 'import json,sys; r=json.load(sys.stdin)["data"]["result"]; print(r[0]["value"][1] if r else 0)' 2>/dev/null)" + if [[ "${named:-0}" == "3" ]]; then + pass "cadvisor sees app, db and proxy by name" + elif grep -q "PASS scrape cadvisor" <<< "${targets:-}"; then + fail "cadvisor is up but sees ${named:-0} of app/db/proxy by name: container panels empty, ContainerDown fires" + # The decisive lines first: why the docker factory failed, or why a + # container could not be attached; else any other recent error. + cad_all="$(docker logs --tail 500 safefamily_cadvisor 2>&1)" + cad_log="$(grep -E 'docker container factory failed|Failed to create existing container' <<< "$cad_all" | tail -n 2)" + [[ -n "$cad_log" ]] || cad_log="$(grep -iE 'fail|error|unable' <<< "$cad_all" \ + | grep -viE 'crio|podman|perf_event|resctrl|product_name|gce\.go' | tail -n 3)" + while IFS= read -r line; do [[ -n "$line" ]] && printf ' log: %s\n' "${line:0:220}"; done <<< "$cad_log" + if grep -qi containerd <<< "$cad_log"; then + hint="set CONTAINERD_SOCKET in .env to the socket preflight finds (Docker section), then docker compose up -d cadvisor" + elif grep -qiE 'layerdb|read-write layer|mount-id' <<< "$cad_log"; then + hint="cadvisor cannot read Docker's layer data under /rootfs$root_dir: see 'cadvisor without container names' in docs/monitoring.md" + elif grep -qi 'docker.sock' <<< "$cad_log"; then + hint="cadvisor cannot reach the Docker socket: check the /var/run mount in $overlay" + else + hint="see 'cadvisor without container names' in docs/monitoring.md" + fi + printf ' fix: %s\n' "$hint" + fi + if ! firing="$(curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/alerts | python3 -c ' +import json, sys +alerts = json.load(sys.stdin)["data"]["alerts"] +print(", ".join(sorted({a["labels"]["alertname"] for a in alerts if a["state"] == "firing"}))) +')"; then + fail "could not read alerts from Prometheus" + elif [[ -z "$firing" ]]; then + pass "no alerts firing" + else + warn "alerts firing: $firing" "see the alert table in docs/monitoring.md" + fi +else + info "Prometheus is not reachable on 127.0.0.1:9090; rerun after 'docker compose up -d'" +fi + +printf '\n%sSummary:%s %d failure(s), %d warning(s)\n' "$B" "$N" "$fails" "$warns" +(( fails == 0 ))