From 801d0088020ae588effb3b8d29d6ca2ffb52be9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Thu, 22 Oct 2026 13:30:47 -0700 Subject: [PATCH] feat(zeta): alert if spore's tailnet-only ports reach the internet Gatus probes each port spore's own services listen on through its public address every 5 minutes and alerts if one accepts a connection. Co-Authored-By: Claude Opus 5.5 --- hosts/zeta/services/gatus.nix | 353 ++++++++++++++++++---------------- 1 file changed, 190 insertions(+), 163 deletions(-) diff --git a/hosts/zeta/services/gatus.nix b/hosts/zeta/services/gatus.nix index c8eb0b72..e6d0e68c 100644 --- a/hosts/zeta/services/gatus.nix +++ b/hosts/zeta/services/gatus.nix @@ -4,6 +4,7 @@ # Slack directly. Grafana in turn alerts if zeta's node exporter goes away. { config, + lib, pkgs, ... }: let @@ -16,6 +17,20 @@ # server can lock every device out (as on 2026-10-11). servers = ["glyph" "spore" "zeta" "stroma"]; + # Ports spore's services listen on that must stay off the internet, + # probed on its public address. zeta reaches that over the internet, not + # the tailnet, so spore's firewall applies. The daily port scan workflow + # covers every port; these catch a regression within minutes. + sporePrivatePorts = { + node-exporter = 9100; + nginx-exporter = 9113; + alloy = 12345; + glances = 61208; + homepage = 8082; + pocket-id = 1411; + oauth2-proxy = 4180; + }; + alert = description: { type = "custom"; inherit description; @@ -74,170 +89,182 @@ in { }; }; - endpoints = [ - (mkEndpoint { - name = "reachable"; - group = "glyph"; - url = "tcp://${glyph}:22"; - conditions = ["[CONNECTED] == true"]; - description = "glyph is unreachable over the tailnet; Prometheus, Loki and Grafana's database are down with it"; - }) - (mkEndpoint { - name = "postgres"; - group = "glyph"; - url = "tcp://${glyph}:5432"; - conditions = ["[CONNECTED] == true"]; - description = "PostgreSQL on glyph is not accepting connections; Grafana, pocket-id and others depend on it"; - }) - (mkEndpoint { - name = "prometheus"; - group = "glyph"; - # An instant query only returns samples from the last 5m, so an empty - # result means scraping or ingestion has stalled. - url = "http://${glyph}:9099/api/v1/query?query=up"; - conditions = [ - "[STATUS] == 200" - "len([BODY].data.result) > 0" - ]; - description = "Prometheus is down or has no samples from the last 5m"; - }) - (mkEndpoint { - name = "loki"; - group = "glyph"; - # zeta's own journal shipped through Alloy proves end-to-end ingestion. - url = "http://${glyph}:3100/loki/api/v1/query?query=sum(count_over_time(%7Bhost%3D%22zeta%22%7D%5B10m%5D))"; - conditions = [ - "[STATUS] == 200" - "len([BODY].data.result) > 0" - ]; - description = "Loki is down or has received no logs from zeta in 10m"; - }) - (mkEndpoint { - name = "audiomuse"; - group = "glyph"; - # Unauthenticated liveness probe; covers the flask container only - url = "http://${glyph}:8001/api/health"; - conditions = [ - "[STATUS] == 200" - "[BODY].status == ok" - ]; - description = "AudioMuse-AI's web container on glyph is not answering, so Navidrome's audiomuseai plugin can't reach it"; - }) - (mkEndpoint { - name = "paseo"; - group = "glyph"; - # Unauthenticated liveness probe, through spore: glyph only accepts - # Paseo's port from spore - url = "https://agents.zx.dev/api/health"; - conditions = ["[STATUS] == 200"]; - description = "Paseo on glyph is not answering through spore, so agents.zx.dev and its agents are down"; - }) - (mkEndpoint { - name = "reachable"; - group = "spore"; - url = "tcp://${spore}:22"; - conditions = ["[CONNECTED] == true"]; - description = "spore is unreachable over the tailnet; *.zx.dev, including monitoring.zx.dev, is down with it"; - }) - (mkEndpoint { - name = "grafana"; - group = "glyph"; - # Through spore's nginx, so this also fails when the proxy does. - url = "https://monitoring.zx.dev/api/health"; - conditions = [ - "[STATUS] == 200" - "[BODY].database == ok" - ]; - description = "Grafana is down or cannot reach its database; Grafana alerts will not fire"; - }) - (mkEndpoint { - name = "certificate"; - group = "spore"; - # *.zx.dev, zx.dev and cjohns.com share one ACME cert on spore. - # NixOS renews at 30 days left, so under 21 means renewal has - # been failing for over a week. - url = "https://zx.dev"; - interval = "1h"; - conditions = ["[CERTIFICATE_EXPIRATION] > 504h"]; - description = "The zx.dev certificate expires in under 21 days; check acme-zx.dev.service on spore"; - }) + endpoints = + [ + (mkEndpoint { + name = "reachable"; + group = "glyph"; + url = "tcp://${glyph}:22"; + conditions = ["[CONNECTED] == true"]; + description = "glyph is unreachable over the tailnet; Prometheus, Loki and Grafana's database are down with it"; + }) + (mkEndpoint { + name = "postgres"; + group = "glyph"; + url = "tcp://${glyph}:5432"; + conditions = ["[CONNECTED] == true"]; + description = "PostgreSQL on glyph is not accepting connections; Grafana, pocket-id and others depend on it"; + }) + (mkEndpoint { + name = "prometheus"; + group = "glyph"; + # An instant query only returns samples from the last 5m, so an empty + # result means scraping or ingestion has stalled. + url = "http://${glyph}:9099/api/v1/query?query=up"; + conditions = [ + "[STATUS] == 200" + "len([BODY].data.result) > 0" + ]; + description = "Prometheus is down or has no samples from the last 5m"; + }) + (mkEndpoint { + name = "loki"; + group = "glyph"; + # zeta's own journal shipped through Alloy proves end-to-end ingestion. + url = "http://${glyph}:3100/loki/api/v1/query?query=sum(count_over_time(%7Bhost%3D%22zeta%22%7D%5B10m%5D))"; + conditions = [ + "[STATUS] == 200" + "len([BODY].data.result) > 0" + ]; + description = "Loki is down or has received no logs from zeta in 10m"; + }) + (mkEndpoint { + name = "audiomuse"; + group = "glyph"; + # Unauthenticated liveness probe; covers the flask container only + url = "http://${glyph}:8001/api/health"; + conditions = [ + "[STATUS] == 200" + "[BODY].status == ok" + ]; + description = "AudioMuse-AI's web container on glyph is not answering, so Navidrome's audiomuseai plugin can't reach it"; + }) + (mkEndpoint { + name = "paseo"; + group = "glyph"; + # Unauthenticated liveness probe, through spore: glyph only accepts + # Paseo's port from spore + url = "https://agents.zx.dev/api/health"; + conditions = ["[STATUS] == 200"]; + description = "Paseo on glyph is not answering through spore, so agents.zx.dev and its agents are down"; + }) + (mkEndpoint { + name = "reachable"; + group = "spore"; + url = "tcp://${spore}:22"; + conditions = ["[CONNECTED] == true"]; + description = "spore is unreachable over the tailnet; *.zx.dev, including monitoring.zx.dev, is down with it"; + }) + (mkEndpoint { + name = "grafana"; + group = "glyph"; + # Through spore's nginx, so this also fails when the proxy does. + url = "https://monitoring.zx.dev/api/health"; + conditions = [ + "[STATUS] == 200" + "[BODY].database == ok" + ]; + description = "Grafana is down or cannot reach its database; Grafana alerts will not fire"; + }) + (mkEndpoint { + name = "certificate"; + group = "spore"; + # *.zx.dev, zx.dev and cjohns.com share one ACME cert on spore. + # NixOS renews at 30 days left, so under 21 means renewal has + # been failing for over a week. + url = "https://zx.dev"; + interval = "1h"; + conditions = ["[CERTIFICATE_EXPIRATION] > 504h"]; + description = "The zx.dev certificate expires in under 21 days; check acme-zx.dev.service on spore"; + }) - # Public sites, checked through their public URLs, so a failure can - # be DNS, the certificate, spore's nginx or the backend on glyph. - (mkEndpoint { - name = "jellyfin"; - group = "sites"; - url = "https://media.zx.dev/health"; - conditions = [ - "[STATUS] == 200" - "[BODY] == Healthy" - ]; - description = "media.zx.dev is down or unhealthy; check jellyfin.service on glyph"; - }) - (mkEndpoint { - name = "navidrome"; - group = "sites"; - url = "https://music.zx.dev/ping"; - conditions = ["[STATUS] == 200"]; - description = "music.zx.dev (Navidrome) is down; check navidrome.service on glyph"; - }) - (mkEndpoint { - name = "open-webui"; - group = "sites"; - url = "https://chat.zx.dev/health"; - conditions = [ - "[STATUS] == 200" - "[BODY].status == true" - ]; - description = "chat.zx.dev (Open WebUI) is down; check open-webui.service on glyph"; - }) - (mkEndpoint { - name = "freshrss"; - group = "sites"; - # Unauthenticated self-test of the Google Reader API that Reeder - # syncs through. It answers FAIL unless it can read an Authorization - # header, so this also catches a proxy that drops it. FreshRSS logs - # a warning per call, hence the longer interval. - url = "https://rss.zx.dev/api/greader.php/check/compatibility"; - headers.Authorization = "GoogleLogin auth=test/1"; - interval = "5m"; - conditions = [ - "[STATUS] == 200" - "[BODY] == PASS" - ]; - description = "rss.zx.dev (FreshRSS) API is down or failing its self-test; Reeder can't sync. Check phpfpm-freshrss.service and nginx on glyph"; - }) - (mkEndpoint { - name = "pocket-id"; - group = "sites"; - # Every requireAuth site and Grafana's login depend on this. - url = "https://id.zx.dev/.well-known/openid-configuration"; - conditions = [ - "[STATUS] == 200" - "[BODY].issuer == https://id.zx.dev" - ]; - description = "id.zx.dev (Pocket ID) is down; logins to Grafana and every auth-protected site fail. Check pocket-id.service on spore and its database on glyph"; - }) - (mkEndpoint { - name = "home-assistant"; - group = "sites"; - # Unauthenticated, so a healthy Home Assistant answers 401. A 400 - # means it rejected spore's proxy (X-Forwarded-For from a proxy - # missing from its UI-managed trusted proxies), as on 2026-10-14. - url = "https://home.zx.dev/api/"; - conditions = ["[STATUS] == 401"]; - description = "home.zx.dev (Home Assistant) is down or rejecting spore's proxy (400 means Trusted proxies in Settings > System > Network no longer includes 100.64.0.0/10); check home-assistant.service on zeta"; - }) - (mkEndpoint { - name = "models"; - group = "sites"; - # spore proxies / to oMLX's admin UI on Stroma. The UI loads without - # an API key, so this needs no credentials. - url = "https://models.zx.dev/"; - conditions = ["[STATUS] == 200"]; - description = "models.zx.dev (oMLX) is down; check oMLX on Stroma (it may be asleep or restarting) and the tailnet link from spore"; - }) - ]; + # Public sites, checked through their public URLs, so a failure can + # be DNS, the certificate, spore's nginx or the backend on glyph. + (mkEndpoint { + name = "jellyfin"; + group = "sites"; + url = "https://media.zx.dev/health"; + conditions = [ + "[STATUS] == 200" + "[BODY] == Healthy" + ]; + description = "media.zx.dev is down or unhealthy; check jellyfin.service on glyph"; + }) + (mkEndpoint { + name = "navidrome"; + group = "sites"; + url = "https://music.zx.dev/ping"; + conditions = ["[STATUS] == 200"]; + description = "music.zx.dev (Navidrome) is down; check navidrome.service on glyph"; + }) + (mkEndpoint { + name = "open-webui"; + group = "sites"; + url = "https://chat.zx.dev/health"; + conditions = [ + "[STATUS] == 200" + "[BODY].status == true" + ]; + description = "chat.zx.dev (Open WebUI) is down; check open-webui.service on glyph"; + }) + (mkEndpoint { + name = "freshrss"; + group = "sites"; + # Unauthenticated self-test of the Google Reader API that Reeder + # syncs through. It answers FAIL unless it can read an Authorization + # header, so this also catches a proxy that drops it. FreshRSS logs + # a warning per call, hence the longer interval. + url = "https://rss.zx.dev/api/greader.php/check/compatibility"; + headers.Authorization = "GoogleLogin auth=test/1"; + interval = "5m"; + conditions = [ + "[STATUS] == 200" + "[BODY] == PASS" + ]; + description = "rss.zx.dev (FreshRSS) API is down or failing its self-test; Reeder can't sync. Check phpfpm-freshrss.service and nginx on glyph"; + }) + (mkEndpoint { + name = "pocket-id"; + group = "sites"; + # Every requireAuth site and Grafana's login depend on this. + url = "https://id.zx.dev/.well-known/openid-configuration"; + conditions = [ + "[STATUS] == 200" + "[BODY].issuer == https://id.zx.dev" + ]; + description = "id.zx.dev (Pocket ID) is down; logins to Grafana and every auth-protected site fail. Check pocket-id.service on spore and its database on glyph"; + }) + (mkEndpoint { + name = "home-assistant"; + group = "sites"; + # Unauthenticated, so a healthy Home Assistant answers 401. A 400 + # means it rejected spore's proxy (X-Forwarded-For from a proxy + # missing from its UI-managed trusted proxies), as on 2026-10-14. + url = "https://home.zx.dev/api/"; + conditions = ["[STATUS] == 401"]; + description = "home.zx.dev (Home Assistant) is down or rejecting spore's proxy (400 means Trusted proxies in Settings > System > Network no longer includes 100.64.0.0/10); check home-assistant.service on zeta"; + }) + (mkEndpoint { + name = "models"; + group = "sites"; + # spore proxies / to oMLX's admin UI on Stroma. The UI loads without + # an API key, so this needs no credentials. + url = "https://models.zx.dev/"; + conditions = ["[STATUS] == 200"]; + description = "models.zx.dev (oMLX) is down; check oMLX on Stroma (it may be asleep or restarting) and the tailnet link from spore"; + }) + ] + ++ lib.mapAttrsToList (name: port: + mkEndpoint { + inherit name; + group = "exposure"; + # zx.dev's A record is spore's public address + url = "tcp://zx.dev:${toString port}"; + interval = "5m"; + conditions = ["[CONNECTED] == false"]; + description = "${name} (port ${toString port}) on spore accepts connections from the internet; it should be tailnet-only. Check networking.firewall on spore and docs/ports.md"; + }) + sporePrivatePorts; # Results pushed by tailscale-key-expiry below; Gatus conditions can't # compare dates in a response body.