Skip to content

Repository navigation

feat(zeta): alert if spore's tailnet-only ports reach the internet #749

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stackptr merged 1 commit into from
Oct 26, 2026
Merged

feat(zeta): alert if spore's tailnet-only ports reach the internet #749

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
353 changes: 190 additions & 163 deletions hosts/zeta/services/gatus.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# Slack directly. Grafana in turn alerts if zeta's node exporter goes away.
{
config,
lib,
pkgs,
...
}: let
Expand All @@ -16,6 +17,20 @@
# server can lock every device out (as on 2026-10-11).
servers = ["glyph" "spore" "zeta" "stroma"];

# Ports spore's services listen on that must stay off the internet,
# probed on its public address. zeta reaches that over the internet, not
# the tailnet, so spore's firewall applies. The daily port scan workflow
# covers every port; these catch a regression within minutes.
sporePrivatePorts = {
node-exporter = 9100;
nginx-exporter = 9113;
alloy = 12345;
glances = 61208;
homepage = 8082;
pocket-id = 1411;
oauth2-proxy = 4180;
};

alert = description: {
type = "custom";
inherit description;
Expand Down Expand Up @@ -74,170 +89,182 @@ in {
};
};

endpoints = [
(mkEndpoint {
name = "reachable";
group = "glyph";
url = "tcp://${glyph}:22";
conditions = ["[CONNECTED] == true"];
description = "glyph is unreachable over the tailnet; Prometheus, Loki and Grafana's database are down with it";
})
(mkEndpoint {
name = "postgres";
group = "glyph";
url = "tcp://${glyph}:5432";
conditions = ["[CONNECTED] == true"];
description = "PostgreSQL on glyph is not accepting connections; Grafana, pocket-id and others depend on it";
})
(mkEndpoint {
name = "prometheus";
group = "glyph";
# An instant query only returns samples from the last 5m, so an empty
# result means scraping or ingestion has stalled.
url = "http://${glyph}:9099/api/v1/query?query=up";
conditions = [
"[STATUS] == 200"
"len([BODY].data.result) > 0"
];
description = "Prometheus is down or has no samples from the last 5m";
})
(mkEndpoint {
name = "loki";
group = "glyph";
# zeta's own journal shipped through Alloy proves end-to-end ingestion.
url = "http://${glyph}:3100/loki/api/v1/query?query=sum(count_over_time(%7Bhost%3D%22zeta%22%7D%5B10m%5D))";
conditions = [
"[STATUS] == 200"
"len([BODY].data.result) > 0"
];
description = "Loki is down or has received no logs from zeta in 10m";
})
(mkEndpoint {
name = "audiomuse";
group = "glyph";
# Unauthenticated liveness probe; covers the flask container only
url = "http://${glyph}:8001/api/health";
conditions = [
"[STATUS] == 200"
"[BODY].status == ok"
];
description = "AudioMuse-AI's web container on glyph is not answering, so Navidrome's audiomuseai plugin can't reach it";
})
(mkEndpoint {
name = "paseo";
group = "glyph";
# Unauthenticated liveness probe, through spore: glyph only accepts
# Paseo's port from spore
url = "https://agents.zx.dev/api/health";
conditions = ["[STATUS] == 200"];
description = "Paseo on glyph is not answering through spore, so agents.zx.dev and its agents are down";
})
(mkEndpoint {
name = "reachable";
group = "spore";
url = "tcp://${spore}:22";
conditions = ["[CONNECTED] == true"];
description = "spore is unreachable over the tailnet; *.zx.dev, including monitoring.zx.dev, is down with it";
})
(mkEndpoint {
name = "grafana";
group = "glyph";
# Through spore's nginx, so this also fails when the proxy does.
url = "https://monitoring.zx.dev/api/health";
conditions = [
"[STATUS] == 200"
"[BODY].database == ok"
];
description = "Grafana is down or cannot reach its database; Grafana alerts will not fire";
})
(mkEndpoint {
name = "certificate";
group = "spore";
# *.zx.dev, zx.dev and cjohns.com share one ACME cert on spore.
# NixOS renews at 30 days left, so under 21 means renewal has
# been failing for over a week.
url = "https://zx.dev";
interval = "1h";
conditions = ["[CERTIFICATE_EXPIRATION] > 504h"];
description = "The zx.dev certificate expires in under 21 days; check acme-zx.dev.service on spore";
})
endpoints =
[
(mkEndpoint {
name = "reachable";
group = "glyph";
url = "tcp://${glyph}:22";
conditions = ["[CONNECTED] == true"];
description = "glyph is unreachable over the tailnet; Prometheus, Loki and Grafana's database are down with it";
})
(mkEndpoint {
name = "postgres";
group = "glyph";
url = "tcp://${glyph}:5432";
conditions = ["[CONNECTED] == true"];
description = "PostgreSQL on glyph is not accepting connections; Grafana, pocket-id and others depend on it";
})
(mkEndpoint {
name = "prometheus";
group = "glyph";
# An instant query only returns samples from the last 5m, so an empty
# result means scraping or ingestion has stalled.
url = "http://${glyph}:9099/api/v1/query?query=up";
conditions = [
"[STATUS] == 200"
"len([BODY].data.result) > 0"
];
description = "Prometheus is down or has no samples from the last 5m";
})
(mkEndpoint {
name = "loki";
group = "glyph";
# zeta's own journal shipped through Alloy proves end-to-end ingestion.
url = "http://${glyph}:3100/loki/api/v1/query?query=sum(count_over_time(%7Bhost%3D%22zeta%22%7D%5B10m%5D))";
conditions = [
"[STATUS] == 200"
"len([BODY].data.result) > 0"
];
description = "Loki is down or has received no logs from zeta in 10m";
})
(mkEndpoint {
name = "audiomuse";
group = "glyph";
# Unauthenticated liveness probe; covers the flask container only
url = "http://${glyph}:8001/api/health";
conditions = [
"[STATUS] == 200"
"[BODY].status == ok"
];
description = "AudioMuse-AI's web container on glyph is not answering, so Navidrome's audiomuseai plugin can't reach it";
})
(mkEndpoint {
name = "paseo";
group = "glyph";
# Unauthenticated liveness probe, through spore: glyph only accepts
# Paseo's port from spore
url = "https://agents.zx.dev/api/health";
conditions = ["[STATUS] == 200"];
description = "Paseo on glyph is not answering through spore, so agents.zx.dev and its agents are down";
})
(mkEndpoint {
name = "reachable";
group = "spore";
url = "tcp://${spore}:22";
conditions = ["[CONNECTED] == true"];
description = "spore is unreachable over the tailnet; *.zx.dev, including monitoring.zx.dev, is down with it";
})
(mkEndpoint {
name = "grafana";
group = "glyph";
# Through spore's nginx, so this also fails when the proxy does.
url = "https://monitoring.zx.dev/api/health";
conditions = [
"[STATUS] == 200"
"[BODY].database == ok"
];
description = "Grafana is down or cannot reach its database; Grafana alerts will not fire";
})
(mkEndpoint {
name = "certificate";
group = "spore";
# *.zx.dev, zx.dev and cjohns.com share one ACME cert on spore.
# NixOS renews at 30 days left, so under 21 means renewal has
# been failing for over a week.
url = "https://zx.dev";
interval = "1h";
conditions = ["[CERTIFICATE_EXPIRATION] > 504h"];
description = "The zx.dev certificate expires in under 21 days; check acme-zx.dev.service on spore";
})

# Public sites, checked through their public URLs, so a failure can
# be DNS, the certificate, spore's nginx or the backend on glyph.
(mkEndpoint {
name = "jellyfin";
group = "sites";
url = "https://media.zx.dev/health";
conditions = [
"[STATUS] == 200"
"[BODY] == Healthy"
];
description = "media.zx.dev is down or unhealthy; check jellyfin.service on glyph";
})
(mkEndpoint {
name = "navidrome";
group = "sites";
url = "https://music.zx.dev/ping";
conditions = ["[STATUS] == 200"];
description = "music.zx.dev (Navidrome) is down; check navidrome.service on glyph";
})
(mkEndpoint {
name = "open-webui";
group = "sites";
url = "https://chat.zx.dev/health";
conditions = [
"[STATUS] == 200"
"[BODY].status == true"
];
description = "chat.zx.dev (Open WebUI) is down; check open-webui.service on glyph";
})
(mkEndpoint {
name = "freshrss";
group = "sites";
# Unauthenticated self-test of the Google Reader API that Reeder
# syncs through. It answers FAIL unless it can read an Authorization
# header, so this also catches a proxy that drops it. FreshRSS logs
# a warning per call, hence the longer interval.
url = "https://rss.zx.dev/api/greader.php/check/compatibility";
headers.Authorization = "GoogleLogin auth=test/1";
interval = "5m";
conditions = [
"[STATUS] == 200"
"[BODY] == PASS"
];
description = "rss.zx.dev (FreshRSS) API is down or failing its self-test; Reeder can't sync. Check phpfpm-freshrss.service and nginx on glyph";
})
(mkEndpoint {
name = "pocket-id";
group = "sites";
# Every requireAuth site and Grafana's login depend on this.
url = "https://id.zx.dev/.well-known/openid-configuration";
conditions = [
"[STATUS] == 200"
"[BODY].issuer == https://id.zx.dev"
];
description = "id.zx.dev (Pocket ID) is down; logins to Grafana and every auth-protected site fail. Check pocket-id.service on spore and its database on glyph";
})
(mkEndpoint {
name = "home-assistant";
group = "sites";
# Unauthenticated, so a healthy Home Assistant answers 401. A 400
# means it rejected spore's proxy (X-Forwarded-For from a proxy
# missing from its UI-managed trusted proxies), as on 2026-10-14.
url = "https://home.zx.dev/api/";
conditions = ["[STATUS] == 401"];
description = "home.zx.dev (Home Assistant) is down or rejecting spore's proxy (400 means Trusted proxies in Settings > System > Network no longer includes 100.64.0.0/10); check home-assistant.service on zeta";
})
(mkEndpoint {
name = "models";
group = "sites";
# spore proxies / to oMLX's admin UI on Stroma. The UI loads without
# an API key, so this needs no credentials.
url = "https://models.zx.dev/";
conditions = ["[STATUS] == 200"];
description = "models.zx.dev (oMLX) is down; check oMLX on Stroma (it may be asleep or restarting) and the tailnet link from spore";
})
];
# Public sites, checked through their public URLs, so a failure can
# be DNS, the certificate, spore's nginx or the backend on glyph.
(mkEndpoint {
name = "jellyfin";
group = "sites";
url = "https://media.zx.dev/health";
conditions = [
"[STATUS] == 200"
"[BODY] == Healthy"
];
description = "media.zx.dev is down or unhealthy; check jellyfin.service on glyph";
})
(mkEndpoint {
name = "navidrome";
group = "sites";
url = "https://music.zx.dev/ping";
conditions = ["[STATUS] == 200"];
description = "music.zx.dev (Navidrome) is down; check navidrome.service on glyph";
})
(mkEndpoint {
name = "open-webui";
group = "sites";
url = "https://chat.zx.dev/health";
conditions = [
"[STATUS] == 200"
"[BODY].status == true"
];
description = "chat.zx.dev (Open WebUI) is down; check open-webui.service on glyph";
})
(mkEndpoint {
name = "freshrss";
group = "sites";
# Unauthenticated self-test of the Google Reader API that Reeder
# syncs through. It answers FAIL unless it can read an Authorization
# header, so this also catches a proxy that drops it. FreshRSS logs
# a warning per call, hence the longer interval.
url = "https://rss.zx.dev/api/greader.php/check/compatibility";
headers.Authorization = "GoogleLogin auth=test/1";
interval = "5m";
conditions = [
"[STATUS] == 200"
"[BODY] == PASS"
];
description = "rss.zx.dev (FreshRSS) API is down or failing its self-test; Reeder can't sync. Check phpfpm-freshrss.service and nginx on glyph";
})
(mkEndpoint {
name = "pocket-id";
group = "sites";
# Every requireAuth site and Grafana's login depend on this.
url = "https://id.zx.dev/.well-known/openid-configuration";
conditions = [
"[STATUS] == 200"
"[BODY].issuer == https://id.zx.dev"
];
description = "id.zx.dev (Pocket ID) is down; logins to Grafana and every auth-protected site fail. Check pocket-id.service on spore and its database on glyph";
})
(mkEndpoint {
name = "home-assistant";
group = "sites";
# Unauthenticated, so a healthy Home Assistant answers 401. A 400
# means it rejected spore's proxy (X-Forwarded-For from a proxy
# missing from its UI-managed trusted proxies), as on 2026-10-14.
url = "https://home.zx.dev/api/";
conditions = ["[STATUS] == 401"];
description = "home.zx.dev (Home Assistant) is down or rejecting spore's proxy (400 means Trusted proxies in Settings > System > Network no longer includes 100.64.0.0/10); check home-assistant.service on zeta";
})
(mkEndpoint {
name = "models";
group = "sites";
# spore proxies / to oMLX's admin UI on Stroma. The UI loads without
# an API key, so this needs no credentials.
url = "https://models.zx.dev/";
conditions = ["[STATUS] == 200"];
description = "models.zx.dev (oMLX) is down; check oMLX on Stroma (it may be asleep or restarting) and the tailnet link from spore";
})
]
++ lib.mapAttrsToList (name: port:
mkEndpoint {
inherit name;
group = "exposure";
# zx.dev's A record is spore's public address
url = "tcp://zx.dev:${toString port}";
interval = "5m";
conditions = ["[CONNECTED] == false"];
description = "${name} (port ${toString port}) on spore accepts connections from the internet; it should be tailnet-only. Check networking.firewall on spore and docs/ports.md";
})
sporePrivatePorts;

# Results pushed by tailscale-key-expiry below; Gatus conditions can't
# compare dates in a response body.
Expand Down
Loading