diff --git a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs index 13de9f2..4aeced6 100644 --- a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs +++ b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs @@ -266,6 +266,113 @@ public void Deploy_DoesNotMaskThePodmanNetworkHelper() Assert.DoesNotContain("mask podman-user-wait-network-online", script); } + // ---- observe: prune timer + opt-in socket (#283 phase 3) -------------- + + [Fact] + public void Deploy_InstallsAWeeklyPruneTimer_ThatNeverTouchesVolumes() + { + var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + + Assert.Contains("podman-system-prune.timer", script); + Assert.Contains("OnCalendar=weekly", script); + Assert.Contains("podman system prune -af --filter until=168h", script); + // NOT --volumes: an unattached volume may hold a stopped service's database, and + // losing it to a housekeeping timer would be unrecoverable. + Assert.DoesNotContain("--volumes", script); + // User units, so it prunes the rootless store rather than root's. + Assert.Contains("/home/podman/.config/systemd/user/podman-system-prune.timer", script); + } + + [Fact] + public void Deploy_LeavesTheApiSocketOff_UnlessExplicitlyRequested() + { + var off = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + // The ROOT socket is masked either way; the point here is that no USER socket appears + // unless a host actually needs one, so the default posture has no API surface at all. + Assert.DoesNotContain("enable --now podman.socket", off); + Assert.Contains("mask podman.socket", off); + + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["userSocket"] = "true"; + var on = Build(shape); + Assert.Contains("systemctl --user enable --now podman.socket", on); + Assert.Contains("mask podman.socket", on); // root socket STILL masked + } + + [Fact] + public void Marker_ChangesWhenTheUserSocketIsToggled() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + var before = PodmanProvisioner.DesiredMarker(shape); + shape.Spec.Config["userSocket"] = "true"; + Assert.NotEqual(before, PodmanProvisioner.DesiredMarker(shape)); + } + + [Fact] + public void Deploy_SetsUpPlatformHousekeeping_BeforeStartingAppUnits() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["userSocket"] = "true"; + var script = Build(shape); + + // Regression guard: when podman-exporter failed to start on CT 4001/5114, `set -e` + // aborted the script and the prune timer + socket steps never ran at all. Platform + // housekeeping must not be hostage to an application unit — and a unit that + // Requires=podman.socket needs the socket enabled before it starts, too. + var timer = script.IndexOf("podman-system-prune.timer", StringComparison.Ordinal); + var sock = script.IndexOf("enable --now podman.socket", StringComparison.Ordinal); + var start = script.IndexOf("systemctl --user restart", StringComparison.Ordinal); + Assert.True(timer >= 0 && sock >= 0 && start >= 0); + Assert.True(timer < start, "prune timer must be installed before app units start"); + Assert.True(sock < start, "the API socket must be enabled before a unit that Requires= it"); + } + + [Fact] + public void Deploy_InstallsCockpit_AndUnlocksTheRootlessUserSoItCanLogIn() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["cockpit"] = "true"; + var script = PodmanProvisioner.BuildDeploy( + shape, "podman", "m", "/home/podman/.homelab-managed", + PodmanProvisioner.QuadletFiles(shape), new Dictionary(), "s3kret"); + + Assert.Contains("cockpit cockpit-podman", script); + // The SOCKET must be enabled AFTER app units restart: binding :9090 while Prometheus + // still holds its old published port fails with `Result: resources` (hit on CT 4001). + var install = script.IndexOf("cockpit cockpit-podman", StringComparison.Ordinal); + var restart = script.IndexOf("systemctl --user restart", StringComparison.Ordinal); + var sock = script.IndexOf("enable --now cockpit.socket", StringComparison.Ordinal); + Assert.True(install < restart, "install the package before units, so it exists even if one fails"); + Assert.True(restart < sock, "enable cockpit.socket only after units have rebound their ports"); + // The point: rootless containers live only in the `podman` user's session, and both + // podman and root ship password-LOCKED on a community-scripts CT — so without this + // Cockpit installs but nobody can log in. + Assert.Contains("'podman:s3kret' | chpasswd", script); + } + + [Fact] + public void Deploy_OmitsCockpit_WhenNotRequested() + { + var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + Assert.DoesNotContain("cockpit", script); + Assert.DoesNotContain("chpasswd", script); + } + + [Fact] + public async Task Fails_WhenCockpitIsRequestedWithoutAPassword() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["cockpit"] = "true"; + Environment.SetEnvironmentVariable("PODMAN_USER_PASSWORD", null); + + var exec = new FakeNodeExec(_ => new ExecResult(0, "", "")); + var result = await new PodmanProvisioner().ApplyAsync(shape, Ctx(exec)); + + // Installing a login UI nobody can log into is worse than not installing it. + Assert.Equal(ApplyOutcome.Failed, result.Outcome); + Assert.Contains("PODMAN_USER_PASSWORD", result.Message); + } + // ---- assets (#303) ---------------------------------------------------- // Adds an assets tree to a shape's stack dir and points config.assets at it. diff --git a/Infrastructure/engine/Converge/PodmanProvisioner.cs b/Infrastructure/engine/Converge/PodmanProvisioner.cs index d91254a..2a09c8c 100644 --- a/Infrastructure/engine/Converge/PodmanProvisioner.cs +++ b/Infrastructure/engine/Converge/PodmanProvisioner.cs @@ -86,6 +86,14 @@ public IEnumerable PlanSteps(Shape s) if (AutoUpdate(s)) yield return "enable podman-auto-update.timer (--user) — replaces Watchtower, no docker.sock"; + + yield return "install + enable podman-system-prune.timer (weekly, images/containers/networks — never volumes)"; + + if (UserSocket(s)) + yield return "enable the ROOTLESS podman.socket (--user) — opt-in, for a metrics exporter; the ROOT socket stays masked"; + + if (Cockpit(s)) + yield return "install cockpit + cockpit-podman on :9090, set the `podman` user's password (PODMAN_USER_PASSWORD) so it can log in"; } // Stable marker over every managed input. Quadlet CONTENT is included (not just names), @@ -99,6 +107,8 @@ public static string DesiredMarker(Shape s) start.ToString(), count.ToString(), AutoUpdate(s) ? "au=1" : "au=0", + UserSocket(s) ? "usock=1" : "usock=0", + Cockpit(s) ? "cockpit=1" : "cockpit=0", string.Join(",", SecretNames(s).Select(kv => $"{kv.Key}={kv.Value}")), }; foreach (var f in QuadletFiles(s)) @@ -159,8 +169,20 @@ public async Task ApplyAsync(Shape s, ConvergeContext ctx) var (assetMsg, assetFailed) = await PushAssetsAsync(s, ctx, node, ctid, user); if (assetFailed is not null) return ApplyResult.Failed(assetFailed); + // Cockpit needs a PAM password for the rootless user; without it the UI installs but + // nobody can log in, which is worse than not installing it. + string? cockpitPassword = null; + if (Cockpit(s)) + { + if (ctx.Secrets.Get("PODMAN_USER_PASSWORD") is not { Length: > 0 } pw) + return ApplyResult.Failed( + "cockpit: true needs PODMAN_USER_PASSWORD in secrets.env — Cockpit authenticates the " + + "rootless user via PAM, and it ships password-locked"); + cockpitPassword = pw; + } + var files = QuadletFiles(s); - var script = BuildDeploy(s, user, marker, markerPath, files, secretValues); + var script = BuildDeploy(s, user, marker, markerPath, files, secretValues, cockpitPassword); var res = await ctx.Exec.InContainerAsync(node, ctid, script); if (!res.Ok) return ApplyResult.Failed($"podman host setup failed: {res.Stderr}"); @@ -348,7 +370,8 @@ internal static Dictionary ParseFeatures(string pctConfig) // user is usable, linger before any `systemctl --user`, and the marker is stamped last. internal static string BuildDeploy( Shape s, string user, string marker, string markerPath, - IReadOnlyList files, IReadOnlyDictionary secrets) + IReadOnlyList files, IReadOnlyDictionary secrets, + string? cockpitPassword = null) { var (start, count) = SubidRange(s); var sb = new StringBuilder(); @@ -491,6 +514,62 @@ internal static string BuildDeploy( sb.Append($"echo {b64} | base64 -d | {UserCmd(user, $"podman secret create {name} -")}\n"); } + // 7b. Weekly prune timer — BEFORE starting app units, deliberately. Platform + // housekeeping must not be hostage to an application unit: when podman-exporter + // failed to start on CT 4001/5114 (#283 phase 3), `set -e` aborted the script and + // the timer + socket steps never ran at all. + // Weekly prune timer — the other half of "podman-native replaces the Watchtower + + // prune sidecars" (ADR-0009). Written as USER units so it prunes the rootless + // store, not root's. + // + // Deliberately NOT `--volumes`: an unattached volume may still hold real data (a + // stopped service's database), and losing it to a housekeeping timer would be + // unrecoverable. Images/containers/networks only, and only things older than a week. + sb.Append($"install -d -o {user} -g {user} -m 755 /home/{user}/.config/systemd/user\n"); + sb.Append($"printf '%s\\n' '[Unit]' 'Description=Prune unused podman images, containers and networks' " + + "'Documentation=ADR-0009 phase 3' '' '[Service]' 'Type=oneshot' " + + "'ExecStart=/usr/bin/podman system prune -af --filter until=168h' " + + $"> /home/{user}/.config/systemd/user/podman-system-prune.service\n"); + sb.Append($"printf '%s\\n' '[Unit]' 'Description=Weekly podman prune' '' '[Timer]' " + + "'OnCalendar=weekly' 'Persistent=true' 'RandomizedDelaySec=1h' '' '[Install]' " + + $"'WantedBy=timers.target' > /home/{user}/.config/systemd/user/podman-system-prune.timer\n"); + sb.Append($"chown -R {user}:{user} /home/{user}/.config/systemd\n"); + sb.Append($"{UserCmd(user, "systemctl --user daemon-reload")}\n"); + sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-system-prune.timer")}\n"); + + // 7c. The ROOTLESS user API socket — opt-in via `config.userSocket: true`. + // + // ADR-0009 masks the ROOT podman.socket, and that stays masked. This is a + // categorically smaller thing: a socket owned by the unprivileged `podman` user, + // inside its own userns, reachable only by that user. It exists because a metrics + // exporter has no other way to enumerate containers (ADR-0009 phase 3, "observe"). + // Off by default, so a host that doesn't export metrics still has no API surface. + if (UserSocket(s)) + sb.Append($"{UserCmd(user, "systemctl --user enable --now podman.socket")}\n"); + + // 7d. Cockpit — the management UI half of ADR-0009 phase 3. + // + // Deliberately logs in as the `podman` USER, not root: rootless containers exist + // only inside that user's session, so a root Cockpit session shows an empty + // container list. That means the user needs a real PAM password — both `podman` + // and `root` ship password-LOCKED on a community-scripts CT, so Cockpit login is + // impossible until one is set. cockpit-podman then reads the same rootless socket + // the exporter uses, which is why this requires userSocket. + // + // Cockpit takes :9090, its default — which is why Prometheus publishes on 9091. + // The package + password land HERE (before app units, so they exist even if a unit + // later fails), but the SOCKET is enabled further down, after units have restarted. + // Enabling it here fails with `Result: resources`: the app unit still holds :9090 + // on its old published port until it restarts. Hit on CT 4001. + if (Cockpit(s)) + { + sb.Append("export DEBIAN_FRONTEND=noninteractive\n"); + sb.Append("if ! dpkg -s cockpit >/dev/null 2>&1; then apt-get update -qq && " + + "apt-get install -y -qq cockpit cockpit-podman; fi\n"); + if (cockpitPassword is { Length: > 0 }) + sb.Append($"printf '%s' '{user}:{cockpitPassword}' | chpasswd\n"); + } + // 8. Reload + start. daemon-reload runs the quadlet generator; each *.container // becomes .service. `restart` (not `start`) so a changed quadlet actually // takes effect on an already-running unit. @@ -505,6 +584,11 @@ internal static string BuildDeploy( if (AutoUpdate(s)) sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-auto-update.timer")}\n"); + // 9c. Cockpit's socket, LAST — after app units have rebound to their new ports. + // See 7d: binding :9090 before Prometheus moves to 9091 fails outright. + if (Cockpit(s)) + sb.Append("systemctl enable --now cockpit.socket\n"); + // 10. Mark-on-SUCCESS — only reached if every step above exited 0 under `set -e`. sb.Append($"printf '%s' '{marker}' > {markerPath}\n"); sb.Append($"chown {user}:{user} {markerPath}"); @@ -527,6 +611,17 @@ internal static string UserCmd(string user, string cmd) => internal static string User(Shape s) => s.Spec.Config.Str("user") ?? DefaultUser; + // Opt-in Cockpit management UI (ADR-0009 phase 3). Requires userSocket, since + // cockpit-podman reads the same rootless API socket to list containers. + internal static bool Cockpit(Shape s) => + s.Spec.Config.TryGetValue("cockpit", out var v) && v is not null + && v.ToString() is not ("false" or "False" or "0"); + + // Opt-in rootless API socket. Only hosts that run a metrics exporter need it. + internal static bool UserSocket(Shape s) => + s.Spec.Config.TryGetValue("userSocket", out var v) && v is not null + && v.ToString() is not ("false" or "False" or "0"); + internal static bool AutoUpdate(Shape s) => s.Spec.Config.TryGetValue("autoUpdate", out var v) && v is not null ? v.ToString() is not ("false" or "False" or "0") diff --git a/secrets.env.template b/secrets.env.template index 2da5c16..d343c31 100644 --- a/secrets.env.template +++ b/secrets.env.template @@ -106,3 +106,10 @@ GF_SECURITY_ADMIN_PASSWORD= RADARR_API_KEY= SONARR_API_KEY= PROWLARR_API_KEY= + +# ── Rootless podman hosts: Cockpit login (#283 phase 3) ── +# Password for the `podman` user on every podman host. Exists ONLY so Cockpit can authenticate +# that user via PAM — rootless containers are visible only inside its own session, so root's +# Cockpit session shows nothing. Not consumed by any service; set by the provisioner when a +# shape declares `cockpit: true`. +PODMAN_USER_PASSWORD= diff --git a/stacks/Media/podman-host.lxc.yaml b/stacks/Media/podman-host.lxc.yaml index a9db6f5..34a2601 100644 --- a/stacks/Media/podman-host.lxc.yaml +++ b/stacks/Media/podman-host.lxc.yaml @@ -62,6 +62,16 @@ spec: user: podman quadlets: podman-host/quadlets + # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009 + # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged + # `podman` user and reachable only by it. + userSocket: true + + # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user + # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and + # userSocket above. + cockpit: true + # podman secrets seeded from secrets.env, add-only. Namespaced per service, because this # host will grow more of them (the SM/template keys are YOUTARR_*, the podman secret names # are lower-case). diff --git a/stacks/Media/podman-host/quadlets/podman-exporter.container b/stacks/Media/podman-host/quadlets/podman-exporter.container new file mode 100644 index 0000000..b20b54b --- /dev/null +++ b/stacks/Media/podman-host/quadlets/podman-exporter.container @@ -0,0 +1,51 @@ +# prometheus-podman-exporter — container/pod/image metrics for this rootless host. +# +# ADR-0009 phase 3 ("observe"). The migration removed docker.sock and put nothing in its +# place, so until now the podman hosts had ZERO container-level visibility: systemd knows +# whether a unit is up, but nothing exported restart counts, image ages or per-container +# resource use. +# +# WHY THIS NEEDS A SOCKET, and why that is acceptable: an exporter has no other way to +# enumerate containers. ADR-0009 masks the ROOT podman.socket and that stays masked — this +# uses the ROOTLESS one, owned by the unprivileged 'podman' user, inside its own userns and +# reachable only by that user. Opt-in per host via config.userSocket; a host that does not +# export metrics keeps no API surface at all. +# +# %t expands to XDG_RUNTIME_DIR in a systemd USER unit, which is where the rootless socket +# lives (/run/user//podman/podman.sock). + +[Unit] +Description=Prometheus podman exporter +Documentation=https://github.com/containers/prometheus-podman-exporter +After=network-online.target podman.socket +Wants=network-online.target +Requires=podman.socket + +[Container] +Image=quay.io/navidys/prometheus-podman-exporter:latest +ContainerName=podman-exporter +Network=youtarr.network +PublishPort=9882:9882 +# Run as root INSIDE the container instead of the image's `nobody` (65534). Under the DEFAULT +# rootless mapping, container-root maps to the unprivileged `podman` user on the host — which +# is exactly who owns the socket below, so access works with no keep-id at all. +# +# Two failed attempts got here: the image's own 65534 is outside the subuid window an LXC's +# 65536-uid map allows, so crun refused it ("setgroups: Invalid argument"); adding +# keep-id:uid=65534 fixed that but left the process with no home, and it died on +# "stat /home/.config: no such file or directory" while looking for containers.conf. +# Running as root sidesteps both, and is no privilege gain — container-root here IS the +# unprivileged podman user. +User=0 +Group=0 +Volume=%t/podman/podman.sock:/run/podman/podman.sock:ro +Environment=CONTAINER_HOST=unix:///run/podman/podman.sock +Exec=--collector.enable-all +AutoUpdate=registry + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/stacks/monitoring/podman-host.lxc.yaml b/stacks/monitoring/podman-host.lxc.yaml index ac03d71..5809be4 100644 --- a/stacks/monitoring/podman-host.lxc.yaml +++ b/stacks/monitoring/podman-host.lxc.yaml @@ -65,6 +65,16 @@ spec: user: podman quadlets: podman-host/quadlets + # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009 + # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged + # `podman` user and reachable only by it. + userSocket: true + + # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user + # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and + # userSocket above. + cockpit: true + # Config tree rendered onto the host (#305). Quadlets bind-mount from here read-only. assets: podman-host/assets assetsTarget: /home/podman/monitoring diff --git a/stacks/monitoring/podman-host/assets/config/prometheus.yml b/stacks/monitoring/podman-host/assets/config/prometheus.yml index 3fa13be..49dec2b 100644 --- a/stacks/monitoring/podman-host/assets/config/prometheus.yml +++ b/stacks/monitoring/podman-host/assets/config/prometheus.yml @@ -34,3 +34,20 @@ scrape_configs: - job_name: 'otel-collector' static_configs: - targets: ['otel-collector:8889'] + + # Rootless podman hosts (ADR-0009 phase 3). Container/pod/image metrics from + # prometheus-podman-exporter on each host. The Monitoring host is scraped by container + # name over the shared quadlet network; the Media host by its internal DNS name (reserved + registered in UniFi, so no IP is + # hard-coded here either). + # + # The SmartHome host (CT 6004) is deliberately ABSENT: it lives on the IoT VLAN 1040, + # which is firewalled off from 1010, so Prometheus cannot reach it. Verified — a TCP probe + # from CT 4001 to that host fails. Adding it needs a narrow UniFi allowance (1010 -> 1040 + # on :9882), which is a deliberate loosening of the IoT VLAN's isolation and therefore an + # owner decision, not an implementation detail. + - job_name: 'podman' + static_configs: + - targets: ['podman-exporter:9882'] + labels: { host: 'monitoring-podman-host', ctid: '4001' } + - targets: ['media.homelab.chrison.internal:9882'] + labels: { host: 'media-podman-host', ctid: '5114' } diff --git a/stacks/monitoring/podman-host/quadlets/podman-exporter.container b/stacks/monitoring/podman-host/quadlets/podman-exporter.container new file mode 100644 index 0000000..7f2bd58 --- /dev/null +++ b/stacks/monitoring/podman-host/quadlets/podman-exporter.container @@ -0,0 +1,51 @@ +# prometheus-podman-exporter — container/pod/image metrics for this rootless host. +# +# ADR-0009 phase 3 ("observe"). The migration removed docker.sock and put nothing in its +# place, so until now the podman hosts had ZERO container-level visibility: systemd knows +# whether a unit is up, but nothing exported restart counts, image ages or per-container +# resource use. +# +# WHY THIS NEEDS A SOCKET, and why that is acceptable: an exporter has no other way to +# enumerate containers. ADR-0009 masks the ROOT podman.socket and that stays masked — this +# uses the ROOTLESS one, owned by the unprivileged 'podman' user, inside its own userns and +# reachable only by that user. Opt-in per host via config.userSocket; a host that does not +# export metrics keeps no API surface at all. +# +# %t expands to XDG_RUNTIME_DIR in a systemd USER unit, which is where the rootless socket +# lives (/run/user//podman/podman.sock). + +[Unit] +Description=Prometheus podman exporter +Documentation=https://github.com/containers/prometheus-podman-exporter +After=network-online.target podman.socket +Wants=network-online.target +Requires=podman.socket + +[Container] +Image=quay.io/navidys/prometheus-podman-exporter:latest +ContainerName=podman-exporter +Network=monitoring.network +PublishPort=9882:9882 +# Run as root INSIDE the container instead of the image's `nobody` (65534). Under the DEFAULT +# rootless mapping, container-root maps to the unprivileged `podman` user on the host — which +# is exactly who owns the socket below, so access works with no keep-id at all. +# +# Two failed attempts got here: the image's own 65534 is outside the subuid window an LXC's +# 65536-uid map allows, so crun refused it ("setgroups: Invalid argument"); adding +# keep-id:uid=65534 fixed that but left the process with no home, and it died on +# "stat /home/.config: no such file or directory" while looking for containers.conf. +# Running as root sidesteps both, and is no privilege gain — container-root here IS the +# unprivileged podman user. +User=0 +Group=0 +Volume=%t/podman/podman.sock:/run/podman/podman.sock:ro +Environment=CONTAINER_HOST=unix:///run/podman/podman.sock +Exec=--collector.enable-all +AutoUpdate=registry + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/stacks/monitoring/podman-host/quadlets/prometheus.container b/stacks/monitoring/podman-host/quadlets/prometheus.container index 2088abe..5be26dd 100644 --- a/stacks/monitoring/podman-host/quadlets/prometheus.container +++ b/stacks/monitoring/podman-host/quadlets/prometheus.container @@ -13,7 +13,11 @@ Wants=network-online.target Image=docker.io/prom/prometheus ContainerName=prometheus Network=monitoring.network -PublishPort=9090:9090 +# Published on 9091, NOT 9090 — Cockpit owns 9090 on the podman hosts as the management UI +# (ADR-0009 phase 3). Only the HOST-side port moves: the container still listens on 9090, so +# Grafana's datasource (http://prometheus:9090, over the quadlet network) is unaffected and +# needs no change. Browse Prometheus at monitoring.homelab.chrison.internal:9091. +PublishPort=9091:9090 # Runs as nobody (uid 65534) — verified on the live container, not assumed. That is outside the # subuid window the LXC's 65536-uid map allows, so without keep-id crun refuses to start it. # There is no data volume here (Prometheus's TSDB is ephemeral in this stack, as it was under