From ef9a239c9cf959039016a96f2fc2bd68ace4befe Mon Sep 17 00:00:00 2001 From: Chrison Simtian Date: Tue, 28 Jul 2026 10:57:13 +1200 Subject: [PATCH 1/2] feat(observe): podman metrics + prune timer (ADR-0009 phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advances #283 phase 3 ("observe"). The migration removed docker.sock and put nothing in its place, so the three rootless hosts had ZERO container-level visibility: systemd knew whether a unit was up, but nothing exported restart counts, image ages or per-container resource use. Engine (platform-wide): - Weekly `podman-system-prune.timer` as USER units, replacing the prune sidecar ADR-0009 retired. Deliberately NOT --volumes: an unattached volume may hold a stopped service's database, and losing it to a housekeeping timer would be unrecoverable. Images/containers/networks only, older than 168h. - Opt-in rootless API socket via `config.userSocket`. The ROOT podman.socket stays masked — this is a socket owned by the unprivileged `podman` user inside its own userns, which exists because an exporter has no other way to enumerate containers. Off by default, so a host that exports nothing has no API surface. - Both now run BEFORE app units start. Found the hard way: when podman-exporter failed to start, `set -e` aborted the script and the timer + socket steps never ran at all — platform housekeeping was hostage to an application unit. A unit that Requires=podman.socket also needs the socket up first. Regression-tested. Stacks: podman-exporter quadlet on the Monitoring and Media hosts, plus a `podman` scrape job. Prometheus now reports 7/7 targets UP. The exporter took three attempts, all recorded in the quadlet header: 1. image's own uid 65534 → crun "setgroups: Invalid argument" (outside the subuid window an LXC's 65536-uid map allows); 2. keep-id:uid=65534 fixed that but left no home → died on "stat /home/.config: no such file or directory" seeking containers.conf; 3. User=0 under the DEFAULT mapping — container-root maps to the unprivileged `podman` user, which is exactly who owns the socket. No keep-id, no home hack, and no privilege gain. CT 5114 also gained the DHCP reservation + DNS name it was missing (media.homelab.chrison.internal), so the new scrape target is a NAME — the IP I first wrote there would have re-introduced exactly what #303 removed. NOT done, and deliberately: - The SmartHome host (CT 6004) is unscraped. It sits on IoT VLAN 1040, firewalled off from 1010; verified with a TCP probe from CT 4001. Reaching it needs a narrow UniFi allowance (1010 -> 1040 on :9882), which loosens the IoT VLAN's isolation and is an owner decision. - Cockpit (the other half of phase 3) is untouched: it defaults to :9090, which Prometheus already publishes on CT 4001, and it sits awkwardly against the repo's stated minimal-UI-reliance. Needs a decision, not an assumption. 138 engine tests pass. Refs #283 --- .../engine.Tests/PodmanProvisionerTests.cs | 61 +++++++++++++++++++ .../engine/Converge/PodmanProvisioner.cs | 44 +++++++++++++ stacks/Media/podman-host.lxc.yaml | 5 ++ .../quadlets/podman-exporter.container | 51 ++++++++++++++++ stacks/monitoring/podman-host.lxc.yaml | 5 ++ .../podman-host/assets/config/prometheus.yml | 17 ++++++ .../quadlets/podman-exporter.container | 51 ++++++++++++++++ 7 files changed, 234 insertions(+) create mode 100644 stacks/Media/podman-host/quadlets/podman-exporter.container create mode 100644 stacks/monitoring/podman-host/quadlets/podman-exporter.container diff --git a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs index 13de9f2..ce7568b 100644 --- a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs +++ b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs @@ -266,6 +266,67 @@ public void Deploy_DoesNotMaskThePodmanNetworkHelper() Assert.DoesNotContain("mask podman-user-wait-network-online", script); } + // ---- observe: prune timer + opt-in socket (#283 phase 3) -------------- + + [Fact] + public void Deploy_InstallsAWeeklyPruneTimer_ThatNeverTouchesVolumes() + { + var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + + Assert.Contains("podman-system-prune.timer", script); + Assert.Contains("OnCalendar=weekly", script); + Assert.Contains("podman system prune -af --filter until=168h", script); + // NOT --volumes: an unattached volume may hold a stopped service's database, and + // losing it to a housekeeping timer would be unrecoverable. + Assert.DoesNotContain("--volumes", script); + // User units, so it prunes the rootless store rather than root's. + Assert.Contains("/home/podman/.config/systemd/user/podman-system-prune.timer", script); + } + + [Fact] + public void Deploy_LeavesTheApiSocketOff_UnlessExplicitlyRequested() + { + var off = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + // The ROOT socket is masked either way; the point here is that no USER socket appears + // unless a host actually needs one, so the default posture has no API surface at all. + Assert.DoesNotContain("enable --now podman.socket", off); + Assert.Contains("mask podman.socket", off); + + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["userSocket"] = "true"; + var on = Build(shape); + Assert.Contains("systemctl --user enable --now podman.socket", on); + Assert.Contains("mask podman.socket", on); // root socket STILL masked + } + + [Fact] + public void Marker_ChangesWhenTheUserSocketIsToggled() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + var before = PodmanProvisioner.DesiredMarker(shape); + shape.Spec.Config["userSocket"] = "true"; + Assert.NotEqual(before, PodmanProvisioner.DesiredMarker(shape)); + } + + [Fact] + public void Deploy_SetsUpPlatformHousekeeping_BeforeStartingAppUnits() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["userSocket"] = "true"; + var script = Build(shape); + + // Regression guard: when podman-exporter failed to start on CT 4001/5114, `set -e` + // aborted the script and the prune timer + socket steps never ran at all. Platform + // housekeeping must not be hostage to an application unit — and a unit that + // Requires=podman.socket needs the socket enabled before it starts, too. + var timer = script.IndexOf("podman-system-prune.timer", StringComparison.Ordinal); + var sock = script.IndexOf("enable --now podman.socket", StringComparison.Ordinal); + var start = script.IndexOf("systemctl --user restart", StringComparison.Ordinal); + Assert.True(timer >= 0 && sock >= 0 && start >= 0); + Assert.True(timer < start, "prune timer must be installed before app units start"); + Assert.True(sock < start, "the API socket must be enabled before a unit that Requires= it"); + } + // ---- assets (#303) ---------------------------------------------------- // Adds an assets tree to a shape's stack dir and points config.assets at it. diff --git a/Infrastructure/engine/Converge/PodmanProvisioner.cs b/Infrastructure/engine/Converge/PodmanProvisioner.cs index d91254a..cf76b1b 100644 --- a/Infrastructure/engine/Converge/PodmanProvisioner.cs +++ b/Infrastructure/engine/Converge/PodmanProvisioner.cs @@ -86,6 +86,11 @@ public IEnumerable PlanSteps(Shape s) if (AutoUpdate(s)) yield return "enable podman-auto-update.timer (--user) — replaces Watchtower, no docker.sock"; + + yield return "install + enable podman-system-prune.timer (weekly, images/containers/networks — never volumes)"; + + if (UserSocket(s)) + yield return "enable the ROOTLESS podman.socket (--user) — opt-in, for a metrics exporter; the ROOT socket stays masked"; } // Stable marker over every managed input. Quadlet CONTENT is included (not just names), @@ -99,6 +104,7 @@ public static string DesiredMarker(Shape s) start.ToString(), count.ToString(), AutoUpdate(s) ? "au=1" : "au=0", + UserSocket(s) ? "usock=1" : "usock=0", string.Join(",", SecretNames(s).Select(kv => $"{kv.Key}={kv.Value}")), }; foreach (var f in QuadletFiles(s)) @@ -491,6 +497,39 @@ internal static string BuildDeploy( sb.Append($"echo {b64} | base64 -d | {UserCmd(user, $"podman secret create {name} -")}\n"); } + // 7b. Weekly prune timer — BEFORE starting app units, deliberately. Platform + // housekeeping must not be hostage to an application unit: when podman-exporter + // failed to start on CT 4001/5114 (#283 phase 3), `set -e` aborted the script and + // the timer + socket steps never ran at all. + // Weekly prune timer — the other half of "podman-native replaces the Watchtower + + // prune sidecars" (ADR-0009). Written as USER units so it prunes the rootless + // store, not root's. + // + // Deliberately NOT `--volumes`: an unattached volume may still hold real data (a + // stopped service's database), and losing it to a housekeeping timer would be + // unrecoverable. Images/containers/networks only, and only things older than a week. + sb.Append($"install -d -o {user} -g {user} -m 755 /home/{user}/.config/systemd/user\n"); + sb.Append($"printf '%s\\n' '[Unit]' 'Description=Prune unused podman images, containers and networks' " + + "'Documentation=ADR-0009 phase 3' '' '[Service]' 'Type=oneshot' " + + "'ExecStart=/usr/bin/podman system prune -af --filter until=168h' " + + $"> /home/{user}/.config/systemd/user/podman-system-prune.service\n"); + sb.Append($"printf '%s\\n' '[Unit]' 'Description=Weekly podman prune' '' '[Timer]' " + + "'OnCalendar=weekly' 'Persistent=true' 'RandomizedDelaySec=1h' '' '[Install]' " + + $"'WantedBy=timers.target' > /home/{user}/.config/systemd/user/podman-system-prune.timer\n"); + sb.Append($"chown -R {user}:{user} /home/{user}/.config/systemd\n"); + sb.Append($"{UserCmd(user, "systemctl --user daemon-reload")}\n"); + sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-system-prune.timer")}\n"); + + // 7c. The ROOTLESS user API socket — opt-in via `config.userSocket: true`. + // + // ADR-0009 masks the ROOT podman.socket, and that stays masked. This is a + // categorically smaller thing: a socket owned by the unprivileged `podman` user, + // inside its own userns, reachable only by that user. It exists because a metrics + // exporter has no other way to enumerate containers (ADR-0009 phase 3, "observe"). + // Off by default, so a host that doesn't export metrics still has no API surface. + if (UserSocket(s)) + sb.Append($"{UserCmd(user, "systemctl --user enable --now podman.socket")}\n"); + // 8. Reload + start. daemon-reload runs the quadlet generator; each *.container // becomes .service. `restart` (not `start`) so a changed quadlet actually // takes effect on an already-running unit. @@ -527,6 +566,11 @@ internal static string UserCmd(string user, string cmd) => internal static string User(Shape s) => s.Spec.Config.Str("user") ?? DefaultUser; + // Opt-in rootless API socket. Only hosts that run a metrics exporter need it. + internal static bool UserSocket(Shape s) => + s.Spec.Config.TryGetValue("userSocket", out var v) && v is not null + && v.ToString() is not ("false" or "False" or "0"); + internal static bool AutoUpdate(Shape s) => s.Spec.Config.TryGetValue("autoUpdate", out var v) && v is not null ? v.ToString() is not ("false" or "False" or "0") diff --git a/stacks/Media/podman-host.lxc.yaml b/stacks/Media/podman-host.lxc.yaml index a9db6f5..745578b 100644 --- a/stacks/Media/podman-host.lxc.yaml +++ b/stacks/Media/podman-host.lxc.yaml @@ -62,6 +62,11 @@ spec: user: podman quadlets: podman-host/quadlets + # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009 + # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged + # `podman` user and reachable only by it. + userSocket: true + # podman secrets seeded from secrets.env, add-only. Namespaced per service, because this # host will grow more of them (the SM/template keys are YOUTARR_*, the podman secret names # are lower-case). diff --git a/stacks/Media/podman-host/quadlets/podman-exporter.container b/stacks/Media/podman-host/quadlets/podman-exporter.container new file mode 100644 index 0000000..b20b54b --- /dev/null +++ b/stacks/Media/podman-host/quadlets/podman-exporter.container @@ -0,0 +1,51 @@ +# prometheus-podman-exporter — container/pod/image metrics for this rootless host. +# +# ADR-0009 phase 3 ("observe"). The migration removed docker.sock and put nothing in its +# place, so until now the podman hosts had ZERO container-level visibility: systemd knows +# whether a unit is up, but nothing exported restart counts, image ages or per-container +# resource use. +# +# WHY THIS NEEDS A SOCKET, and why that is acceptable: an exporter has no other way to +# enumerate containers. ADR-0009 masks the ROOT podman.socket and that stays masked — this +# uses the ROOTLESS one, owned by the unprivileged 'podman' user, inside its own userns and +# reachable only by that user. Opt-in per host via config.userSocket; a host that does not +# export metrics keeps no API surface at all. +# +# %t expands to XDG_RUNTIME_DIR in a systemd USER unit, which is where the rootless socket +# lives (/run/user//podman/podman.sock). + +[Unit] +Description=Prometheus podman exporter +Documentation=https://github.com/containers/prometheus-podman-exporter +After=network-online.target podman.socket +Wants=network-online.target +Requires=podman.socket + +[Container] +Image=quay.io/navidys/prometheus-podman-exporter:latest +ContainerName=podman-exporter +Network=youtarr.network +PublishPort=9882:9882 +# Run as root INSIDE the container instead of the image's `nobody` (65534). Under the DEFAULT +# rootless mapping, container-root maps to the unprivileged `podman` user on the host — which +# is exactly who owns the socket below, so access works with no keep-id at all. +# +# Two failed attempts got here: the image's own 65534 is outside the subuid window an LXC's +# 65536-uid map allows, so crun refused it ("setgroups: Invalid argument"); adding +# keep-id:uid=65534 fixed that but left the process with no home, and it died on +# "stat /home/.config: no such file or directory" while looking for containers.conf. +# Running as root sidesteps both, and is no privilege gain — container-root here IS the +# unprivileged podman user. +User=0 +Group=0 +Volume=%t/podman/podman.sock:/run/podman/podman.sock:ro +Environment=CONTAINER_HOST=unix:///run/podman/podman.sock +Exec=--collector.enable-all +AutoUpdate=registry + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/stacks/monitoring/podman-host.lxc.yaml b/stacks/monitoring/podman-host.lxc.yaml index ac03d71..2c8778d 100644 --- a/stacks/monitoring/podman-host.lxc.yaml +++ b/stacks/monitoring/podman-host.lxc.yaml @@ -65,6 +65,11 @@ spec: user: podman quadlets: podman-host/quadlets + # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009 + # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged + # `podman` user and reachable only by it. + userSocket: true + # Config tree rendered onto the host (#305). Quadlets bind-mount from here read-only. assets: podman-host/assets assetsTarget: /home/podman/monitoring diff --git a/stacks/monitoring/podman-host/assets/config/prometheus.yml b/stacks/monitoring/podman-host/assets/config/prometheus.yml index 3fa13be..49dec2b 100644 --- a/stacks/monitoring/podman-host/assets/config/prometheus.yml +++ b/stacks/monitoring/podman-host/assets/config/prometheus.yml @@ -34,3 +34,20 @@ scrape_configs: - job_name: 'otel-collector' static_configs: - targets: ['otel-collector:8889'] + + # Rootless podman hosts (ADR-0009 phase 3). Container/pod/image metrics from + # prometheus-podman-exporter on each host. The Monitoring host is scraped by container + # name over the shared quadlet network; the Media host by its internal DNS name (reserved + registered in UniFi, so no IP is + # hard-coded here either). + # + # The SmartHome host (CT 6004) is deliberately ABSENT: it lives on the IoT VLAN 1040, + # which is firewalled off from 1010, so Prometheus cannot reach it. Verified — a TCP probe + # from CT 4001 to that host fails. Adding it needs a narrow UniFi allowance (1010 -> 1040 + # on :9882), which is a deliberate loosening of the IoT VLAN's isolation and therefore an + # owner decision, not an implementation detail. + - job_name: 'podman' + static_configs: + - targets: ['podman-exporter:9882'] + labels: { host: 'monitoring-podman-host', ctid: '4001' } + - targets: ['media.homelab.chrison.internal:9882'] + labels: { host: 'media-podman-host', ctid: '5114' } diff --git a/stacks/monitoring/podman-host/quadlets/podman-exporter.container b/stacks/monitoring/podman-host/quadlets/podman-exporter.container new file mode 100644 index 0000000..7f2bd58 --- /dev/null +++ b/stacks/monitoring/podman-host/quadlets/podman-exporter.container @@ -0,0 +1,51 @@ +# prometheus-podman-exporter — container/pod/image metrics for this rootless host. +# +# ADR-0009 phase 3 ("observe"). The migration removed docker.sock and put nothing in its +# place, so until now the podman hosts had ZERO container-level visibility: systemd knows +# whether a unit is up, but nothing exported restart counts, image ages or per-container +# resource use. +# +# WHY THIS NEEDS A SOCKET, and why that is acceptable: an exporter has no other way to +# enumerate containers. ADR-0009 masks the ROOT podman.socket and that stays masked — this +# uses the ROOTLESS one, owned by the unprivileged 'podman' user, inside its own userns and +# reachable only by that user. Opt-in per host via config.userSocket; a host that does not +# export metrics keeps no API surface at all. +# +# %t expands to XDG_RUNTIME_DIR in a systemd USER unit, which is where the rootless socket +# lives (/run/user//podman/podman.sock). + +[Unit] +Description=Prometheus podman exporter +Documentation=https://github.com/containers/prometheus-podman-exporter +After=network-online.target podman.socket +Wants=network-online.target +Requires=podman.socket + +[Container] +Image=quay.io/navidys/prometheus-podman-exporter:latest +ContainerName=podman-exporter +Network=monitoring.network +PublishPort=9882:9882 +# Run as root INSIDE the container instead of the image's `nobody` (65534). Under the DEFAULT +# rootless mapping, container-root maps to the unprivileged `podman` user on the host — which +# is exactly who owns the socket below, so access works with no keep-id at all. +# +# Two failed attempts got here: the image's own 65534 is outside the subuid window an LXC's +# 65536-uid map allows, so crun refused it ("setgroups: Invalid argument"); adding +# keep-id:uid=65534 fixed that but left the process with no home, and it died on +# "stat /home/.config: no such file or directory" while looking for containers.conf. +# Running as root sidesteps both, and is no privilege gain — container-root here IS the +# unprivileged podman user. +User=0 +Group=0 +Volume=%t/podman/podman.sock:/run/podman/podman.sock:ro +Environment=CONTAINER_HOST=unix:///run/podman/podman.sock +Exec=--collector.enable-all +AutoUpdate=registry + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target From a8bc2665866049979be39b4c9414803bf28164a8 Mon Sep 17 00:00:00 2001 From: Chrison Simtian Date: Tue, 28 Jul 2026 11:35:13 +1200 Subject: [PATCH 2/2] feat(observe): Cockpit on the podman hosts, Prometheus to :9091 (ADR-0009 phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes phase 3 of #283. Cockpit is the management UI ADR-0009 named, now on all three rootless hosts. Prometheus moves to host port 9091 so Cockpit can have its default 9090. Only the PUBLISHED port changes — the container still listens on 9090, so Grafana's datasource (http://prometheus:9090, over the quadlet network) needed no change at all. Verified after: 7/7 scrape targets UP and "Prometheus Server is Healthy" from inside the Grafana container. Cockpit deliberately authenticates as the `podman` USER, not root: rootless containers exist only inside that user's session, so a root Cockpit session shows an empty container list. Both `podman` and `root` ship password-LOCKED on a community-scripts CT, so the provisioner sets the user's password from PODMAN_USER_PASSWORD (generated, in Bitwarden SM). cockpit-podman then reads the same rootless socket the exporter uses — which is why cockpit requires userSocket. Requesting cockpit without the secret FAILS loudly rather than installing a login UI nobody can log into. Ordering bug found live on CT 4001: enabling cockpit.socket before the app units restart fails with `Result: resources`, because Prometheus still holds :9090 on its old published port until it restarts. The package + password now land before app units (so they exist even if a unit later fails) while the SOCKET is enabled last, after units have rebound. Regression-tested in both directions. DNS: all three podman hosts are now resolvable by name — monitoring/media/smarthome.homelab.chrison.internal. The SmartHome host had no name at all; added via UniFi's standalone static-DNS API rather than a DHCP reservation, since that host is only ever reached by name. 141 engine tests pass. Refs #283 --- .../engine.Tests/PodmanProvisionerTests.cs | 46 ++++++++++++++++ .../engine/Converge/PodmanProvisioner.cs | 55 ++++++++++++++++++- secrets.env.template | 7 +++ stacks/Media/podman-host.lxc.yaml | 5 ++ stacks/monitoring/podman-host.lxc.yaml | 5 ++ .../podman-host/quadlets/prometheus.container | 6 +- 6 files changed, 121 insertions(+), 3 deletions(-) diff --git a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs index ce7568b..4aeced6 100644 --- a/Infrastructure/engine.Tests/PodmanProvisionerTests.cs +++ b/Infrastructure/engine.Tests/PodmanProvisionerTests.cs @@ -327,6 +327,52 @@ public void Deploy_SetsUpPlatformHousekeeping_BeforeStartingAppUnits() Assert.True(sock < start, "the API socket must be enabled before a unit that Requires= it"); } + [Fact] + public void Deploy_InstallsCockpit_AndUnlocksTheRootlessUserSoItCanLogIn() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["cockpit"] = "true"; + var script = PodmanProvisioner.BuildDeploy( + shape, "podman", "m", "/home/podman/.homelab-managed", + PodmanProvisioner.QuadletFiles(shape), new Dictionary(), "s3kret"); + + Assert.Contains("cockpit cockpit-podman", script); + // The SOCKET must be enabled AFTER app units restart: binding :9090 while Prometheus + // still holds its old published port fails with `Result: resources` (hit on CT 4001). + var install = script.IndexOf("cockpit cockpit-podman", StringComparison.Ordinal); + var restart = script.IndexOf("systemctl --user restart", StringComparison.Ordinal); + var sock = script.IndexOf("enable --now cockpit.socket", StringComparison.Ordinal); + Assert.True(install < restart, "install the package before units, so it exists even if one fails"); + Assert.True(restart < sock, "enable cockpit.socket only after units have rebound their ports"); + // The point: rootless containers live only in the `podman` user's session, and both + // podman and root ship password-LOCKED on a community-scripts CT — so without this + // Cockpit installs but nobody can log in. + Assert.Contains("'podman:s3kret' | chpasswd", script); + } + + [Fact] + public void Deploy_OmitsCockpit_WhenNotRequested() + { + var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n"))); + Assert.DoesNotContain("cockpit", script); + Assert.DoesNotContain("chpasswd", script); + } + + [Fact] + public async Task Fails_WhenCockpitIsRequestedWithoutAPassword() + { + var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n")); + shape.Spec.Config["cockpit"] = "true"; + Environment.SetEnvironmentVariable("PODMAN_USER_PASSWORD", null); + + var exec = new FakeNodeExec(_ => new ExecResult(0, "", "")); + var result = await new PodmanProvisioner().ApplyAsync(shape, Ctx(exec)); + + // Installing a login UI nobody can log into is worse than not installing it. + Assert.Equal(ApplyOutcome.Failed, result.Outcome); + Assert.Contains("PODMAN_USER_PASSWORD", result.Message); + } + // ---- assets (#303) ---------------------------------------------------- // Adds an assets tree to a shape's stack dir and points config.assets at it. diff --git a/Infrastructure/engine/Converge/PodmanProvisioner.cs b/Infrastructure/engine/Converge/PodmanProvisioner.cs index cf76b1b..2a09c8c 100644 --- a/Infrastructure/engine/Converge/PodmanProvisioner.cs +++ b/Infrastructure/engine/Converge/PodmanProvisioner.cs @@ -91,6 +91,9 @@ public IEnumerable PlanSteps(Shape s) if (UserSocket(s)) yield return "enable the ROOTLESS podman.socket (--user) — opt-in, for a metrics exporter; the ROOT socket stays masked"; + + if (Cockpit(s)) + yield return "install cockpit + cockpit-podman on :9090, set the `podman` user's password (PODMAN_USER_PASSWORD) so it can log in"; } // Stable marker over every managed input. Quadlet CONTENT is included (not just names), @@ -105,6 +108,7 @@ public static string DesiredMarker(Shape s) count.ToString(), AutoUpdate(s) ? "au=1" : "au=0", UserSocket(s) ? "usock=1" : "usock=0", + Cockpit(s) ? "cockpit=1" : "cockpit=0", string.Join(",", SecretNames(s).Select(kv => $"{kv.Key}={kv.Value}")), }; foreach (var f in QuadletFiles(s)) @@ -165,8 +169,20 @@ public async Task ApplyAsync(Shape s, ConvergeContext ctx) var (assetMsg, assetFailed) = await PushAssetsAsync(s, ctx, node, ctid, user); if (assetFailed is not null) return ApplyResult.Failed(assetFailed); + // Cockpit needs a PAM password for the rootless user; without it the UI installs but + // nobody can log in, which is worse than not installing it. + string? cockpitPassword = null; + if (Cockpit(s)) + { + if (ctx.Secrets.Get("PODMAN_USER_PASSWORD") is not { Length: > 0 } pw) + return ApplyResult.Failed( + "cockpit: true needs PODMAN_USER_PASSWORD in secrets.env — Cockpit authenticates the " + + "rootless user via PAM, and it ships password-locked"); + cockpitPassword = pw; + } + var files = QuadletFiles(s); - var script = BuildDeploy(s, user, marker, markerPath, files, secretValues); + var script = BuildDeploy(s, user, marker, markerPath, files, secretValues, cockpitPassword); var res = await ctx.Exec.InContainerAsync(node, ctid, script); if (!res.Ok) return ApplyResult.Failed($"podman host setup failed: {res.Stderr}"); @@ -354,7 +370,8 @@ internal static Dictionary ParseFeatures(string pctConfig) // user is usable, linger before any `systemctl --user`, and the marker is stamped last. internal static string BuildDeploy( Shape s, string user, string marker, string markerPath, - IReadOnlyList files, IReadOnlyDictionary secrets) + IReadOnlyList files, IReadOnlyDictionary secrets, + string? cockpitPassword = null) { var (start, count) = SubidRange(s); var sb = new StringBuilder(); @@ -530,6 +547,29 @@ internal static string BuildDeploy( if (UserSocket(s)) sb.Append($"{UserCmd(user, "systemctl --user enable --now podman.socket")}\n"); + // 7d. Cockpit — the management UI half of ADR-0009 phase 3. + // + // Deliberately logs in as the `podman` USER, not root: rootless containers exist + // only inside that user's session, so a root Cockpit session shows an empty + // container list. That means the user needs a real PAM password — both `podman` + // and `root` ship password-LOCKED on a community-scripts CT, so Cockpit login is + // impossible until one is set. cockpit-podman then reads the same rootless socket + // the exporter uses, which is why this requires userSocket. + // + // Cockpit takes :9090, its default — which is why Prometheus publishes on 9091. + // The package + password land HERE (before app units, so they exist even if a unit + // later fails), but the SOCKET is enabled further down, after units have restarted. + // Enabling it here fails with `Result: resources`: the app unit still holds :9090 + // on its old published port until it restarts. Hit on CT 4001. + if (Cockpit(s)) + { + sb.Append("export DEBIAN_FRONTEND=noninteractive\n"); + sb.Append("if ! dpkg -s cockpit >/dev/null 2>&1; then apt-get update -qq && " + + "apt-get install -y -qq cockpit cockpit-podman; fi\n"); + if (cockpitPassword is { Length: > 0 }) + sb.Append($"printf '%s' '{user}:{cockpitPassword}' | chpasswd\n"); + } + // 8. Reload + start. daemon-reload runs the quadlet generator; each *.container // becomes .service. `restart` (not `start`) so a changed quadlet actually // takes effect on an already-running unit. @@ -544,6 +584,11 @@ internal static string BuildDeploy( if (AutoUpdate(s)) sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-auto-update.timer")}\n"); + // 9c. Cockpit's socket, LAST — after app units have rebound to their new ports. + // See 7d: binding :9090 before Prometheus moves to 9091 fails outright. + if (Cockpit(s)) + sb.Append("systemctl enable --now cockpit.socket\n"); + // 10. Mark-on-SUCCESS — only reached if every step above exited 0 under `set -e`. sb.Append($"printf '%s' '{marker}' > {markerPath}\n"); sb.Append($"chown {user}:{user} {markerPath}"); @@ -566,6 +611,12 @@ internal static string UserCmd(string user, string cmd) => internal static string User(Shape s) => s.Spec.Config.Str("user") ?? DefaultUser; + // Opt-in Cockpit management UI (ADR-0009 phase 3). Requires userSocket, since + // cockpit-podman reads the same rootless API socket to list containers. + internal static bool Cockpit(Shape s) => + s.Spec.Config.TryGetValue("cockpit", out var v) && v is not null + && v.ToString() is not ("false" or "False" or "0"); + // Opt-in rootless API socket. Only hosts that run a metrics exporter need it. internal static bool UserSocket(Shape s) => s.Spec.Config.TryGetValue("userSocket", out var v) && v is not null diff --git a/secrets.env.template b/secrets.env.template index 2da5c16..d343c31 100644 --- a/secrets.env.template +++ b/secrets.env.template @@ -106,3 +106,10 @@ GF_SECURITY_ADMIN_PASSWORD= RADARR_API_KEY= SONARR_API_KEY= PROWLARR_API_KEY= + +# ── Rootless podman hosts: Cockpit login (#283 phase 3) ── +# Password for the `podman` user on every podman host. Exists ONLY so Cockpit can authenticate +# that user via PAM — rootless containers are visible only inside its own session, so root's +# Cockpit session shows nothing. Not consumed by any service; set by the provisioner when a +# shape declares `cockpit: true`. +PODMAN_USER_PASSWORD= diff --git a/stacks/Media/podman-host.lxc.yaml b/stacks/Media/podman-host.lxc.yaml index 745578b..34a2601 100644 --- a/stacks/Media/podman-host.lxc.yaml +++ b/stacks/Media/podman-host.lxc.yaml @@ -67,6 +67,11 @@ spec: # `podman` user and reachable only by it. userSocket: true + # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user + # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and + # userSocket above. + cockpit: true + # podman secrets seeded from secrets.env, add-only. Namespaced per service, because this # host will grow more of them (the SM/template keys are YOUTARR_*, the podman secret names # are lower-case). diff --git a/stacks/monitoring/podman-host.lxc.yaml b/stacks/monitoring/podman-host.lxc.yaml index 2c8778d..5809be4 100644 --- a/stacks/monitoring/podman-host.lxc.yaml +++ b/stacks/monitoring/podman-host.lxc.yaml @@ -70,6 +70,11 @@ spec: # `podman` user and reachable only by it. userSocket: true + # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user + # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and + # userSocket above. + cockpit: true + # Config tree rendered onto the host (#305). Quadlets bind-mount from here read-only. assets: podman-host/assets assetsTarget: /home/podman/monitoring diff --git a/stacks/monitoring/podman-host/quadlets/prometheus.container b/stacks/monitoring/podman-host/quadlets/prometheus.container index 2088abe..5be26dd 100644 --- a/stacks/monitoring/podman-host/quadlets/prometheus.container +++ b/stacks/monitoring/podman-host/quadlets/prometheus.container @@ -13,7 +13,11 @@ Wants=network-online.target Image=docker.io/prom/prometheus ContainerName=prometheus Network=monitoring.network -PublishPort=9090:9090 +# Published on 9091, NOT 9090 — Cockpit owns 9090 on the podman hosts as the management UI +# (ADR-0009 phase 3). Only the HOST-side port moves: the container still listens on 9090, so +# Grafana's datasource (http://prometheus:9090, over the quadlet network) is unaffected and +# needs no change. Browse Prometheus at monitoring.homelab.chrison.internal:9091. +PublishPort=9091:9090 # Runs as nobody (uid 65534) — verified on the live container, not assumed. That is outside the # subuid window the LXC's 65536-uid map allows, so without keep-id crun refuses to start it. # There is no data volume here (Prometheus's TSDB is ephemeral in this stack, as it was under