feat(observe): podman metrics + prune timer (ADR-0009 phase 3)

Infrastructure/engine.Tests/PodmanProvisionerTests.cs

@@ -266,6 +266,113 @@ public void Deploy_DoesNotMaskThePodmanNetworkHelper()
         Assert.DoesNotContain("mask podman-user-wait-network-online", script);
     }
 
+    // ---- observe: prune timer + opt-in socket (#283 phase 3) --------------
+
+    [Fact]
+    public void Deploy_InstallsAWeeklyPruneTimer_ThatNeverTouchesVolumes()
+    {
+        var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n")));
+
+        Assert.Contains("podman-system-prune.timer", script);
+        Assert.Contains("OnCalendar=weekly", script);
+        Assert.Contains("podman system prune -af --filter until=168h", script);
+        // NOT --volumes: an unattached volume may hold a stopped service's database, and
+        // losing it to a housekeeping timer would be unrecoverable.
+        Assert.DoesNotContain("--volumes", script);
+        // User units, so it prunes the rootless store rather than root's.
+        Assert.Contains("/home/podman/.config/systemd/user/podman-system-prune.timer", script);
+    }
+
+    [Fact]
+    public void Deploy_LeavesTheApiSocketOff_UnlessExplicitlyRequested()
+    {
+        var off = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n")));
+        // The ROOT socket is masked either way; the point here is that no USER socket appears
+        // unless a host actually needs one, so the default posture has no API surface at all.
+        Assert.DoesNotContain("enable --now podman.socket", off);
+        Assert.Contains("mask podman.socket", off);
+
+        var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n"));
+        shape.Spec.Config["userSocket"] = "true";
+        var on = Build(shape);
+        Assert.Contains("systemctl --user enable --now podman.socket", on);
+        Assert.Contains("mask podman.socket", on);   // root socket STILL masked
+    }
+
+    [Fact]
+    public void Marker_ChangesWhenTheUserSocketIsToggled()
+    {
+        var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n"));
+        var before = PodmanProvisioner.DesiredMarker(shape);
+        shape.Spec.Config["userSocket"] = "true";
+        Assert.NotEqual(before, PodmanProvisioner.DesiredMarker(shape));
+    }
+
+    [Fact]
+    public void Deploy_SetsUpPlatformHousekeeping_BeforeStartingAppUnits()
+    {
+        var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n"));
+        shape.Spec.Config["userSocket"] = "true";
+        var script = Build(shape);
+
+        // Regression guard: when podman-exporter failed to start on CT 4001/5114, `set -e`
+        // aborted the script and the prune timer + socket steps never ran at all. Platform
+        // housekeeping must not be hostage to an application unit — and a unit that
+        // Requires=podman.socket needs the socket enabled before it starts, too.
+        var timer = script.IndexOf("podman-system-prune.timer", StringComparison.Ordinal);
+        var sock  = script.IndexOf("enable --now podman.socket", StringComparison.Ordinal);
+        var start = script.IndexOf("systemctl --user restart", StringComparison.Ordinal);
+        Assert.True(timer >= 0 && sock >= 0 && start >= 0);
+        Assert.True(timer < start, "prune timer must be installed before app units start");
+        Assert.True(sock  < start, "the API socket must be enabled before a unit that Requires= it");
+    }
+
+    [Fact]
+    public void Deploy_InstallsCockpit_AndUnlocksTheRootlessUserSoItCanLogIn()
+    {
+        var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n"));
+        shape.Spec.Config["cockpit"] = "true";
+        var script = PodmanProvisioner.BuildDeploy(
+            shape, "podman", "m", "/home/podman/.homelab-managed",
+            PodmanProvisioner.QuadletFiles(shape), new Dictionary<string, string>(), "s3kret");
+
+        Assert.Contains("cockpit cockpit-podman", script);
+        // The SOCKET must be enabled AFTER app units restart: binding :9090 while Prometheus
+        // still holds its old published port fails with `Result: resources` (hit on CT 4001).
+        var install = script.IndexOf("cockpit cockpit-podman", StringComparison.Ordinal);
+        var restart = script.IndexOf("systemctl --user restart", StringComparison.Ordinal);
+        var sock    = script.IndexOf("enable --now cockpit.socket", StringComparison.Ordinal);
+        Assert.True(install < restart, "install the package before units, so it exists even if one fails");
+        Assert.True(restart < sock, "enable cockpit.socket only after units have rebound their ports");
+        // The point: rootless containers live only in the `podman` user's session, and both
+        // podman and root ship password-LOCKED on a community-scripts CT — so without this
+        // Cockpit installs but nobody can log in.
+        Assert.Contains("'podman:s3kret' | chpasswd", script);
+    }
+
+    [Fact]
+    public void Deploy_OmitsCockpit_WhenNotRequested()
+    {
+        var script = Build(PodmanShape(("mate.container", "[Container]\nImage=x:1\n")));
+        Assert.DoesNotContain("cockpit", script);
+        Assert.DoesNotContain("chpasswd", script);
+    }
+
+    [Fact]
+    public async Task Fails_WhenCockpitIsRequestedWithoutAPassword()
+    {
+        var shape = PodmanShape(("mate.container", "[Container]\nImage=x:1\n"));
+        shape.Spec.Config["cockpit"] = "true";
+        Environment.SetEnvironmentVariable("PODMAN_USER_PASSWORD", null);
+
+        var exec = new FakeNodeExec(_ => new ExecResult(0, "", ""));
+        var result = await new PodmanProvisioner().ApplyAsync(shape, Ctx(exec));
+
+        // Installing a login UI nobody can log into is worse than not installing it.
+        Assert.Equal(ApplyOutcome.Failed, result.Outcome);
+        Assert.Contains("PODMAN_USER_PASSWORD", result.Message);
+    }
+
     // ---- assets (#303) ----------------------------------------------------
 
     // Adds an assets tree to a shape's stack dir and points config.assets at it.

Infrastructure/engine/Converge/PodmanProvisioner.cs

@@ -86,6 +86,14 @@ public IEnumerable<string> PlanSteps(Shape s)
 
         if (AutoUpdate(s))
             yield return "enable podman-auto-update.timer (--user) — replaces Watchtower, no docker.sock";
+
+        yield return "install + enable podman-system-prune.timer (weekly, images/containers/networks — never volumes)";
+
+        if (UserSocket(s))
+            yield return "enable the ROOTLESS podman.socket (--user) — opt-in, for a metrics exporter; the ROOT socket stays masked";
+
+        if (Cockpit(s))
+            yield return "install cockpit + cockpit-podman on :9090, set the `podman` user's password (PODMAN_USER_PASSWORD) so it can log in";
     }
 
     // Stable marker over every managed input. Quadlet CONTENT is included (not just names),
@@ -99,6 +107,8 @@ public static string DesiredMarker(Shape s)
             start.ToString(),
             count.ToString(),
             AutoUpdate(s) ? "au=1" : "au=0",
+            UserSocket(s) ? "usock=1" : "usock=0",
+            Cockpit(s) ? "cockpit=1" : "cockpit=0",
             string.Join(",", SecretNames(s).Select(kv => $"{kv.Key}={kv.Value}")),
         };
         foreach (var f in QuadletFiles(s))
@@ -159,8 +169,20 @@ public async Task<ApplyResult> ApplyAsync(Shape s, ConvergeContext ctx)
         var (assetMsg, assetFailed) = await PushAssetsAsync(s, ctx, node, ctid, user);
         if (assetFailed is not null) return ApplyResult.Failed(assetFailed);
 
+        // Cockpit needs a PAM password for the rootless user; without it the UI installs but
+        // nobody can log in, which is worse than not installing it.
+        string? cockpitPassword = null;
+        if (Cockpit(s))
+        {
+            if (ctx.Secrets.Get("PODMAN_USER_PASSWORD") is not { Length: > 0 } pw)
+                return ApplyResult.Failed(
+                    "cockpit: true needs PODMAN_USER_PASSWORD in secrets.env — Cockpit authenticates the " +
+                    "rootless user via PAM, and it ships password-locked");
+            cockpitPassword = pw;
+        }
+
         var files = QuadletFiles(s);
-        var script = BuildDeploy(s, user, marker, markerPath, files, secretValues);
+        var script = BuildDeploy(s, user, marker, markerPath, files, secretValues, cockpitPassword);
 
         var res = await ctx.Exec.InContainerAsync(node, ctid, script);
         if (!res.Ok) return ApplyResult.Failed($"podman host setup failed: {res.Stderr}");
@@ -348,7 +370,8 @@ internal static Dictionary<string, string> ParseFeatures(string pctConfig)
     // user is usable, linger before any `systemctl --user`, and the marker is stamped last.
     internal static string BuildDeploy(
         Shape s, string user, string marker, string markerPath,
-        IReadOnlyList<string> files, IReadOnlyDictionary<string, string> secrets)
+        IReadOnlyList<string> files, IReadOnlyDictionary<string, string> secrets,
+        string? cockpitPassword = null)
     {
         var (start, count) = SubidRange(s);
         var sb = new StringBuilder();
@@ -491,6 +514,62 @@ internal static string BuildDeploy(
             sb.Append($"echo {b64} | base64 -d | {UserCmd(user, $"podman secret create {name} -")}\n");
         }
 
+        // 7b. Weekly prune timer — BEFORE starting app units, deliberately. Platform
+        //     housekeeping must not be hostage to an application unit: when podman-exporter
+        //     failed to start on CT 4001/5114 (#283 phase 3), `set -e` aborted the script and
+        //     the timer + socket steps never ran at all.
+        //     Weekly prune timer — the other half of "podman-native replaces the Watchtower +
+        //     prune sidecars" (ADR-0009). Written as USER units so it prunes the rootless
+        //     store, not root's.
+        //
+        //     Deliberately NOT `--volumes`: an unattached volume may still hold real data (a
+        //     stopped service's database), and losing it to a housekeeping timer would be
+        //     unrecoverable. Images/containers/networks only, and only things older than a week.
+        sb.Append($"install -d -o {user} -g {user} -m 755 /home/{user}/.config/systemd/user\n");
+        sb.Append($"printf '%s\\n' '[Unit]' 'Description=Prune unused podman images, containers and networks' " +
+                  "'Documentation=ADR-0009 phase 3' '' '[Service]' 'Type=oneshot' " +
+                  "'ExecStart=/usr/bin/podman system prune -af --filter until=168h' " +
+                  $"> /home/{user}/.config/systemd/user/podman-system-prune.service\n");
+        sb.Append($"printf '%s\\n' '[Unit]' 'Description=Weekly podman prune' '' '[Timer]' " +
+                  "'OnCalendar=weekly' 'Persistent=true' 'RandomizedDelaySec=1h' '' '[Install]' " +
+                  $"'WantedBy=timers.target' > /home/{user}/.config/systemd/user/podman-system-prune.timer\n");
+        sb.Append($"chown -R {user}:{user} /home/{user}/.config/systemd\n");
+        sb.Append($"{UserCmd(user, "systemctl --user daemon-reload")}\n");
+        sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-system-prune.timer")}\n");
+
+        // 7c. The ROOTLESS user API socket — opt-in via `config.userSocket: true`.
+        //
+        //     ADR-0009 masks the ROOT podman.socket, and that stays masked. This is a
+        //     categorically smaller thing: a socket owned by the unprivileged `podman` user,
+        //     inside its own userns, reachable only by that user. It exists because a metrics
+        //     exporter has no other way to enumerate containers (ADR-0009 phase 3, "observe").
+        //     Off by default, so a host that doesn't export metrics still has no API surface.
+        if (UserSocket(s))
+            sb.Append($"{UserCmd(user, "systemctl --user enable --now podman.socket")}\n");
+
+        // 7d. Cockpit — the management UI half of ADR-0009 phase 3.
+        //
+        //     Deliberately logs in as the `podman` USER, not root: rootless containers exist
+        //     only inside that user's session, so a root Cockpit session shows an empty
+        //     container list. That means the user needs a real PAM password — both `podman`
+        //     and `root` ship password-LOCKED on a community-scripts CT, so Cockpit login is
+        //     impossible until one is set. cockpit-podman then reads the same rootless socket
+        //     the exporter uses, which is why this requires userSocket.
+        //
+        //     Cockpit takes :9090, its default — which is why Prometheus publishes on 9091.
+        //     The package + password land HERE (before app units, so they exist even if a unit
+        //     later fails), but the SOCKET is enabled further down, after units have restarted.
+        //     Enabling it here fails with `Result: resources`: the app unit still holds :9090
+        //     on its old published port until it restarts. Hit on CT 4001.
+        if (Cockpit(s))
+        {
+            sb.Append("export DEBIAN_FRONTEND=noninteractive\n");
+            sb.Append("if ! dpkg -s cockpit >/dev/null 2>&1; then apt-get update -qq && " +
+                      "apt-get install -y -qq cockpit cockpit-podman; fi\n");
+            if (cockpitPassword is { Length: > 0 })
+                sb.Append($"printf '%s' '{user}:{cockpitPassword}' | chpasswd\n");
+        }
+
         // 8. Reload + start. daemon-reload runs the quadlet generator; each *.container
         //    becomes <name>.service. `restart` (not `start`) so a changed quadlet actually
         //    takes effect on an already-running unit.
@@ -505,6 +584,11 @@ internal static string BuildDeploy(
         if (AutoUpdate(s))
             sb.Append($"{UserCmd(user, "systemctl --user enable --now podman-auto-update.timer")}\n");
 
+        // 9c. Cockpit's socket, LAST — after app units have rebound to their new ports.
+        //     See 7d: binding :9090 before Prometheus moves to 9091 fails outright.
+        if (Cockpit(s))
+            sb.Append("systemctl enable --now cockpit.socket\n");
+
         // 10. Mark-on-SUCCESS — only reached if every step above exited 0 under `set -e`.
         sb.Append($"printf '%s' '{marker}' > {markerPath}\n");
         sb.Append($"chown {user}:{user} {markerPath}");
@@ -527,6 +611,17 @@ internal static string UserCmd(string user, string cmd) =>
 
     internal static string User(Shape s) => s.Spec.Config.Str("user") ?? DefaultUser;
 
+    // Opt-in Cockpit management UI (ADR-0009 phase 3). Requires userSocket, since
+    // cockpit-podman reads the same rootless API socket to list containers.
+    internal static bool Cockpit(Shape s) =>
+        s.Spec.Config.TryGetValue("cockpit", out var v) && v is not null
+        && v.ToString() is not ("false" or "False" or "0");
+
+    // Opt-in rootless API socket. Only hosts that run a metrics exporter need it.
+    internal static bool UserSocket(Shape s) =>
+        s.Spec.Config.TryGetValue("userSocket", out var v) && v is not null
+        && v.ToString() is not ("false" or "False" or "0");
+
     internal static bool AutoUpdate(Shape s) =>
         s.Spec.Config.TryGetValue("autoUpdate", out var v) && v is not null
             ? v.ToString() is not ("false" or "False" or "0")

secrets.env.template

@@ -106,3 +106,10 @@ GF_SECURITY_ADMIN_PASSWORD=
 RADARR_API_KEY=
 SONARR_API_KEY=
 PROWLARR_API_KEY=
+
+# ── Rootless podman hosts: Cockpit login (#283 phase 3) ──
+# Password for the `podman` user on every podman host. Exists ONLY so Cockpit can authenticate
+# that user via PAM — rootless containers are visible only inside its own session, so root's
+# Cockpit session shows nothing. Not consumed by any service; set by the provisioner when a
+# shape declares `cockpit: true`.
+PODMAN_USER_PASSWORD=

stacks/Media/podman-host.lxc.yaml

@@ -62,6 +62,16 @@ spec:
     user: podman
     quadlets: podman-host/quadlets
 
+    # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009
+    # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged
+    # `podman` user and reachable only by it.
+    userSocket: true
+
+    # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user
+    # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and
+    # userSocket above.
+    cockpit: true
+
     # podman secrets seeded from secrets.env, add-only. Namespaced per service, because this
     # host will grow more of them (the SM/template keys are YOUTARR_*, the podman secret names
     # are lower-case).

stacks/Media/podman-host/quadlets/podman-exporter.container

@@ -0,0 +1,51 @@
+# prometheus-podman-exporter — container/pod/image metrics for this rootless host.
+#
+# ADR-0009 phase 3 ("observe"). The migration removed docker.sock and put nothing in its
+# place, so until now the podman hosts had ZERO container-level visibility: systemd knows
+# whether a unit is up, but nothing exported restart counts, image ages or per-container
+# resource use.
+#
+# WHY THIS NEEDS A SOCKET, and why that is acceptable: an exporter has no other way to
+# enumerate containers. ADR-0009 masks the ROOT podman.socket and that stays masked — this
+# uses the ROOTLESS one, owned by the unprivileged 'podman' user, inside its own userns and
+# reachable only by that user. Opt-in per host via config.userSocket; a host that does not
+# export metrics keeps no API surface at all.
+#
+# %t expands to XDG_RUNTIME_DIR in a systemd USER unit, which is where the rootless socket
+# lives (/run/user/<uid>/podman/podman.sock).
+
+[Unit]
+Description=Prometheus podman exporter
+Documentation=https://github.com/containers/prometheus-podman-exporter
+After=network-online.target podman.socket
+Wants=network-online.target
+Requires=podman.socket
+
+[Container]
+Image=quay.io/navidys/prometheus-podman-exporter:latest
+ContainerName=podman-exporter
+Network=youtarr.network
+PublishPort=9882:9882
+# Run as root INSIDE the container instead of the image's `nobody` (65534). Under the DEFAULT
+# rootless mapping, container-root maps to the unprivileged `podman` user on the host — which
+# is exactly who owns the socket below, so access works with no keep-id at all.
+#
+# Two failed attempts got here: the image's own 65534 is outside the subuid window an LXC's
+# 65536-uid map allows, so crun refused it ("setgroups: Invalid argument"); adding
+# keep-id:uid=65534 fixed that but left the process with no home, and it died on
+# "stat /home/.config: no such file or directory" while looking for containers.conf.
+# Running as root sidesteps both, and is no privilege gain — container-root here IS the
+# unprivileged podman user.
+User=0
+Group=0
+Volume=%t/podman/podman.sock:/run/podman/podman.sock:ro
+Environment=CONTAINER_HOST=unix:///run/podman/podman.sock
+Exec=--collector.enable-all
+AutoUpdate=registry
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

stacks/monitoring/podman-host.lxc.yaml

@@ -65,6 +65,16 @@ spec:
     user: podman
     quadlets: podman-host/quadlets
 
+    # Opt in to the ROOTLESS podman API socket — required by podman-exporter (ADR-0009
+    # phase 3, #283). The ROOT socket stays masked; this one is owned by the unprivileged
+    # `podman` user and reachable only by it.
+    userSocket: true
+
+    # Cockpit management UI on :9090 (ADR-0009 phase 3, #283). Logs in as the `podman` user
+    # so cockpit-podman can see the rootless containers — needs PODMAN_USER_PASSWORD and
+    # userSocket above.
+    cockpit: true
+
     # Config tree rendered onto the host (#305). Quadlets bind-mount from here read-only.
     assets: podman-host/assets
     assetsTarget: /home/podman/monitoring