From 4514dd4bab2ca318d14528a77c2ebc7fd181c449 Mon Sep 17 00:00:00 2001 From: politerealism Date: Sat, 3 Oct 2026 07:26:59 -0400 Subject: [PATCH 1/3] ci(podman): report e2e-podman archive selected-vs-eligible coverage The e2e-podman nextest archive is a curated subset of everything eligible under the e2e-podman feature: podmanE2eFollowUpBinaries in tests/artifacts.nix excludes targets that still depend on wrapper-owned gateway controls, host fixtures, or other unmigrated test infrastructure. CI reported a per-binary pass/fail count but never the selected-vs-eligible split, so a narrow run could be mistaken for full Podman coverage. Add podman-e2e-coverage-summary, a Nix app that computes the eligible count via a real cargo nextest list (the same mechanism generate-podman-e2e-ci-tests already uses) and compares it against podmanE2eFollowUpBinaries -- the same declarative exclusion list that drives the archive filter, so this can't drift from what actually runs. Wire it into integration-runner.yml as a job-summary step for every e2e-podman run, independent of pass/fail. Addresses the selected-vs-eligible reporting acceptance criterion in #3712. Signed-off-by: politerealism --- .github/workflows/integration-runner.yml | 9 ++++++ TESTING.md | 8 +++++ flake.nix | 4 +++ tests/artifacts.nix | 39 ++++++++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 89cc0ca920..b51bd9f1e1 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -89,3 +89,12 @@ jobs: # Retry dependency preparation, then execute the test suite once. nix build --no-link .#tmachine || nix build --no-link .#tmachine nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" + + # The e2e-podman archive is a curated subset of everything eligible + # under the e2e-podman feature (see podmanE2eFollowUpBinaries in + # tests/artifacts.nix) -- report the selected-vs-eligible split here so + # this narrow run can't be mistaken for full Podman coverage (#3712). + # Runs regardless of pass/fail so the scope is visible either way. + - name: Report Podman e2e-podman archive coverage + if: always() && matrix.testsuite == 'e2e-podman' + run: nix run .#podman-e2e-coverage-summary >> "$GITHUB_STEP_SUMMARY" diff --git a/TESTING.md b/TESTING.md index a1c2f9476e..77ac2e51e7 100644 --- a/TESTING.md +++ b/TESTING.md @@ -300,6 +300,14 @@ Print the exact tmachine archive selection as a shell `PODMAN_CI_TESTS` array: nix run .#generate-podman-e2e-ci-tests ``` +Print how many of the `e2e-podman`-eligible targets the archive actually +selects, so a narrow run cannot be mistaken for full Podman coverage — CI +reports this in the job summary for every `e2e-podman` run: + +```shell +nix run .#podman-e2e-coverage-summary +``` + The `e2e-podman` testsuite runs a nextest archive built with the corresponding Rust feature and preloads its Python workload image into the rootless Podman store. The separate `driver-podman` testsuite compares OpenShell and direct diff --git a/flake.nix b/flake.nix index 4928d2f151..0490b991ac 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,10 @@ type = "app"; program = "${artifacts.podmanE2eCiTests}/bin/generate-podman-e2e-ci-tests"; }; + podman-e2e-coverage-summary = { + type = "app"; + program = "${artifacts.podmanE2eCoverageSummary}/bin/podman-e2e-coverage-summary"; + }; build-artifacts-helm = { type = "app"; program = "${artifacts.helm}/bin/build-artifacts-helm"; diff --git a/tests/artifacts.nix b/tests/artifacts.nix index c86d8f6d30..f2d432c690 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -175,6 +175,44 @@ let ''; }; + # Reports how many of the e2e-podman-eligible targets the archive actually + # selects, so a narrow CI run cannot be mistaken for full coverage (#3712). + # Computes the eligible count the same way podmanE2eCiTests does (a real + # `cargo nextest list`, not a hand-maintained number) and compares it + # against podmanE2eFollowUpBinaries, the single declarative exclusion list + # that also drives the archive filter above — so this can't drift from what + # actually runs. + podmanE2eCoverageSummary = pkgs.writeShellApplication { + name = "podman-e2e-coverage-summary"; + runtimeInputs = [ + pkgs.cargo-nextest + pkgs.git + pkgs.jq + rustToolchain + ]; + runtimeEnv = toolchainEnv; + text = '' + root=$(git rev-parse --show-toplevel) + cd "$root" + + eligible=$(cargo nextest list \ + --manifest-path e2e/rust/Cargo.toml \ + --target ${muslToolchain.target} \ + -p openshell-e2e \ + --features e2e-podman \ + --list-type binaries-only \ + --message-format json \ + | jq '[.["rust-binaries"] | to_entries[] | select(.value.kind == "test")] | length') + + excluded=${toString (builtins.length podmanE2eFollowUpBinaries)} + selected=$((eligible - excluded)) + + echo "### Podman \`e2e-podman\` archive coverage" + echo + echo "**$selected of $eligible** eligible targets selected ($excluded excluded — see \`podmanE2eFollowUpBinaries\` in \`tests/artifacts.nix\` for the documented reason behind each one)." + ''; + }; + podmanDriverArchive = mkTestArchive { name = "podman-driver"; workspacePath = "tests/suites/drivers"; @@ -201,6 +239,7 @@ rec { podmanDriverArchive podmanE2eArchive podmanE2eCiTests + podmanE2eCoverageSummary ; binaries = pkgs.writeShellApplication { From 7b272b5965a96c37445d7579c445e139cc108d1a Mon Sep 17 00:00:00 2001 From: politerealism Date: Sat, 3 Oct 2026 12:14:03 -0400 Subject: [PATCH 2/3] fix(podman): distinguish selected from executing in coverage summary Address review feedback on the previous commit: - The coverage summary's "selected" count included targets whose only test is a manual-only #[ignore]'d benchmark, so it overstated what actually executes by the count of those benchmarks. Switch from --list-type binaries-only to the full nextest listing so per-test ignored status is visible, and report eligible, excluded, selected, and actually-executing as distinct numbers, naming any selected manual-only benchmarks explicitly instead of folding them into "selected" silently. - Document the new job-summary step in CI.md (current implemented CI behavior) alongside the existing TESTING.md entry (local entry point), matching this repo's documented split between the two files. Signed-off-by: politerealism --- CI.md | 6 +++++ tests/artifacts.nix | 57 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/CI.md b/CI.md index cb7121b2cf..71c70fcd82 100644 --- a/CI.md +++ b/CI.md @@ -82,6 +82,12 @@ retain the binary installer because their fixtures configure its system service, local HTTP gateway, and CLI path. The manual Integration Tests workflow defaults to the package installers and downloads the packages selected by its matrix. +The `e2e-podman` driver-specific run posts a job summary reporting how many of +the `e2e-podman`-eligible targets the archive actually selects and executes, +naming any selected manual-only benchmarks separately, so a narrow run cannot +be mistaken for full Podman coverage. See `nix run .#podman-e2e-coverage-summary` +in TESTING.md. + Three opt-in labels enable the long-running E2E suites: - `test:e2e` runs the Docker, rootless Podman, Kubernetes, and VM E2E suites diff --git a/tests/artifacts.nix b/tests/artifacts.nix index f2d432c690..b76d314a90 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -176,12 +176,16 @@ let }; # Reports how many of the e2e-podman-eligible targets the archive actually - # selects, so a narrow CI run cannot be mistaken for full coverage (#3712). - # Computes the eligible count the same way podmanE2eCiTests does (a real - # `cargo nextest list`, not a hand-maintained number) and compares it - # against podmanE2eFollowUpBinaries, the single declarative exclusion list - # that also drives the archive filter above — so this can't drift from what - # actually runs. + # selects and executes, so a narrow CI run cannot be mistaken for full + # coverage (#3712). Computes the eligible set the same way podmanE2eCiTests + # does (a real `cargo nextest list`, not a hand-maintained number) and + # compares it against podmanE2eFollowUpBinaries, the single declarative + # exclusion list that also drives the archive filter above — so this can't + # drift from what actually runs. Uses the full (not binaries-only) listing + # so per-test `#[ignore]` status is visible: a selected binary whose only + # test is a manual-only benchmark (see the two perf targets elsewhere in + # this file) is archive-scoped but never actually executes, and reporting + # it as "selected" without that distinction would itself be misleading. podmanE2eCoverageSummary = pkgs.writeShellApplication { name = "podman-e2e-coverage-summary"; runtimeInputs = [ @@ -195,21 +199,48 @@ let root=$(git rev-parse --show-toplevel) cd "$root" - eligible=$(cargo nextest list \ + counts=$(cargo nextest list \ --manifest-path e2e/rust/Cargo.toml \ --target ${muslToolchain.target} \ -p openshell-e2e \ --features e2e-podman \ - --list-type binaries-only \ --message-format json \ - | jq '[.["rust-binaries"] | to_entries[] | select(.value.kind == "test")] | length') - - excluded=${toString (builtins.length podmanE2eFollowUpBinaries)} - selected=$((eligible - excluded)) + | jq --argjson excluded ${pkgs.lib.escapeShellArg (builtins.toJSON podmanE2eFollowUpBinaries)} ' + [ + .["rust-suites"][] + | select(.kind == "test") + | { + name: .["binary-name"], + selected: (.["binary-name"] as $name | $excluded | index($name) | not), + all_ignored: ((.testcases | length) > 0 and (.testcases | to_entries | all(.value.ignored == true))) + } + ] as $targets + | ($targets | map(select(.selected))) as $selected + | { + eligible: ($targets | length), + excluded_count: ($excluded | length), + selected: ($selected | length), + manual_only: [$selected[] | select(.all_ignored) | .name] + } + ') + + eligible=$(jq -r '.eligible' <<<"$counts") + excluded_count=$(jq -r '.excluded_count' <<<"$counts") + selected=$(jq -r '.selected' <<<"$counts") + manual_only_count=$(jq -r '.manual_only | length' <<<"$counts") + manual_only_names=$(jq -r '.manual_only | join(", ")' <<<"$counts") + executing=$((selected - manual_only_count)) echo "### Podman \`e2e-podman\` archive coverage" echo - echo "**$selected of $eligible** eligible targets selected ($excluded excluded — see \`podmanE2eFollowUpBinaries\` in \`tests/artifacts.nix\` for the documented reason behind each one)." + echo "- **$eligible** targets eligible under the \`e2e-podman\` feature" + echo "- **$excluded_count** excluded — see \`podmanE2eFollowUpBinaries\` in \`tests/artifacts.nix\` for the documented reason behind each one" + if [ "$manual_only_count" -gt 0 ]; then + echo "- **$selected** selected into the archive, of which **$manual_only_count** are manual-only benchmarks (\`#[ignore]\`) that don't run automatically: $manual_only_names" + else + echo "- **$selected** selected into the archive" + fi + echo "- **$executing** actually execute in this run" ''; }; From 6015c8ec85ada2c119fb362ce21d34b76da53eed Mon Sep 17 00:00:00 2001 From: politerealism Date: Sat, 3 Oct 2026 12:42:14 -0400 Subject: [PATCH 3/3] docs(podman): document and track the e2e-podman rootful coverage gap Address the rootless/rootful-scope acceptance criterion in #3712: document why the e2e-podman archive runs rootless-only in branch CI (unlike driver-podman, which runs both), and file a tracking issue for the one confirmed gap. - podman_host_gateway was checked directly rather than assumed: the outer container's host-gateway alias uses Podman's own native "host-gateway" value (resolved identically in both modes), and the supervisor-side resolution defaults to a hardcoded 127.0.0.1 on Linux regardless of rootful/rootless. No real coverage gap there. - podman_resource_limits is the one confirmed exception: it reads real cgroup v2 state, and rootless cgroup delegation is the harder, already-covered case. Rootful is unverified. Tracked in #4163, which also notes the e2e-podman tmachine playbook hardcodes rootless-only values and would need the same tmachine_container_runtime role pattern #3663 already proved out for the driver-podman suite. - CI.md and tests/artifacts.nix both say plainly that closing #4163 means running the whole archive rootful, not just the one target that needs it -- nextest archive filters select whole binaries, so there's no cheaper way to isolate just podman_resource_limits. Signed-off-by: politerealism --- CI.md | 8 ++++++++ tests/artifacts.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/CI.md b/CI.md index 71c70fcd82..938fb1fdbe 100644 --- a/CI.md +++ b/CI.md @@ -88,6 +88,14 @@ naming any selected manual-only benchmarks separately, so a narrow run cannot be mistaken for full Podman coverage. See `nix run .#podman-e2e-coverage-summary` in TESTING.md. +The `driver-podman` testsuite runs both rootful and rootless Fedora Podman; +`e2e-podman` runs rootless only. This is deliberate, not an oversight: see the +comment on `podmanE2eArchive` in `tests/artifacts.nix` for which targets would +and would not benefit from a rootful leg. Only `podman_resource_limits` does, +but nextest archive filters select whole binaries, so closing that gap (tracked +in #4163) means running this entire archive rootful too, not just that one +target. + Three opt-in labels enable the long-running E2E suites: - `test:e2e` runs the Docker, rootless Podman, Kubernetes, and VM E2E suites diff --git a/tests/artifacts.nix b/tests/artifacts.nix index b76d314a90..c1289743bf 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -252,6 +252,32 @@ let target = muslToolchain.target; output = "artifacts/test-archives/${muslToolchain.target}/openshell-podman-tests.tar"; }; + # Runs rootless-only in branch CI today (see .github/workflows/branch-e2e.yml's + # driver-specific-integration matrix), unlike podmanDriverArchive above, which + # runs both rootful and rootless. This is a deliberate scoping decision, not an + # oversight (#3712): most of this archive's selected targets (sandbox/workspace + # lifecycle, labels, templates, port forwarding, uploads, settings) exercise + # driver-agnostic gateway/policy logic with no privilege-model sensitivity, so a + # rootful leg would duplicate coverage without catching anything new. + # + # podman_host_gateway was checked specifically (host.openshell.internal + # resolution plausibly depends on the rootless pasta/slirp4netns vs. rootful + # netavark/CNI network backend) and confirmed NOT privilege-sensitive: the + # outer container's /etc/hosts entry uses Podman's own native "host-gateway" + # alias value (container.rs), which Podman resolves identically in both + # modes, and the supervisor-side resolution defaults to a hardcoded + # 127.0.0.1 on Linux regardless of rootful/rootless + # (PodmanComputeConfig::resolved_host_gateway_ip in config.rs). No rootful + # coverage gap here. + # + # The one confirmed exception is podman_resource_limits, which reads real + # cgroup v2 state from inside the sandbox: rootless cgroup delegation + # depends on systemd-user `Delegate=` and can silently no-op if + # misconfigured, making rootless the harder, higher-risk case -- already + # covered here. Rootful is the lower-risk, currently-unverified path for + # that one target. Tracked in #4163; since nextest archive filters select + # whole binaries, closing that gap means running this entire archive + # rootful too, not just the one target that needs it. podmanE2eArchive = mkTestArchive { name = "podman-e2e"; workspacePath = "e2e/rust";