test(kyverno): prove the stale policy report prune on a kind cluster

.github/workflows/prove-kyverno-stale-report-prune.yaml

@@ -0,0 +1,51 @@
+name: Prove Kyverno Stale Report Prune
+
+# #2573: prove the stale-report DeletingPolicy on a throwaway kind cluster
+# before it reaches production. Runs no production credentials.
+on:
+  pull_request:
+    paths:
+      - .github/workflows/prove-kyverno-stale-report-prune.yaml
+      - scripts/prove-kyverno-stale-report-prune.sh
+      - tests/kyverno-stale-report-prune/**
+      - k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: prove-kyverno-stale-report-prune-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prove:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Create a kind cluster
+        env:
+          # renovate: datasource=go depName=sigs.k8s.io/kind
+          KIND_VERSION: v0.33.0
+        run: |
+          go install "sigs.k8s.io/kind@${KIND_VERSION}"
+          "$(go env GOPATH)/bin/kind" create cluster --name prune --wait 5m
+
+      - name: Prove the stale report prune
+        run: |
+          # Prove against the chart version production runs.
+          KYVERNO_CHART_VERSION="$(yq '.spec.chart.spec.version' k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml)"
+          export KYVERNO_CHART_VERSION
+          scripts/prove-kyverno-stale-report-prune.sh

scripts/prove-kyverno-stale-report-prune.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Proves, on a throwaway cluster, that tests/kyverno-stale-report-prune/deleting-policy.yaml
+# removes a Kyverno result that a name exclusion left stale, that the next scan
+# recreates the report with only its current results, and that a report whose
+# results are all current is left alone.
+#
+# Requires kubectl, helm and a cluster in the current kubeconfig context with
+# nothing else on it. CI creates one with kind (.github/workflows/prove-kyverno-stale-report-prune.yaml).
+set -euo pipefail
+
+dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/tests/kyverno-stale-report-prune"
+chart_version="${KYVERNO_CHART_VERSION:?set KYVERNO_CHART_VERSION}"
+# Short enough to finish in minutes, still several one-minute scan intervals.
+stale_after="${STALE_AFTER:-4m}"
+ns=prune-test
+
+log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
+fail() {
+  log "FAIL: $*"
+  kubectl get policyreports -n "$ns" -o yaml || true
+  kubectl get deletingpolicies.policies.kyverno.io -o yaml || true
+  kubectl -n kyverno logs deploy/kyverno-cleanup-controller --tail=200 || true
+  kubectl -n kyverno logs deploy/kyverno-reports-controller --tail=100 || true
+  exit 1
+}
+
+# Prints "<report uid> <space-separated policy/rule list>" for a ConfigMap's report,
+# or nothing when the report does not exist.
+report_state() {
+  local cm_uid
+  cm_uid="$(kubectl -n "$ns" get configmap "$1" -o jsonpath='{.metadata.uid}')"
+  kubectl -n "$ns" get policyreport "$cm_uid" -o json 2>/dev/null |
+    jq -r '[.metadata.uid, ([.results[]? | "\(.policy)/\(.rule)"] | sort | join(" "))] | join(" ")' || true
+}
+
+# Prints the unix seconds of one result's timestamp in a ConfigMap's report.
+result_timestamp() { # <configmap> <policy> <rule>
+  local cm_uid
+  cm_uid="$(kubectl -n "$ns" get configmap "$1" -o jsonpath='{.metadata.uid}')"
+  kubectl -n "$ns" get policyreport "$cm_uid" -o json 2>/dev/null |
+    jq -r --arg policy "$2" --arg rule "$3" \
+      'first(.results[]? | select(.policy == $policy and .rule == $rule) | .timestamp.seconds) // empty' ||
+    true
+}
+
+# wait_for <description> <seconds> <command...>: retries the command every 10s.
+wait_for() {
+  local what="$1" budget="$2"
+  shift 2
+  local deadline=$((SECONDS + budget))
+  until "$@"; do
+    ((SECONDS < deadline)) || fail "timed out after ${budget}s waiting for: $what"
+    sleep 10
+  done
+  log "ok: $what"
+}
+
+has_results() { # <configmap> <expected sorted policy/rule list>
+  local state
+  state="$(report_state "$1")"
+  [[ "${state#* }" == "$2" ]]
+}
+
+log "installing kyverno chart $chart_version"
+helm repo add kyverno https://kyverno.github.io/kyverno/ >/dev/null
+helm upgrade --install kyverno kyverno/kyverno --version "$chart_version" \
+  -n kyverno --create-namespace -f "$dir/values.yaml" --wait --timeout 10m >/dev/null
+
+kubectl apply -f "$dir/cleanup-controller-role.yaml"
+kubectl apply -f "$dir/fixtures.yaml"
+
+both="require-owner-label/owner-label require-team-label/team-label"
+wait_for "both results reported for excluded-later" 600 has_results excluded-later "$both"
+wait_for "both results reported for always-current" 600 has_results always-current "$both"
+
+team_before="$(result_timestamp excluded-later require-team-label team-label)"
+owner_before="$(result_timestamp excluded-later require-owner-label owner-label)"
+[[ -n "$team_before" && -n "$owner_before" ]] || fail "could not read the result timestamps to compare against"
+
+log "excluding excluded-later from require-team-label"
+kubectl patch clusterpolicy require-team-label --type=json -p '[{"op":"add","path":"/spec/rules/0/exclude","value":{"any":[{"resources":{"names":["excluded-later"]}}]}}]'
+
+# Reproduce the defect before relying on the fix. A scan that has actually run
+# since the exclusion rewrites the still-evaluated result, so waiting for the
+# owner-label timestamp to advance proves a scan completed — a plain sleep would
+# not, and the stale result could then simply be one the scanner never reached.
+scan_ran() {
+  local now
+  now="$(result_timestamp excluded-later require-owner-label owner-label)"
+  [[ -n "$now" ]] && ((now > owner_before))
+}
+
+wait_for "a background scan to complete after the exclusion" 600 scan_ran
+
+team_after="$(result_timestamp excluded-later require-team-label team-label)"
+[[ "$team_after" == "$team_before" ]] ||
+  fail "the excluded rule's result was rewritten ($team_before -> $team_after), so it is not stale"
+has_results excluded-later "$both" || fail "defect did not reproduce: the stale result cleared without the policy"
+log "ok: defect reproduced, the excluded rule's result survives a completed scan unchanged"
+
+control_before="$(report_state always-current)"
+control_before="${control_before%% *}"
+stale_report_uid="$(report_state excluded-later)"
+stale_report_uid="${stale_report_uid%% *}"
+
+log "applying the deleting policy (stale after $stale_after, every minute)"
+sed -e "s/STALE_AFTER/$stale_after/" -e 's#schedule: ".*"#schedule: "* * * * *"#' "$dir/deleting-policy.yaml" | kubectl apply -f -
+
+wait_for "excluded-later's stale result pruned and its report recreated with only the current result" 900 \
+  has_results excluded-later "require-owner-label/owner-label"
+
+recreated_uid="$(report_state excluded-later)"
+[[ "${recreated_uid%% *}" != "$stale_report_uid" ]] ||
+  fail "the report still carries its original uid ($stale_report_uid), so it was never deleted and recreated"
+log "ok: report deleted and recreated ($stale_report_uid -> ${recreated_uid%% *})"
+
+control_after="$(report_state always-current)"
+[[ "${control_after%% *}" == "$control_before" ]] ||
+  fail "the control report was deleted although all its results were current ($control_before -> ${control_after%% *})"
+has_results always-current "$both" || fail "the control report lost a current result"
+log "ok: control report untouched"
+
+log "PASS"

tests/kyverno-stale-report-prune/cleanup-controller-role.yaml

@@ -0,0 +1,13 @@
+# Lets the Kyverno cleanup controller read and delete policy reports, which a
+# DeletingPolicy over them requires. The label aggregates this role into the
+# cleanup controller's chart-managed ClusterRole.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kyverno:cleanup-controller:policy-reports
+  labels:
+    rbac.kyverno.io/aggregate-to-cleanup-controller: "true"
+rules:
+  - apiGroups: [wgpolicyk8s.io]
+    resources: [policyreports, clusterpolicyreports]
+    verbs: [get, list, watch, delete]

tests/kyverno-stale-report-prune/deleting-policy.yaml

@@ -0,0 +1,28 @@
+# Deletes a PolicyReport whose newest scan left at least one result behind.
+#
+# Kyverno's background scan rewrites every result it still evaluates on each
+# pass, so a current result is never older than one scan interval. A result
+# for a rule that stopped matching the resource (a name exclusion, an exclude
+# block or a precondition) is never rewritten and never removed, and stays in
+# the report indefinitely. Deleting the whole report is safe: the next scan
+# recreates it holding only the results that are still evaluated.
+#
+# STALE_AFTER must be several scan intervals, so a scan that is merely late
+# never deletes a current report.
+apiVersion: policies.kyverno.io/v1beta1
+kind: DeletingPolicy
+metadata:
+  name: prune-stale-policy-reports
+spec:
+  schedule: "*/10 * * * *"
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [wgpolicyk8s.io]
+        apiVersions: [v1alpha2]
+        resources: [policyreports]
+  conditions:
+    - name: has-stale-result
+      expression: >-
+        object.?results.orValue([]).exists(r,
+          has(r.timestamp) && has(r.timestamp.seconds) &&
+          timestamp(int(r.timestamp.seconds)) < time.now() - duration('STALE_AFTER'))

tests/kyverno-stale-report-prune/fixtures.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: prune-test
+---
+# Its team-label result becomes stale once require-team-label excludes it.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: excluded-later
+  namespace: prune-test
+data: {}
+---
+# Control: every result stays current, so its report must never be deleted.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: always-current
+  namespace: prune-test
+data: {}
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-team-label
+spec:
+  validationFailureAction: Audit
+  background: true
+  rules:
+    - name: team-label
+      match:
+        any:
+          - resources:
+              kinds: [ConfigMap]
+              namespaces: [prune-test]
+      validate:
+        message: team label required
+        pattern:
+          metadata:
+            labels:
+              team: "?*"
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-owner-label
+spec:
+  validationFailureAction: Audit
+  background: true
+  rules:
+    - name: owner-label
+      match:
+        any:
+          - resources:
+              kinds: [ConfigMap]
+              namespaces: [prune-test]
+      validate:
+        message: owner label required
+        pattern:
+          metadata:
+            labels:
+              owner: "?*"

tests/kyverno-stale-report-prune/values.yaml

@@ -0,0 +1,17 @@
+# Production's report settings (k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml)
+# with a one-minute scan so the proof finishes in minutes.
+features:
+  backgroundScan:
+    skipResourceFilters: false
+    backgroundScanInterval: 1m
+config:
+  resourceFiltersInclude:
+    - "[Lease,*,*]"
+    - "[PolicyReport,*,*]"
+    - "[ClusterPolicyReport,*,*]"
+    - "[AdmissionReport,*,*]"
+    - "[ClusterAdmissionReport,*,*]"
+    - "[BackgroundScanReport,*,*]"
+    - "[ClusterBackgroundScanReport,*,*]"
+  resourceFiltersExclude:
+    - "[Node,*,*]"