diff --git a/.github/workflows/prove-kyverno-stale-report-prune.yaml b/.github/workflows/prove-kyverno-stale-report-prune.yaml new file mode 100644 index 000000000..1ca71443e --- /dev/null +++ b/.github/workflows/prove-kyverno-stale-report-prune.yaml @@ -0,0 +1,51 @@ +name: Prove Kyverno Stale Report Prune + +# #2573: prove the stale-report DeletingPolicy on a throwaway kind cluster +# before it reaches production. Runs no production credentials. +on: + pull_request: + paths: + - .github/workflows/prove-kyverno-stale-report-prune.yaml + - scripts/prove-kyverno-stale-report-prune.sh + - tests/kyverno-stale-report-prune/** + - k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml + workflow_dispatch: + +permissions: {} + +concurrency: + group: prove-kyverno-stale-report-prune-${{ github.ref }} + cancel-in-progress: true + +jobs: + prove: + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Create a kind cluster + env: + # renovate: datasource=go depName=sigs.k8s.io/kind + KIND_VERSION: v0.33.0 + run: | + go install "sigs.k8s.io/kind@${KIND_VERSION}" + "$(go env GOPATH)/bin/kind" create cluster --name prune --wait 5m + + - name: Prove the stale report prune + run: | + # Prove against the chart version production runs. + KYVERNO_CHART_VERSION="$(yq '.spec.chart.spec.version' k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml)" + export KYVERNO_CHART_VERSION + scripts/prove-kyverno-stale-report-prune.sh diff --git a/scripts/prove-kyverno-stale-report-prune.sh b/scripts/prove-kyverno-stale-report-prune.sh new file mode 100755 index 000000000..ac0d26385 --- /dev/null +++ b/scripts/prove-kyverno-stale-report-prune.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Proves, on a throwaway cluster, that tests/kyverno-stale-report-prune/deleting-policy.yaml +# removes a Kyverno result that a name exclusion left stale, that the next scan +# recreates the report with only its current results, and that a report whose +# results are all current is left alone. +# +# Requires kubectl, helm and a cluster in the current kubeconfig context with +# nothing else on it. CI creates one with kind (.github/workflows/prove-kyverno-stale-report-prune.yaml). +set -euo pipefail + +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/tests/kyverno-stale-report-prune" +chart_version="${KYVERNO_CHART_VERSION:?set KYVERNO_CHART_VERSION}" +# Short enough to finish in minutes, still several one-minute scan intervals. +stale_after="${STALE_AFTER:-4m}" +ns=prune-test + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } +fail() { + log "FAIL: $*" + kubectl get policyreports -n "$ns" -o yaml || true + kubectl get deletingpolicies.policies.kyverno.io -o yaml || true + kubectl -n kyverno logs deploy/kyverno-cleanup-controller --tail=200 || true + kubectl -n kyverno logs deploy/kyverno-reports-controller --tail=100 || true + exit 1 +} + +# Prints " " for a ConfigMap's report, +# or nothing when the report does not exist. +report_state() { + local cm_uid + cm_uid="$(kubectl -n "$ns" get configmap "$1" -o jsonpath='{.metadata.uid}')" + kubectl -n "$ns" get policyreport "$cm_uid" -o json 2>/dev/null | + jq -r '[.metadata.uid, ([.results[]? | "\(.policy)/\(.rule)"] | sort | join(" "))] | join(" ")' || true +} + +# Prints the unix seconds of one result's timestamp in a ConfigMap's report. +result_timestamp() { # + local cm_uid + cm_uid="$(kubectl -n "$ns" get configmap "$1" -o jsonpath='{.metadata.uid}')" + kubectl -n "$ns" get policyreport "$cm_uid" -o json 2>/dev/null | + jq -r --arg policy "$2" --arg rule "$3" \ + 'first(.results[]? | select(.policy == $policy and .rule == $rule) | .timestamp.seconds) // empty' || + true +} + +# wait_for : retries the command every 10s. +wait_for() { + local what="$1" budget="$2" + shift 2 + local deadline=$((SECONDS + budget)) + until "$@"; do + ((SECONDS < deadline)) || fail "timed out after ${budget}s waiting for: $what" + sleep 10 + done + log "ok: $what" +} + +has_results() { # + local state + state="$(report_state "$1")" + [[ "${state#* }" == "$2" ]] +} + +log "installing kyverno chart $chart_version" +helm repo add kyverno https://kyverno.github.io/kyverno/ >/dev/null +helm upgrade --install kyverno kyverno/kyverno --version "$chart_version" \ + -n kyverno --create-namespace -f "$dir/values.yaml" --wait --timeout 10m >/dev/null + +kubectl apply -f "$dir/cleanup-controller-role.yaml" +kubectl apply -f "$dir/fixtures.yaml" + +both="require-owner-label/owner-label require-team-label/team-label" +wait_for "both results reported for excluded-later" 600 has_results excluded-later "$both" +wait_for "both results reported for always-current" 600 has_results always-current "$both" + +team_before="$(result_timestamp excluded-later require-team-label team-label)" +owner_before="$(result_timestamp excluded-later require-owner-label owner-label)" +[[ -n "$team_before" && -n "$owner_before" ]] || fail "could not read the result timestamps to compare against" + +log "excluding excluded-later from require-team-label" +kubectl patch clusterpolicy require-team-label --type=json -p '[{"op":"add","path":"/spec/rules/0/exclude","value":{"any":[{"resources":{"names":["excluded-later"]}}]}}]' + +# Reproduce the defect before relying on the fix. A scan that has actually run +# since the exclusion rewrites the still-evaluated result, so waiting for the +# owner-label timestamp to advance proves a scan completed — a plain sleep would +# not, and the stale result could then simply be one the scanner never reached. +scan_ran() { + local now + now="$(result_timestamp excluded-later require-owner-label owner-label)" + [[ -n "$now" ]] && ((now > owner_before)) +} + +wait_for "a background scan to complete after the exclusion" 600 scan_ran + +team_after="$(result_timestamp excluded-later require-team-label team-label)" +[[ "$team_after" == "$team_before" ]] || + fail "the excluded rule's result was rewritten ($team_before -> $team_after), so it is not stale" +has_results excluded-later "$both" || fail "defect did not reproduce: the stale result cleared without the policy" +log "ok: defect reproduced, the excluded rule's result survives a completed scan unchanged" + +control_before="$(report_state always-current)" +control_before="${control_before%% *}" +stale_report_uid="$(report_state excluded-later)" +stale_report_uid="${stale_report_uid%% *}" + +log "applying the deleting policy (stale after $stale_after, every minute)" +sed -e "s/STALE_AFTER/$stale_after/" -e 's#schedule: ".*"#schedule: "* * * * *"#' "$dir/deleting-policy.yaml" | kubectl apply -f - + +wait_for "excluded-later's stale result pruned and its report recreated with only the current result" 900 \ + has_results excluded-later "require-owner-label/owner-label" + +recreated_uid="$(report_state excluded-later)" +[[ "${recreated_uid%% *}" != "$stale_report_uid" ]] || + fail "the report still carries its original uid ($stale_report_uid), so it was never deleted and recreated" +log "ok: report deleted and recreated ($stale_report_uid -> ${recreated_uid%% *})" + +control_after="$(report_state always-current)" +[[ "${control_after%% *}" == "$control_before" ]] || + fail "the control report was deleted although all its results were current ($control_before -> ${control_after%% *})" +has_results always-current "$both" || fail "the control report lost a current result" +log "ok: control report untouched" + +log "PASS" diff --git a/tests/kyverno-stale-report-prune/cleanup-controller-role.yaml b/tests/kyverno-stale-report-prune/cleanup-controller-role.yaml new file mode 100644 index 000000000..1218cd25a --- /dev/null +++ b/tests/kyverno-stale-report-prune/cleanup-controller-role.yaml @@ -0,0 +1,13 @@ +# Lets the Kyverno cleanup controller read and delete policy reports, which a +# DeletingPolicy over them requires. The label aggregates this role into the +# cleanup controller's chart-managed ClusterRole. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:cleanup-controller:policy-reports + labels: + rbac.kyverno.io/aggregate-to-cleanup-controller: "true" +rules: + - apiGroups: [wgpolicyk8s.io] + resources: [policyreports, clusterpolicyreports] + verbs: [get, list, watch, delete] diff --git a/tests/kyverno-stale-report-prune/deleting-policy.yaml b/tests/kyverno-stale-report-prune/deleting-policy.yaml new file mode 100644 index 000000000..5885a8b14 --- /dev/null +++ b/tests/kyverno-stale-report-prune/deleting-policy.yaml @@ -0,0 +1,28 @@ +# Deletes a PolicyReport whose newest scan left at least one result behind. +# +# Kyverno's background scan rewrites every result it still evaluates on each +# pass, so a current result is never older than one scan interval. A result +# for a rule that stopped matching the resource (a name exclusion, an exclude +# block or a precondition) is never rewritten and never removed, and stays in +# the report indefinitely. Deleting the whole report is safe: the next scan +# recreates it holding only the results that are still evaluated. +# +# STALE_AFTER must be several scan intervals, so a scan that is merely late +# never deletes a current report. +apiVersion: policies.kyverno.io/v1beta1 +kind: DeletingPolicy +metadata: + name: prune-stale-policy-reports +spec: + schedule: "*/10 * * * *" + matchConstraints: + resourceRules: + - apiGroups: [wgpolicyk8s.io] + apiVersions: [v1alpha2] + resources: [policyreports] + conditions: + - name: has-stale-result + expression: >- + object.?results.orValue([]).exists(r, + has(r.timestamp) && has(r.timestamp.seconds) && + timestamp(int(r.timestamp.seconds)) < time.now() - duration('STALE_AFTER')) diff --git a/tests/kyverno-stale-report-prune/fixtures.yaml b/tests/kyverno-stale-report-prune/fixtures.yaml new file mode 100644 index 000000000..7c2f65db0 --- /dev/null +++ b/tests/kyverno-stale-report-prune/fixtures.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: prune-test +--- +# Its team-label result becomes stale once require-team-label excludes it. +apiVersion: v1 +kind: ConfigMap +metadata: + name: excluded-later + namespace: prune-test +data: {} +--- +# Control: every result stays current, so its report must never be deleted. +apiVersion: v1 +kind: ConfigMap +metadata: + name: always-current + namespace: prune-test +data: {} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-team-label +spec: + validationFailureAction: Audit + background: true + rules: + - name: team-label + match: + any: + - resources: + kinds: [ConfigMap] + namespaces: [prune-test] + validate: + message: team label required + pattern: + metadata: + labels: + team: "?*" +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-owner-label +spec: + validationFailureAction: Audit + background: true + rules: + - name: owner-label + match: + any: + - resources: + kinds: [ConfigMap] + namespaces: [prune-test] + validate: + message: owner label required + pattern: + metadata: + labels: + owner: "?*" diff --git a/tests/kyverno-stale-report-prune/values.yaml b/tests/kyverno-stale-report-prune/values.yaml new file mode 100644 index 000000000..6ce660030 --- /dev/null +++ b/tests/kyverno-stale-report-prune/values.yaml @@ -0,0 +1,17 @@ +# Production's report settings (k8s/bases/infrastructure/controllers/kyverno/helm-release.yaml) +# with a one-minute scan so the proof finishes in minutes. +features: + backgroundScan: + skipResourceFilters: false + backgroundScanInterval: 1m +config: + resourceFiltersInclude: + - "[Lease,*,*]" + - "[PolicyReport,*,*]" + - "[ClusterPolicyReport,*,*]" + - "[AdmissionReport,*,*]" + - "[ClusterAdmissionReport,*,*]" + - "[BackgroundScanReport,*,*]" + - "[ClusterBackgroundScanReport,*,*]" + resourceFiltersExclude: + - "[Node,*,*]"