diff --git a/client/Chart.yaml b/client/Chart.yaml index bdaf83cb..de42aa09 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.109 -appVersion: "1.9.109" +version: 1.9.110 +appVersion: "1.9.110" keywords: - tracebloc - kubernetes diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index 2b371ebe..d664eab1 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -41,33 +41,36 @@ data: # resource-monitor was likewise untouched (and a chart release does not # change its image ref either), so it only ever moved by accident. # - # TWO KNOWN, BOUNDED LIMITATIONS of keeping the annotation (rather than the - # live pod spec) as the source of truth. Both are self-healing at the next - # upstream image change, which at the control plane's release cadence is - # days, and neither can break a running edge — the pods stay offline-safe + # ONE KNOWN, BOUNDED LIMITATION of keeping the annotation (rather than the + # live pod spec) as the DIGEST source of truth. It is self-healing at the + # next upstream image change, which at the control plane's release cadence is + # days, and cannot break a running edge — the pods stay offline-safe # throughout because IfNotPresent does not depend on any of this. # - # 1. HELM RE-RENDER. `helm upgrade --reset-then-reuse-values` (the fleet - # auto-upgrade path) re-renders the templates, which write `repo:tag` - # and so revert an earlier `set image` pin. This tick will NOT re-pin: - # the annotation still records that digest, so `recorded == latest` - # and the loop no-ops. The edge floats on the tag until the next - # upstream release re-pins it. Note this only happens on a chart - # VERSION bump — auto-upgrade compares versions and skips otherwise — - # so it is not an hourly revert. + # PRE-EXISTING SKEW. requests-proxy and jobs-manager may already be + # running different builds of the same image on an edge upgrading INTO + # this version, because nothing reconciled requests-proxy before now. + # This script converges them on the next digest change (both are set + # in the same tick); it does not detect and repair skew that predates + # it, because it compares registry-vs-annotation, never pod-vs-pod. # - # 2. PRE-EXISTING SKEW. requests-proxy and jobs-manager may already be - # running different builds of the same image on an edge upgrading INTO - # this version, because nothing reconciled requests-proxy before now. - # This script converges them on the next digest change (both are set - # in the same tick); it does not detect and repair skew that predates - # it, because it compares registry-vs-annotation, never pod-vs-pod. + # HELM RE-RENDER, by contrast, is HANDLED rather than tolerated (client-runtime#199). + # `helm upgrade --reset-then-reuse-values` (the fleet auto-upgrade path) + # re-renders the templates, which write `repo:tag` and so revert an earlier + # `set image` pin. `recorded == latest` no longer means no-op: the loop reads + # each workload's LIVE image and re-pins the digest whenever the workload is + # off it. The revert therefore lasts ONE tick — the next tick puts the digest + # back — instead of floating on the tag until the next upstream release. It + # is compared on the @sha256 digest, so a registry-prefix rewrite (a mutating + # webhook) is not mistaken for a revert. This only happens on a chart VERSION + # bump anyway — auto-upgrade compares versions and skips otherwise — so it is + # not an hourly revert. # - # Fixing either properly means reconciling against each workload's LIVE - # container image instead of a shared annotation — a declarative reconcile, - # which `set image` makes possible for the first time (`rollout restart` - # was a blind action, which is why the annotation existed at all). That is - # a deliberate follow-up, not an oversight. + # Reconciling against the live spec is what makes that possible — the + # declarative reconcile `set image` enables (`rollout restart` was a blind + # action, which is why the annotation existed at all). The remaining + # pre-existing-skew case would need pod-vs-pod comparison; that is a + # deliberate follow-up, not an oversight. # # Source of truth: annotations on the JOBS-MANAGER deployment metadata # (`tracebloc.io/last-refreshed--digest`) — comparing the registry @@ -91,14 +94,29 @@ data: # # First-tick contract: annotation missing → record the current registry # digest WITHOUT touching the workload (no evidence of drift, no reason to - # churn pods). #569 keeps this deliberately. Pinning on the first tick + # churn pods). #569 keeps this deliberately. Pinning on the FIRST tick # would rewrite `repo:tag` to `repo@digest` on every fresh install — a spec # change, therefore a rollout, for byte-identical content, and for the - # resource-monitor DaemonSet that is a rollout across every node. The cost - # is that a freshly installed edge runs `repo:tag` until the first real - # digest change: still restart-safe offline (IfNotPresent), just not yet - # reproducible. Offline-safety is what #569 is fixing; reproducibility - # follows on the next upstream release. + # resource-monitor DaemonSet that is a rollout across every node. So the + # first tick only RECORDS. The NEXT tick, seeing `recorded == latest` but the + # workload still on `:tag`, pins the digest (client-runtime#199) — so a fresh edge + # becomes reproducible ~one interval post-install, NOT "at the next upstream + # release". Between the two it is still restart-safe offline (IfNotPresent). + # + # COST, stated honestly (@shujaatTracebloc / @LukasWodka on #1008): that + # re-pin is not "one cheap rollout". It enters the shared #563 flap path — + # `rollout status` on the resource-monitor DaemonSet, whose + # `desiredNumberScheduled` counts every node (tolerations: Exists), so it can + # never settle on a fleet with one NotReady/cordoned node; three such ticks + # (~45 min at the default 15m schedule) latch the SHARED MAX_REFRESH_ATTEMPTS + # lockout and stop refresh for ALL control-plane images until a human clears + # ATTEMPT_KEY, while the CronJob stays green. And jobs-manager is + # `strategy: Recreate`, so its extra rollout is full downtime + wait-for-mysql + # for byte-identical content, on every fresh install and again after each + # chart-version bump. The widening is kept deliberately — it also repairs a + # reinstall onto a node whose `:tag` layer is already stale — but that is the + # price, and a follow-up may gate the re-pin on "have we ever applied a digest + # here?" so a genuine fresh install skips the flap path entirely. # # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys # or container/env-array filtering motivates it; the rest stays in pure @@ -300,6 +318,55 @@ data: printf '%s\n' "$_json" | jq -r --arg k "$_key" '.metadata.annotations[$k] // empty' } + # The image reference the LIVE workload currently runs for $repo's primary + # container. Used to detect when `helm upgrade --reset-then-reuse-values` + # (the auto-upgrade) has re-rendered the workload back to the + # chart's `repo:tag` and so DISCARDED an earlier `set image repo@digest` + # pin. That revert is invisible to the digest comparison below -- the + # annotation still equals the registry digest, so `recorded == latest` + # reads "unchanged" and never re-pins -- while the workload sits on the + # bare tag (IfNotPresent per tracebloc.controlPlanePullPolicy). On a node + # whose `:tag` layer is stale that silently runs an OLD control-plane image + # (backend#2896-adjacent; it ran a pre-#416 jobs-manager under a sealed + # egress netpol on the stg/prod fleets, client-runtime#199). Container + # names are contractual with the deployment/daemonset templates -- keep in + # sync with the `case` block in the reconcile loop below. An empty result + # (read error / container absent) makes the caller SKIP the re-pin this tick + # and retry -- NOT re-assert (which would burn a #563 flap attempt on a + # healthy edge) and NOT assume agreement. That is the fail-closed stance + # get_annotation and the settled guard already take. + workload_image_for_repo() { + case "$1" in + tracebloc/jobs-manager) + kubectl get deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}' \ + --request-timeout=15s 2>/dev/null ;; + tracebloc/pods-monitor) + kubectl get deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="pods-monitor-container")].image}' \ + --request-timeout=15s 2>/dev/null ;; + tracebloc/resource-monitor) + kubectl get daemonset -n "$NODE_AGENTS_NAMESPACE" "$RESOURCE_MONITOR_DAEMONSET" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="tracebloc-resource-monitor")].image}' \ + --request-timeout=15s 2>/dev/null ;; + esac + } + + # The requests-proxy is a SEPARATE deployment that runs the SAME + # tracebloc/jobs-manager image (container `proxy`). `workload_image_for_repo` + # above reads only the jobs-manager `api` container, so the no-op decision + # below cannot see the proxy on its own. Read it here so a proxy left on + # `:tag` -- a tick that pinned `api` then died before the rp rollout, or a + # helm re-render that reverted only the proxy -- is still re-pinned instead + # of being declared "unchanged" forever because `api` happens to match + # (Bugbot on #1008). Empty (read error / absent) makes the caller SKIP this + # tick, same fail-closed stance as `workload_image_for_repo`. + requests_proxy_image() { + kubectl get deployment -n "$RELEASE_NAMESPACE" "$REQUESTS_PROXY_DEPLOYMENT" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="proxy")].image}' \ + --request-timeout=15s 2>/dev/null + } + # Skip the whole tick if the deployment isn't currently SETTLED (#546). A rollout # already in progress, or a pod stuck (e.g. Pending on volume binding), means a restart # can't help — it only churns ReplicaSets, and on a single-node local-path cluster that @@ -409,6 +476,23 @@ data: rp_set_args="" rm_set_args="" + # An UNFINISHED re-image attempt (client-runtime#199, Bugbot High on #1008). The + # restart block below increments ATTEMPT_KEY BEFORE the rollout and only + # resets it on a settled one; a rollout that times out exits the tick under + # set -e with the counter still raised. `kubectl set image` has by then + # updated the SPEC to repo@digest, so the live-image check reads "on digest" + # even though the rollout never completed -- and requests-proxy / + # resource-monitor sit OUTSIDE the top-of-tick settled guard, so a stuck + # rollout on either would no-op here forever, the counter raised and stale + # :tag pods still running. Read the counter once up front: a raised value + # forces the no-op branch to re-enter the re-image path so `rollout status` + # is retried -- resolving it (success resets the counter) or advancing it to + # the #563 flap lockout, which SURFACES the stuck rollout rather than hiding + # it. Best-effort: an unreadable/absent counter is treated as 0 (no forced + # retry), since the restart block's own read is the fail-closed authority. + pending_attempt="$(get_annotation "$ATTEMPT_KEY" || true)" + case "$pending_attempt" in ''|*[!0-9]*) pending_attempt=0 ;; esac + # Each entry: "|||". set -- \ "tracebloc/jobs-manager|tracebloc.io/last-refreshed-jobs-manager-digest|${JOBS_MANAGER_PINNED}|${JOBS_MANAGER_PIN:-}" \ @@ -525,11 +609,120 @@ data: fi if [ "$recorded" = "$latest" ]; then - log " digest unchanged since last refresh; no-op" - continue + # The registry digest has not moved since we recorded it -- but that + # alone does NOT prove the workload is running it. A `helm upgrade + # --reset-then-reuse-values` (the auto-upgrade) re-renders the + # Deployment back to `repo:tag` and discards our `set image repo@digest` + # pin; with `recorded == latest` this used to no-op, leaving the workload + # on the bare tag until the NEXT registry publish -- and on a node whose + # `:tag` layer is stale that silently runs an OLD image (client-runtime#199: + # a pre-#416 jobs-manager under a sealed egress netpol). So re-assert the + # pin whenever the live workload is not already on the pinned digest. + # + # The re-pin is BOUNDED TO ONE TICK: it writes the digest back and the + # next tick sees the workload on it and no-ops. It also converges a FRESH + # install to the digest one tick after first observation -- the first-tick + # contract in the header records without re-imaging, and this completes + # it, because staying on `:tag` is exactly the steady-state stale-`:tag` + # exposure this fix closes. + # + # Compare on the @sha256 DIGEST, not the whole image reference: a mutating + # admission webhook that rewrites the registry PREFIX to an internal mirror + # (seen behind hospital proxies) keeps the digest, so a prefix-only rewrite + # must NOT read as a revert -- otherwise the ref never equals the pinned one, + # every tick re-pins, the webhook rewrites it again, and three ticks trip the + # #563 flap lockout for ALL control-plane images (LukasWodka on #1008). A + # genuine revert to `:tag` carries no `@sha256` suffix, so `${ref##*@}` (the + # digest for `repo@sha256:...`, the whole ref otherwise) still mismatches a + # bare `sha256:...` and re-pins. + have="$(workload_image_for_repo "$repo" || true)" + # jobs-manager: the requests-proxy runs this SAME image as its own + # deployment. When it follows this digest (not operator-pinned) it must + # ALSO be on it, or a partial re-pin (api pinned, proxy still on :tag) is + # declared no-op forever off the api match alone and never retried (Bugbot + # on #1008). Any mismatch falls through to the re-image path, whose `case` + # block re-derives BOTH `jm_set_args`/`rp_set_args`. + proxy_follows=0 + rp_have="" + if [ "$repo" = "tracebloc/jobs-manager" ] && [ "$REQUESTS_PROXY_PINNED" != "1" ]; then + proxy_follows=1 + rp_have="$(requests_proxy_image || true)" + fi + # Unreadable live image (read error / container absent): SKIP the re-pin + # this tick and retry, rather than re-assert. Re-asserting on an unreadable + # read would burn a #563 flap attempt on a possibly-healthy edge and log a + # revert that may not have happened -- the same fail-closed stance the + # SKIP_KEY read and the settled guard take. A real `:tag` ref is readable + # and is NOT this case; it falls through and re-pins. + if [ -z "$have" ]; then + log " digest unchanged, but the live ${repo} image is unreadable (API read error / container absent) -- skipping re-pin this tick, will retry when readable" + continue + fi + if [ "$proxy_follows" = "1" ] && [ -z "$rp_have" ]; then + log " digest unchanged, but the live requests-proxy image is unreadable (API read error / container absent) -- skipping re-pin this tick, will retry when readable" + continue + fi + api_on_digest=1 + [ "${have##*@}" = "$latest" ] || api_on_digest=0 + proxy_on_digest=1 + if [ "$proxy_follows" = "1" ]; then + [ "${rp_have##*@}" = "$latest" ] || proxy_on_digest=0 + fi + if [ "$api_on_digest" = "1" ] && [ "$proxy_on_digest" = "1" ]; then + if [ "$pending_attempt" -gt 0 ] && [ "$pending_attempt" -lt "$MAX_REFRESH_ATTEMPTS" ]; then + # Only re-enter the rollout while there is budget left to resolve it. + # Once ATTEMPT_KEY has reached MAX_REFRESH_ATTEMPTS the flap guard below + # annotates FLAP_KEY and `exit 0`s BEFORE any `set image`/`rollout status` + # runs, so a latched forced-retry resolves nothing and surfaces nothing -- + # it only forces restart_needed=1 and skips the rest of the tick, dropping + # the annotation write (first-observation records, stale-pin clears) every + # tick, forever. Gating on `< MAX` lets a latched image fall to the no-op + # branch so the tick completes and its annotations land (@shujaatTracebloc + # on #1008, blocking 1 & 2). + # + # The SPEC reads on-digest, but a prior re-image attempt never reached + # its success-reset (ATTEMPT_KEY is raised): its rollout timed out and + # `set image` had already moved the spec, so this "on digest" can be a + # rollout that never settled -- and for requests-proxy / resource-monitor + # nothing else would catch it (they are outside the settled guard). + # Re-enter the re-image path so `rollout status` runs again: a settled + # workload resets the counter, a stuck one advances it to the #563 flap + # lockout, which surfaces it (Bugbot High on #1008). Re-`set image` with + # the same ref is an idempotent no-op patch, so a genuinely-settled + # workload pays only one fast `rollout status`. + log " workload spec is on the pinned digest, but ATTEMPT_KEY=${pending_attempt} marks an unfinished re-image (a rollout that never settled) -- re-running the rollout to resolve it or surface it via the flap guard" + else + # Latched on-digest: the spec is on the pinned digest but a prior + # re-image never reset ATTEMPT_KEY, and the `< MAX` gate above now + # keeps restart_needed=0 so the flap guard below (the only other + # writer of FLAP_KEY / the MANUAL ATTENTION WARN) never runs on this + # tick. Surface the latch HERE, mirroring that guard, so a + # stopped-and-silent refresh is never inferable only from the + # CronJob's green (#1964): without this the tick would log a bare + # "no-op" on the exact tick refresh is dead for ALL control-plane + # images (@shujaatTracebloc / @LukasWodka / @saadqbal on #1008). + if [ "$pending_attempt" -ge "$MAX_REFRESH_ATTEMPTS" ]; then + log " WARN: workload is on the pinned digest but ${ATTEMPT_KEY}=${pending_attempt} (>= MAX_REFRESH_ATTEMPTS=${MAX_REFRESH_ATTEMPTS}) -- FLAP LATCHED: image refresh is STOPPED for ALL control-plane images and does not auto-resume. MANUAL ATTENTION NEEDED: clear the ${ATTEMPT_KEY} annotation on deployment/${DEPLOYMENT_NAME} to re-arm refresh." + kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + "${FLAP_KEY}=${pending_attempt}" --overwrite --request-timeout=15s + fi + log " digest unchanged and workload already on the pinned digest; no-op" + continue + fi + elif [ "$api_on_digest" = "1" ]; then + log " digest unchanged and jobs-manager already on the pinned digest, but" + log " deployment/${REQUESTS_PROXY_DEPLOYMENT} runs '${rp_have}', not digest ${latest}" + log " -- re-pinning the requests-proxy digest" + else + log " digest unchanged, but the workload runs '${have}', not digest ${latest}" + log " (fresh install, or a helm re-render reverted the pin onto :${IMAGE_TAG}) -- re-pinning the digest" + fi + # Fall through to the re-image path (ref + case block) with recorded + # already == latest: the annotate below is an idempotent re-write, and + # restart_needed drives the rollout that puts the digest back on. + else + log " digest changed (${recorded} -> ${latest}); re-image needed" fi - - log " digest changed (${recorded} -> ${latest}); re-image needed" annotate_args="$annotate_args ${key}=${latest}" restart_needed=1 diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index 0e48630a..3787fbc7 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -223,6 +223,21 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: "pinned by digest in values" + # Regression guard (Bugbot #1008): the requests-proxy is a SEPARATE + # deployment running the SAME jobs-manager image, so the no-op "already on + # the pinned digest" decision MUST also read the proxy and fall through + # when it is off the digest -- else a partial re-pin (api pinned, proxy + # still on :tag) is declared unchanged forever off the api match alone and + # never retried. Lock the reader and the per-workload digest check in + # place. (The BEHAVIOUR -- that an inverted check reddens -- is asserted in + # scripts/tests/image-refresh-repin-on-revert.bats; this only pins that the + # two pieces still exist in the shipped script.) + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'requests_proxy_image\(\)' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'proxy_on_digest' # Regression guard: the script must HEAD the manifest with all # four Accept media types in a SINGLE comma-separated Accept # header per the Docker registry v2 spec (some proxies have been @@ -935,3 +950,23 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: 'rm_set_args tracebloc-resource-monitor=' + + - it: reconcile re-pins when a helm re-render reverted the workload off the digest + # Guards the client-runtime#199 fix: with `recorded == latest` the loop must + # NOT unconditionally no-op -- it must read the live workload image and + # re-pin when it is not `repo@latest` (a `helm upgrade --reset-then-reuse-values` + # reverted the pin onto the bare :tag, where a stale node cache serves an old + # image). The helper + the fall-through into the re-image path are the fix. + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'workload_image_for_repo\(\)' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'have="\$\(workload_image_for_repo "\$repo"' + # the true no-op now requires BOTH digest-unchanged AND workload-on-digest + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'workload already on the pinned digest; no-op' diff --git a/client/values.yaml b/client/values.yaml index 4dbebf55..564c0962 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -1718,11 +1718,16 @@ autoUpgrade: # - First observation (annotation absent on a fresh install): record # the current digest without re-imaging. Rewriting repo:tag to # repo@digest for byte-identical content would roll every workload — -# including the DaemonSet on every node — for nothing. The cost is -# that a fresh edge runs repo:tag until the first real digest change: -# still restart-safe offline, just not yet reproducible. -# - Idle-cheap: when the recorded digest matches today's digest, the -# script exits without touching anything. Steady state is one HEAD +# including the DaemonSet on every node — for nothing. The NEXT tick, +# seeing the digest recorded but the workload still on repo:tag, pins +# it (client-runtime#199) — so a fresh edge becomes reproducible ~one interval +# post-install, not at the next upstream release. Restart-safe offline +# throughout. +# - Idle-cheap: when the recorded digest matches today's digest AND the +# workload already runs that digest, the script exits without touching +# anything. If a helm re-render reverted the pin back to repo:tag it +# re-pins that one tick (compared on the @sha256 digest, so a mirror +# prefix rewrite is not mistaken for a revert). Steady state is one HEAD # per image per tick, well under Docker Hub's 100/6h anonymous # pull-rate limit. # - Private mirrors (global.imageRegistry): the script resolves digests diff --git a/scripts/tests/image-refresh-repin-on-revert.bats b/scripts/tests/image-refresh-repin-on-revert.bats new file mode 100644 index 00000000..9ebfb874 --- /dev/null +++ b/scripts/tests/image-refresh-repin-on-revert.bats @@ -0,0 +1,237 @@ +#!/usr/bin/env bats +# image-refresh RE-PINS the digest when a helm re-render reverted the workload +# to `repo:tag`, instead of no-op'ing off the annotation alone. +# +# client-runtime#199. `recorded == latest` proves the REGISTRY digest has not moved; it +# does NOT prove the workload is running it. `helm upgrade --reset-then-reuse-values` +# (the fleet auto-upgrade) re-renders the Deployment back to `repo:tag` and discards +# an earlier `set image repo@digest` pin -- and on a node whose `:tag` layer is +# stale that silently runs an OLD control-plane image (client-runtime#199). So the +# loop reads each workload's LIVE image and re-pins whenever it is off the digest. +# +# These assert BEHAVIOUR, not text presence: the earlier helm-unittest checks that +# `workload_image_for_repo`/`have=`/`proxy_on_digest` merely APPEAR in the script +# still pass if the comparison is inverted. This extracts the shipped branch from +# the RENDERED chart and drives it with the registry + live-workload reads stubbed, +# so an inverted comparison reddens. + +setup() { + TMP="$(mktemp -d)" + CHART="${BATS_TEST_DIRNAME}/../../client" + helm template t "$CHART" --set clientId=x --set clientPassword=y \ + --set storageClass.create=false > "$TMP/rendered.yaml" + python3 - "$TMP/rendered.yaml" "$TMP/branch.sh" <<'PYX' +import sys + +try: + import yaml +except ImportError: + sys.exit("[ERROR] PyYAML required (pip install pyyaml)") + +MARKER = "already on the pinned digest; no-op" + +def walk(o): + if isinstance(o, str) and MARKER in o: + return o + if isinstance(o, dict): + for v in o.values(): + r = walk(v) + if r: + return r + if isinstance(o, list): + for v in o: + r = walk(v) + if r: + return r + +script = None +for d in yaml.safe_load_all(open(sys.argv[1])): + if not d: + continue + script = walk(d) + if script: + break +assert script, "no rendered script containing the re-pin branch" + +lines = script.splitlines() +start = next(i for i, l in enumerate(lines) + if l.strip() == 'if [ "$recorded" = "$latest" ]; then') +# The branch ends at the `esac` that closes the `case "$repo" in` re-image block +# (no nested `case`, so the first `esac` after it closes it). +case_at = next(i for i in range(start, len(lines)) + if lines[i].strip() == 'case "$repo" in') +end = next(i for i in range(case_at, len(lines)) if lines[i].strip() == "esac") +body = "\n".join(l[6:] if l.startswith(" " * 6) else l.lstrip() + for l in lines[start:end + 1]) +open(sys.argv[2], "w").write(body) +PYX +} +teardown() { rm -rf "$TMP"; } + +# Runs the shipped re-pin branch with the registry HEAD (already known: recorded +# == latest) and the two LIVE-image reads stubbed. +# $1 = STUB_API what workload_image_for_repo returns ("" = unreadable) +# $2 = STUB_PROXY what requests_proxy_image returns ("" = unreadable) +# $3 = RP_PINNED "1" opts the requests-proxy out of following the digest +# $4 = PENDING the ATTEMPT_KEY value carried in (0 = no unfinished re-image) +# +# The branch is wrapped in a ONE-ITERATION loop so its `continue` statements run +# as they ship, rather than being stripped (which would change control flow). +run_branch() { + cat > "$TMP/harness.sh" <= MAX) is a no-op, not a forced re-run" { + # Once ATTEMPT_KEY has reached MAX_REFRESH_ATTEMPTS the flap guard downstream + # annotates FLAP_KEY and exit 0s BEFORE any set image / rollout status, so a + # forced re-run there resolves nothing and, worse, skips the tick's annotation + # write forever. The branch must fall to the no-op path instead + # (@shujaatTracebloc on #1008, blocking 1 & 2). MAX is 3, so pending=3 latches. + run run_branch "docker.io/tracebloc/jobs-manager@sha256:aaa" \ + "docker.io/tracebloc/jobs-manager@sha256:aaa" "0" "3" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"; no-op"* ]] || return 1 + [[ "$output" == *"RESTART:0"* ]] || return 1 + [[ "$output" != *"unfinished re-image"* ]] || return 1 +} + +@test "a LATCHED tick still SURFACES the stopped refresh (WARN + FLAP_KEY), not a bare no-op" { + # With the `< MAX` gate a latched tick keeps restart_needed=0 and never enters + # the downstream flap guard -- the only other writer of FLAP_KEY and the MANUAL + # ATTENTION WARN. So refresh is dead for ALL control-plane images while the + # CronJob stays green, and #1964 forbids "images did not update" being + # inferable only from the Job's colour. The latched arm must itself emit the + # WARN naming the refresh-attempt clear and annotate FLAP_KEY before the no-op + # (@shujaatTracebloc / @LukasWodka / @saadqbal on #1008). MAX is 3. + run run_branch "docker.io/tracebloc/jobs-manager@sha256:aaa" \ + "docker.io/tracebloc/jobs-manager@sha256:aaa" "0" "3" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"FLAP LATCHED"* ]] || return 1 + [[ "$output" == *"MANUAL ATTENTION NEEDED"* ]] || return 1 + [[ "$output" == *"clear the tracebloc.io/refresh-attempt annotation"* ]] || return 1 + [[ "$output" == *"KUBECTL:annotate deployment"*"tracebloc.io/refresh-flap-detected=3"* ]] || return 1 +} + +@test "a NON-latched no-op (pending=MAX; a clean on-digest + # tick (pending=0) and a bounded-attempt tick must not annotate FLAP_KEY. + run run_branch "docker.io/tracebloc/jobs-manager@sha256:aaa" \ + "docker.io/tracebloc/jobs-manager@sha256:aaa" "0" "0" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"; no-op"* ]] || return 1 + [[ "$output" != *"FLAP LATCHED"* ]] || return 1 + [[ "$output" != *"KUBECTL:"* ]] || return 1 +} + +@test "api on digest but proxy reverted re-pins the PROXY, not the api" { + run run_branch "docker.io/tracebloc/jobs-manager@sha256:aaa" \ + "docker.io/tracebloc/jobs-manager:dev" "0" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"re-pinning the requests-proxy digest"* ]] || return 1 + [[ "$output" == *"RESTART:1"* ]] || return 1 + [[ "$output" == *"proxy=docker.io/tracebloc/jobs-manager@sha256:aaa"* ]] || return 1 + # the fall-through re-derives BOTH set args; re-setting the api to the digest + # it already runs is an idempotent no-op patch (no rollout), which is why the + # proxy-only revert is repaired without special-casing it. + [[ "$output" == *"api=docker.io/tracebloc/jobs-manager@sha256:aaa"* ]] || return 1 +} + +@test "an unreadable api image SKIPS the re-pin this tick (no restart, no churn)" { + run run_branch "" "" "1" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"unreadable"* ]] || return 1 + [[ "$output" == *"skipping re-pin this tick"* ]] || return 1 + [[ "$output" == *"RESTART:0"* ]] || return 1 +} + +@test "an unreadable requests-proxy image SKIPS the re-pin this tick too" { + run run_branch "docker.io/tracebloc/jobs-manager@sha256:aaa" "" "0" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"requests-proxy image is unreadable"* ]] || return 1 + [[ "$output" == *"RESTART:0"* ]] || return 1 +} + +@test "a registry-prefix rewrite (mutating webhook) is NOT mistaken for a revert" { + # The webhook keeps the @sha256 suffix; comparing on the digest must read this + # as already-pinned, or every tick re-pins and three ticks trip the #563 flap + # lockout for all images (LukasWodka on #1008). + run run_branch "mirror.internal/tracebloc/jobs-manager@sha256:aaa" \ + "mirror.internal/tracebloc/jobs-manager@sha256:aaa" "0" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"no-op"* ]] || return 1 + [[ "$output" == *"RESTART:0"* ]] || return 1 +}