release-train: develop -> staging

.github/workflows/mirror-publish.yaml

@@ -277,6 +277,16 @@ jobs:
           curl -fsSL --tlsv1.2 --retry 3 --connect-timeout 10 --max-time 120 \
             -o "$RUNNER_TEMP/gitleaks.tgz" \
             "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
+          # Assert the digest is a full SHA-256 before trusting it. `sha256sum -c`
+          # reports a malformed line as "no properly formatted checksum lines
+          # found", and whether that is a non-zero exit depends on the coreutils
+          # build (GNU exits 1; the macOS sha256sum exits 0). An empty or
+          # truncated env var must be a hard failure here, not a verification
+          # that quietly checks nothing and lets the install proceed.
+          if [[ ! "${GITLEAKS_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then
+            echo "::error::GITLEAKS_SHA256 is not a 64-character hex SHA-256 digest."
+            exit 1
+          fi
           echo "${GITLEAKS_SHA256}  $RUNNER_TEMP/gitleaks.tgz" | sha256sum -c -
           tar -xzf "$RUNNER_TEMP/gitleaks.tgz" -C "$RUNNER_TEMP/bin" gitleaks
           chmod 0755 "$RUNNER_TEMP/bin/gitleaks"

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.115
-appVersion: "1.9.115"
+version: 1.9.117
+appVersion: "1.9.117"
 keywords:
   - tracebloc
   - kubernetes

client/templates/NOTES.txt

@@ -5,7 +5,7 @@
 {{ "\033[1;35m" }}Components:{{ "\033[0m" }}
   - {{ "\033[1;34m" }}Jobs Manager:{{ "\033[0m" }}            {{ "\033[0;33m" }}{{ include "tracebloc.fullname" . }}-jobs-manager{{ "\033[0m" }}
   - {{ "\033[1;34m" }}MySQL Host:{{ "\033[0m" }}                   {{ "\033[0;33m" }}mysql-client{{ "\033[0m" }}
-{{- if ne .Values.resourceMonitor false }}
+{{- if (include "tracebloc.resourceMonitorEnabled" .) }}
   - {{ "\033[1;34m" }}Resource Monitor (DS):{{ "\033[0m" }}   {{ "\033[0;33m" }}{{ include "tracebloc.resourceMonitorName" . }}{{ "\033[0m" }}
 {{- end }}
 

client/templates/_helpers.tpl

@@ -126,6 +126,42 @@ tracebloc.io/seal-check-name: {{ .name | quote }}
 {{ include "tracebloc.fullname" . }}-resource-monitor
 {{- end }}
 
+{{/*
+  tracebloc.resourceMonitorEnabled — the SINGLE reader of "is the resource-monitor
+  on", coalescing the two value shapes during the RFC-0076 alias window
+  (remove_by: 2026-12-31, client#1009):
+
+    legacy scalar   resourceMonitor: <bool>
+    new object      resourceMonitor.enabled: <bool>   (D2: <component>.enabled)
+
+  This is a bool→object rename, so a stored values.yaml or a bare
+  `--set resourceMonitor=true` still arrives as a SCALAR. Reading
+  `.Values.resourceMonitor.enabled` blindly would `fail` with "can't evaluate
+  field enabled in interface {}" on the scalar and, on a `--reuse-values`
+  upgrade that carries the scalar forward, silently drop the setting. So decide
+  the shape with kindIs and prefer the new `.enabled` form:
+
+    map     -> .enabled, defaulting to true when the key is absent
+    bool    -> the scalar itself
+    absent  -> enabled (the historical default: `ne <nil> false` was true)
+
+  Effective behaviour is unchanged: resourceMonitor.enabled=true does exactly
+  what resourceMonitor=true did. Emits "true" or nothing, so callers use
+  `(include "tracebloc.resourceMonitorEnabled" .)` in an `and`/`or` and
+  `not (include ...)` for the disabled case — the same idiom as
+  tracebloc.nodeAgentsInUse.
+*/}}
+{{- define "tracebloc.resourceMonitorEnabled" -}}
+{{- $rm := .Values.resourceMonitor -}}
+{{- if kindIs "map" $rm -}}
+{{- if ne (dig "enabled" true $rm) false -}}true{{- end -}}
+{{- else if kindIs "invalid" $rm -}}
+{{- "true" -}}
+{{- else -}}
+{{- if ne $rm false -}}true{{- end -}}
+{{- end -}}
+{{- end }}
+
 {{- define "tracebloc.rbacName" -}}
 {{ include "tracebloc.fullname" . }}-jobs-manager-rbac
 {{- end }}
@@ -384,11 +420,12 @@ nvidia-device-plugin-daemonset
     * `resourceMonitor: false` — there is no DaemonSet at all, so there is
       nothing to reconcile and a cross-namespace `set image` would just fail.
 
-  Nil-safe: `.Values.resourceMonitor` absent reads as enabled, matching the
-  `ne .Values.resourceMonitor false` gate on the DaemonSet itself.
+  Nil-safe via tracebloc.resourceMonitorEnabled, which absent reads as enabled,
+  matching the gate on the DaemonSet itself and honouring both the legacy scalar
+  and the new resourceMonitor.enabled object form.
 */}}
 {{- define "tracebloc.resourceMonitorRefreshPinned" -}}
-{{- if eq .Values.resourceMonitor false -}}
+{{- if not (include "tracebloc.resourceMonitorEnabled" .) -}}
 true
 {{- else if (default dict (default dict .Values.images).resourceMonitor).digest -}}
 true
@@ -1556,7 +1593,7 @@ https://api.tracebloc.io/
   became a second tenant, two of them were widened and the rest were not." The
   tri-state made `enabled` a second copy of the answer for a third time.
 */ -}}
-{{- if or (ne .Values.resourceMonitor false) (eq (include "tracebloc.telemetryCollectorState" .) "enabled") }}true{{ end -}}
+{{- if or (include "tracebloc.resourceMonitorEnabled" .) (eq (include "tracebloc.telemetryCollectorState" .) "enabled") }}true{{ end -}}
 {{- end -}}
 
 {{/*

client/templates/image-refresh-cronjob.yaml

@@ -500,6 +500,13 @@ data:
 
     restart_needed=0
     annotate_args=""
+    # Pass-0 diagnostic annotations -- the `stale-pin-<image>` clears and
+    # finding-sets -- accumulate SEPARATELY from annotate_args and are written in
+    # their own bounded annotate BEFORE the restart block (#1008 item 1).
+    # They touch only the stale-pin keys, never `last-refreshed-<image>-digest`,
+    # so they are safe to land before the rollout; keeping them out of the final
+    # annotate is what lets them survive a latched-flap tick (see the write site).
+    stale_pin_args=""
     # `kubectl set image` argument lists, accumulated per WORKLOAD (a single
     # `set image` call can carry several container=ref pairs, so the
     # two-container jobs-manager Deployment is re-imaged in one patch and
@@ -585,7 +592,7 @@ data:
           disabled_stale_key="tracebloc.io/stale-pin-${repo#*/}"
           if [ -n "$(get_annotation "$disabled_stale_key" || true)" ]; then
             log "        clearing ${disabled_stale_key}: this image is no longer pinned by a digest"
-            annotate_args="$annotate_args ${disabled_stale_key}-"
+            stale_pin_args="$stale_pin_args ${disabled_stale_key}-"
           fi
           continue
         fi
@@ -613,7 +620,7 @@ data:
           # (@saqlainsyed007 + Bugbot on client#824.)
           if [ -n "$(get_annotation "$stale_key" || true)" ]; then
             log "        clearing a previous ${stale_key}: the pin is current again"
-            annotate_args="$annotate_args ${stale_key}-"
+            stale_pin_args="$stale_pin_args ${stale_key}-"
           fi
         else
           log "  WARN: PIN IS STALE. values pin ${pin_digest}"
@@ -624,7 +631,7 @@ data:
           log "        in values (backend#2458)."
           # Queryable after the log ages out, on the same object the refresh
           # annotations use, so `kubectl describe` shows pin state beside refresh state.
-          annotate_args="$annotate_args ${stale_key}=${pin_latest}"
+          stale_pin_args="$stale_pin_args ${stale_key}=${pin_latest}"
         fi
         continue
       fi
@@ -637,7 +644,7 @@ data:
         unpinned_stale_key="tracebloc.io/stale-pin-${repo#*/}"
         if [ -n "$(get_annotation "$unpinned_stale_key" || true)" ]; then
           log "  clearing ${unpinned_stale_key}: this image is no longer pinned"
-          annotate_args="$annotate_args ${unpinned_stale_key}-"
+          stale_pin_args="$stale_pin_args ${unpinned_stale_key}-"
         fi
 
       latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)"
@@ -881,6 +888,32 @@ data:
       esac
     done
 
+    # Pass-0 annotations land HERE, before the restart block (#1008 item 1).
+    # The restart block's #563 flap guard does `WARN + FLAP_KEY + exit 0`
+    # once refresh-attempt >= MAX_REFRESH_ATTEMPTS -- BEFORE the digest-record
+    # annotate at the end of the tick. So on a tick that is both off-digest
+    # (restart_needed=1) and latched, batching the stale-pin writes into that
+    # final annotate dropped them: a stale-pin CLEAR that never landed leaves a
+    # FALSE "pin is stale" finding to persist forever (a write-only annotation
+    # outliving its problem -- the class client#824's clear paths fixed), and it
+    # is dropped on the exact tick refresh is dead, when the finding matters most.
+    # These touch only the `stale-pin-<image>` keys, never
+    # `last-refreshed-<image>-digest`, so writing them before the rollout cannot
+    # affect the `recorded == latest` skip logic -- unlike the digest record,
+    # which MUST stay after a successful `rollout status`, since annotating the
+    # digest before a failed rollout would freeze the workload on the old image
+    # (@shujaatTracebloc on #1008). NON-FATAL, like the SKIP_KEY clear above
+    # (backend#2007): a transient failure on a diagnostic annotation must not
+    # abort the tick before the re-image; a stale value is re-reconciled next tick.
+    if [ -n "$stale_pin_args" ]; then
+      log "updating stale-pin annotations:$stale_pin_args"
+      # shellcheck disable=SC2086  # word-split stale_pin_args intentional
+      if ! sp_err="$(kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \
+        $stale_pin_args --overwrite --request-timeout=15s 2>&1 >/dev/null)"; then
+        log "  WARNING: could not update stale-pin annotations on deployment/${DEPLOYMENT_NAME}: ${sp_err:-unknown error}. Continuing -- these are diagnostic bookkeeping and a stale value is re-reconciled on the next tick."
+      fi
+    fi
+
     # Order matters: rollout FIRST, annotate AFTER `rollout status`
     # succeeds. Annotating first would let a failed rollout silently
     # freeze the deployment on the old image (next tick sees

client/templates/jobs-manager-deployment.yaml

@@ -750,7 +750,7 @@ spec:
         # populated would list a namespace that isn't rendered — a 403 every
         # heartbeat for a workload the operator turned off.
         - name: NODE_AGENTS_NAMESPACE
-          value: {{ if ne .Values.resourceMonitor false }}{{ dig "namespace" "name" "" (.Values.nodeAgents | default dict) | quote }}{{ else }}""{{ end }}
+          value: {{ if (include "tracebloc.resourceMonitorEnabled" .) }}{{ dig "namespace" "name" "" (.Values.nodeAgents | default dict) | quote }}{{ else }}""{{ end }}
         # backend#664 (Utilization Ladder L0): with NEITHER env.RESOURCE_REQUESTS
         # nor env.RESOURCE_LIMITS set, BOTH vars are omitted and jobs-manager
         # sizes the envelope from node allocatable — BUT ONLY IF

client/templates/rbac.yaml

@@ -232,7 +232,7 @@ roleRef:
 {{- end }}
 
 {{- $nodeAgentsNs := dig "namespace" "name" "" (.Values.nodeAgents | default dict) }}
-{{- if and (ne .Values.resourceMonitor false) $nodeAgentsNs (ne $nodeAgentsNs .Release.Namespace) }}
+{{- if and (include "tracebloc.resourceMonitorEnabled" .) $nodeAgentsNs (ne $nodeAgentsNs .Release.Namespace) }}
 ---
 {{/*
   jobs-manager reads the resource-monitor DaemonSet for the heartbeat version

client/templates/resource-monitor-daemonset.yaml

@@ -1,4 +1,4 @@
-{{- if ne .Values.resourceMonitor false }}
+{{- if (include "tracebloc.resourceMonitorEnabled" .) }}
 {{/*
   Pre-flight: resource-monitor polls the metrics.k8s.io API, so metrics-server
   must be registered. We probe kube-system via `lookup` first — that returns
@@ -66,7 +66,7 @@
     {{- $preflight = "skipped-by-values" -}}
   {{- else -}}
     {{- if not (lookup "apiregistration.k8s.io/v1" "APIService" "" "v1beta1.metrics.k8s.io") -}}
-      {{- fail "resourceMonitor is enabled but the metrics.k8s.io/v1beta1 API is not registered. Install metrics-server (https://github.com/kubernetes-sigs/metrics-server) or set resourceMonitor: false. See SECURITY.md.\n\nIf THIS line was instead an `apiservices ... is forbidden` error, the problem is the caller's RBAC, not metrics-server: APIService is cluster-scoped and the built-in `admin` ClusterRole excludes it. Set nodeAgents.metricsServerPreflight: false to skip this check, or run the upgrade with cluster-scope read on apiservices (backend#2469)." -}}
+      {{- fail "resourceMonitor is enabled but the metrics.k8s.io/v1beta1 API is not registered. Install metrics-server (https://github.com/kubernetes-sigs/metrics-server) or set resourceMonitor.enabled: false. See SECURITY.md.\n\nIf THIS line was instead an `apiservices ... is forbidden` error, the problem is the caller's RBAC, not metrics-server: APIService is cluster-scoped and the built-in `admin` ClusterRole excludes it. Set nodeAgents.metricsServerPreflight: false to skip this check, or run the upgrade with cluster-scope read on apiservices (backend#2469)." -}}
     {{- end -}}
     {{- $preflight = "satisfied-by-apiservice" -}}
   {{- end -}}

client/templates/resource-monitor-rbac.yaml

@@ -1,4 +1,4 @@
-{{- if ne .Values.resourceMonitor false }}
+{{- if (include "tracebloc.resourceMonitorEnabled" .) }}
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -27,7 +27,7 @@ metadata:
   the training/jobs isolation footprint elsewhere -- it must not cripple node
   telemetry by leaving the DaemonSet without the permissions it cannot run without.
   If a deployment genuinely cannot allow any cluster-scoped read, disable the
-  monitor entirely via .Values.resourceMonitor=false rather than deploying it broken.
+  monitor entirely via .Values.resourceMonitor.enabled=false rather than deploying it broken.
 */}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1

client/templates/resource-monitor-scc.yaml

@@ -1,4 +1,4 @@
-{{- if and (ne .Values.resourceMonitor false) .Values.openshift.scc.enabled }}
+{{- if and (include "tracebloc.resourceMonitorEnabled" .) .Values.openshift.scc.enabled }}
 ---
 apiVersion: security.openshift.io/v1
 kind: SecurityContextConstraints

client/templates/secrets.yaml

@@ -510,7 +510,7 @@ data:
 {{- if (include "tracebloc.bootstrapDbReparent" .) }}
   DB_BOOTSTRAP_PASSWORD: {{ $bootstrapDbPassword | b64enc | quote }}
 {{- end }}
-{{- if and (ne .Values.resourceMonitor false) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }}
+{{- if and (include "tracebloc.resourceMonitorEnabled" .) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }}
 ---
 # Mirrored into the node-agents namespace so the resource-monitor DaemonSet
 # can read CLIENT_ID / CLIENT_PASSWORD via secretKeyRef. Secrets are

client/tests/image_refresh_test.yaml

@@ -1020,3 +1020,49 @@ tests:
       - notMatchRegex:
           path: data["image-refresh.sh"]
           pattern: 'recorded="\$\(get_annotation "\$key" \|\| true\)"'
+
+  - it: stale-pin annotations are written BEFORE the restart block, so they survive a latched flap
+    # Guards #1008 item 1. The #563 flap guard does WARN + FLAP_KEY +
+    # exit 0 once refresh-attempt >= MAX, BEFORE the final digest-record annotate.
+    # Batching the stale-pin CLEARS into that final annotate dropped them on a
+    # tick that is both off-digest and latched -- leaving a FALSE stale-pin
+    # finding to persist. They now accumulate in their own list and are annotated
+    # above the restart block. image-refresh-latched-annotate.bats asserts the
+    # BEHAVIOUR; these lock the code shapes.
+    template: templates/image-refresh-cronjob.yaml
+    documentIndex: 0
+    asserts:
+      # ALL FOUR stale-pin writes go to their own accumulator, not annotate_args.
+      # Every key gets a positive (goes to stale_pin_args) AND a negative (does NOT
+      # ride annotate_args) -- Bugbot Low + @saadqbal on #1039: a key covered by
+      # neither could be re-batched into annotate_args and dropped on a latched
+      # tick while both suites stayed green (this guard's own thesis, one level up).
+      # The two clears:
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'stale_pin_args="\$stale_pin_args \$\{stale_key\}-"'
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'stale_pin_args="\$stale_pin_args \$\{unpinned_stale_key\}-"'
+      # the disabled-monitor clear (the third clear -- was uncovered):
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'stale_pin_args="\$stale_pin_args \$\{disabled_stale_key\}-"'
+      # the finding-SET (a lost set means a real staleness goes unreported):
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'stale_pin_args="\$stale_pin_args \$\{stale_key\}=\$\{pin_latest\}"'
+      # and that accumulator is annotated BEFORE the restart block (ordering)
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: '(?s)if \[ -n "\$stale_pin_args" \]; then.*if \[ "\$restart_needed" -eq 1 \]; then'
+      # NONE of the stale-pin keys may ride the final digest-record annotate_args
+      - notMatchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'annotate_args="\$annotate_args \$\{stale_key\}'
+      - notMatchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'annotate_args="\$annotate_args \$\{unpinned_stale_key\}'
+      - notMatchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'annotate_args="\$annotate_args \$\{disabled_stale_key\}'