fix(chart): interim control-plane trim so the smallest training envelope fits a 2-core / 8 GiB node again (1.9.112)

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.111
-appVersion: "1.9.111"
+version: 1.9.112
+appVersion: "1.9.112"
 keywords:
   - tracebloc
   - kubernetes

client/templates/egress-proxy-deployment.yaml

@@ -66,7 +66,7 @@ spec:
           resources:
             requests:
               cpu: {{ $epReq.cpu | default "50m" | quote }}
-              memory: {{ $epReq.memory | default "64Mi" | quote }}
+              memory: {{ $epReq.memory | default "160Mi" | quote }}
             limits:
               cpu: {{ $epLim.cpu | default "500m" | quote }}
               memory: {{ $epLim.memory | default "256Mi" | quote }}

client/templates/jobs-manager-deployment.yaml

@@ -333,13 +333,16 @@ spec:
         # unresourced-init set, and neither is reachability.
         # `scripts/tests/qos-reachability.sh` asserts it now.
         #
-        # requests.cpu is
-        # also raised (100m -> 250m) so a mass restart doesn't throttle its
-        # startup to a crawl (the ~9m recovery seen on lukas-test).
+        # requests.cpu was raised (100m -> 250m) so a mass restart doesn't throttle
+        # its startup to a crawl (the ~9m recovery seen on lukas-test); the interim
+        # trim (backend#2461) brings it to 150m -- the whole plane idles at 36m --
+        # and lowers requests.memory to 640Mi while the LIMIT stays 1Gi, so the
+        # #1144 equality is deliberately broken here: see the derivation on
+        # values.yaml#resources for why the request moved and the limit did not.
         resources:
           requests:
-            cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }}
-            memory: {{ .Values.resources.jobsManager.requests.memory | default "1Gi" | quote }}
+            cpu: {{ .Values.resources.jobsManager.requests.cpu | default "150m" | quote }}
+            memory: {{ .Values.resources.jobsManager.requests.memory | default "640Mi" | quote }}
           limits:
             cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }}
             memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }}
@@ -915,17 +918,20 @@ spec:
           capabilities:
             drop: ["ALL"]
         # #1144: same cluster-recovery treatment as the api container above —
-        # pods-monitor was also exit-137 killed in the mass restart. Pin
+        # pods-monitor was also exit-137 killed in the mass restart. #1144 pinned
         # requests.memory == limits.memory (512Mi) to lower its oom_score_adj
         # (a less-preferred node OOM victim; the pod stays Burstable — and cpu is
         # not the only reason: see the init-container note on the api container
-        # above, backend#2872), and raise
+        # above, backend#2872), and raised
         # requests.cpu (50m -> 100m) so its startup is not starved under
-        # contention.
+        # contention. The interim trim (backend#2461) lowers the REQUEST to 384Mi
+        # and keeps the 512Mi LIMIT: nothing has measured this container yet, so
+        # the ceiling that #1144's restart peak justified stays where it was
+        # (derivation on values.yaml#resources).
         resources:
           requests:
             cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }}
-            memory: {{ .Values.resources.podsMonitor.requests.memory | default "512Mi" | quote }}
+            memory: {{ .Values.resources.podsMonitor.requests.memory | default "384Mi" | quote }}
           limits:
             cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }}
             memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }}

client/templates/mysql-deployment.yaml

@@ -158,10 +158,10 @@ spec:
         # PodDisruptionBudget cover the eviction angle instead.
         resources:
           requests:
-            memory: {{ .Values.resources.mysql.requests.memory | default "1Gi" | quote }}
-            cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }}
+            memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }}
+            cpu: {{ .Values.resources.mysql.requests.cpu | default "150m" | quote }}
           limits:
-            memory: {{ .Values.resources.mysql.limits.memory | default "1Gi" | quote }}
+            memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }}
             {{- /*
               CPU limit is intentionally unset by default (see comment block
               above). Schema permits it, so honour an explicit override here

client/templates/requests-proxy-deployment.yaml

@@ -134,8 +134,8 @@ spec:
           {{- $rpLim := default dict $rp.limits }}
           resources:
             requests:
-              cpu: {{ $rpReq.cpu | default "100m" | quote }}
-              memory: {{ $rpReq.memory | default "256Mi" | quote }}
+              cpu: {{ $rpReq.cpu | default "50m" | quote }}
+              memory: {{ $rpReq.memory | default "128Mi" | quote }}
             limits:
               cpu: {{ $rpLim.cpu | default "1000m" | quote }}
               memory: {{ $rpLim.memory | default "512Mi" | quote }}

client/tests/jobs_manager_test.yaml

@@ -70,17 +70,21 @@ tests:
           path: spec.template.spec.containers[0].resources.requests
       - isNotEmpty:
           path: spec.template.spec.containers[0].resources.limits
-      # #1144: requests.memory pinned == limits.memory (1Gi api / 512Mi
-      # pods-monitor) to lower oom_score_adj during a mass restart.
+      # #1144 pinned requests.memory == limits.memory (1Gi api / 512Mi
+      # pods-monitor) to lower oom_score_adj during a mass restart. The interim
+      # trim (backend#2461, client#1032) lowers the REQUESTS (640Mi / 384Mi) and
+      # keeps the LIMITS (1Gi / 512Mi) -- the limit is the hard ceiling the
+      # restart peak justified and nothing has re-measured it; the request is
+      # what the scheduler and admission subtract. Derivation: values.yaml#resources.
       - equal:
           path: spec.template.spec.containers[0].resources.requests.memory
-          value: 1Gi
+          value: 640Mi
       - equal:
           path: spec.template.spec.containers[0].resources.limits.memory
           value: 1Gi
       - equal:
           path: spec.template.spec.containers[1].resources.requests.memory
-          value: 512Mi
+          value: 384Mi
       - equal:
           path: spec.template.spec.containers[1].resources.limits.memory
           value: 512Mi

client/tests/mysql_test.yaml

@@ -40,13 +40,13 @@ tests:
     asserts:
       - equal:
           path: spec.template.spec.containers[0].resources.requests.memory
-          value: 1Gi
+          value: 704Mi
       - equal:
           path: spec.template.spec.containers[0].resources.limits.memory
-          value: 1Gi
+          value: 704Mi
       - equal:
           path: spec.template.spec.containers[0].resources.requests.cpu
-          value: 250m
+          value: 150m
       # CPU limit deliberately unset — InnoDB lock-wait timeouts under throttling.
       - notExists:
           path: spec.template.spec.containers[0].resources.limits.cpu

client/tests/requests_proxy_test.yaml

@@ -127,10 +127,10 @@ tests:
     asserts:
       - equal:
           path: spec.template.spec.containers[0].resources.requests.cpu
-          value: 100m
+          value: 50m
       - equal:
           path: spec.template.spec.containers[0].resources.requests.memory
-          value: 256Mi
+          value: 128Mi
       - equal:
           path: spec.template.spec.containers[0].resources.limits.cpu
           value: 1000m

client/values.yaml

@@ -612,7 +612,7 @@ egressProxy:
   resources:
     requests:
       cpu: "50m"
-      memory: "64Mi"
+      memory: "160Mi"
     limits:
       cpu: "500m"
       memory: "256Mi"
@@ -1355,35 +1355,80 @@ images:
 # a mass restart does not throttle their startup. Same requests.memory==limits
 # philosophy mysql already uses (mysql likewise omits its cpu limit).
 resources:
+  # INTERIM TRIM (backend#2461, 2026-09-10) -- read before "fixing" any number back up.
+  #
+  # Requests are what the scheduler and the jobs-manager's admission SUBTRACT from
+  # the node; they are not what the process uses. Measured on an installer edge
+  # (backend#2461): the control plane requested 1050m / 3276Mi while using
+  # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest
+  # (#1017) and since client-runtime#544 the admission subtracts the chart's
+  # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the
+  # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests,
+  # minus k3s's own coredns + metrics-server (200m / 140Mi), left 900m / 2390Mi,
+  # and the smallest training envelope needs 1000m / 3072Mi. Every task on
+  # staging was refused NEVER_STEADY_STATE (e2e-test-agent fleet 34473120598).
+  #
+  # THE FOOTPRINT COUNTS THE COLLECTOR. `helm template` cannot look up the token
+  # Secret, so the render -- and therefore control-plane-footprint.sh, the
+  # installer embed and this derivation -- carries the telemetry collector's
+  # 100m / 128Mi. That is the right worst case: on a real edge the collector IS
+  # rendered once jobs-manager has written the token Secret (backend#2274), so an
+  # edge that trains on install must still train after that. With it counted the
+  # render is 650m / 2272Mi, and the node above leaves 1050m / 3126Mi (was
+  # 900m / 2390Mi): 50m and 54Mi over the envelope. Thin, and honest.
+  #
+  #   mysql           250m/1Gi   -> 150m/704Mi   peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom; requests==limits kept
+  #   jobs-manager    250m/1Gi   -> 150m/640Mi   168Mi idle (0.16x); LIMIT STAYS 1Gi (see below); cpu: the whole plane idles at 36m
+  #   pods-monitor    100m/512Mi -> 100m/384Mi   no measurement yet; LIMIT STAYS 512Mi (see below)
+  #   requests-proxy  100m/256Mi ->  50m/128Mi   a few requests a minute; limits unchanged (1000m/512Mi)
+  #   egress-proxy     50m/64Mi  ->  50m/160Mi   UP: measured 2.45x OVER its request (backend#2461)
+  #   collector       100m/128Mi unchanged       unmeasured (distroless, no readable cgroup) -- not trimmed on a guess
+  #
+  # WHY jobs-manager AND pods-monitor NOW HAVE requests < limits. #1144 pinned
+  # requests.memory == limits.memory on both so a cluster-recovery mass restart
+  # would not pick them as OOM victims; the PODS were Burstable then and still
+  # are (the unresourced init containers, see above). The trim lowers only the
+  # REQUEST -- what the scheduler and admission subtract -- and leaves the LIMIT
+  # where it was, because the limit is the hard cgroup ceiling and the restart
+  # peak that justifies it is exactly the leg backend#2461's load run has not
+  # produced yet (review on client#1032). The cost is a higher oom_score_adj
+  # under node pressure; the alternative was a hard ceiling nothing has measured.
+  # When the restart leg lands, set BOTH from its p95 and restore the equality.
+  #
+  # What this is NOT: the final trim. Re-derive from the load run; do not
+  # hand-tune here. Guards that read these values: control-plane-footprint.sh
+  # (ceiling), the generated embed in scripts/lib/install-client-helm.sh
+  # (scripts/gen-footprint-embed.sh --check), envelope-schedulability.sh.
   mysql:
     requests:
-      cpu: "250m"
-      memory: "1Gi"
+      cpu: "150m"
+      memory: "704Mi"
     limits:
-      memory: "1Gi"
+      memory: "704Mi"
       # No cpu limit — see comment above.
   jobsManager:
     requests:
-      cpu: "250m"
-      memory: "1Gi"
+      cpu: "150m"
+      memory: "640Mi"
     limits:
       cpu: "1000m"
       memory: "1Gi"
   podsMonitor:
     requests:
       cpu: "100m"
-      memory: "512Mi"
+      memory: "384Mi"
     limits:
       cpu: "500m"
       memory: "512Mi"
   # requests-proxy serves the Service Bus / backend communication path and
-  # is mostly idle (a few req/min). 100m/256Mi requests with headroom for
-  # the occasional burst; revisit if heap profiling shows growth. Schema
-  # at values.schema.json#resources.requestsProxy validates overrides.
+  # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi,
+  # backend#2461) with the limits left at 1000m/512Mi for the occasional burst;
+  # revisit if heap profiling shows growth. Schema at
+  # values.schema.json#resources.requestsProxy validates overrides.
   requestsProxy:
     requests:
-      cpu: "100m"
-      memory: "256Mi"
+      cpu: "50m"
+      memory: "128Mi"
     limits:
       cpu: "1000m"
       memory: "512Mi"

scripts/install-k8s.ps1

@@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes   = 5368709120
 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it
 # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both
 # installers' values must equal a fresh render of the chart in the same tree.
-$script:TbCpFootprintMemBytes = 3288334336
-$script:TbCpFootprintCpuMilli = 900
+$script:TbCpFootprintMemBytes = 2382364672
+$script:TbCpFootprintCpuMilli = 650
 # ── end generated footprint ─────────────────────────────────────────────────
 
 # Set by Get-TrainingResources when the machine is readable but below the

scripts/lib/install-client-helm.sh

@@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120
 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in
 # `make drift` (the required Source-of-truth drift job): the value below must
 # equal a fresh render of the chart in the same tree, or CI reddens.
-_TB_CP_FOOTPRINT_MEM_BYTES=3288334336
-_TB_CP_FOOTPRINT_CPU_MILLI=900
+_TB_CP_FOOTPRINT_MEM_BYTES=2382364672
+_TB_CP_FOOTPRINT_CPU_MILLI=650
 # ── end generated ───────────────────────────────────────────────────────────
 
 # ── the fallback training envelope (precedence step 4) ──────────────────────
@@ -724,12 +724,17 @@ _training_provenance() {
 #
 # Everything above sizes the envelope against ALLOCATABLE and stops. Nothing then
 # asked whether the number written can be scheduled beside what else runs on the
-# node: the chart's own control plane (3136 MiB / 900 m, from the render) plus
-# the distribution's system pods (~140 MiB / 200 m of coredns and metrics-server
-# on k3s). Measured on a real single-node install, the platform out-requests the
-# 3 GiB reserve the envelope subtracts, so `allocatable - 3 GiB` over-asked at
-# EVERY machine size and the training pod sat `Pending / Insufficient memory`
+# node: the chart's own control plane (the embedded _TB_CP_FOOTPRINT_* constants,
+# generated from the render by scripts/gen-footprint-embed.sh) plus the
+# distribution's system pods (~140 MiB / 200 m of coredns and metrics-server on
+# k3s). When this was written the chart requested 3136 MiB / 900 m -- MORE than
+# the 3 GiB reserve the envelope subtracts -- so `allocatable - 3 GiB` over-asked
+# at EVERY machine size and the training pod sat `Pending / Insufficient memory`
 # while the installer printed `Training size: ...` and cli doctor said OK.
+# backend#2461's interim trim (chart 1.9.112) brought the control plane under the
+# reserve, so today the fit below is a no-op on every golden vector; it stays
+# because the constants are generated, not promised, and the next chart that
+# out-requests the reserve should be reduced here rather than sit Pending again.
 #
 # The two functions below make the installer own that question at the moment it
 # writes the envelope. The 3 GiB constant itself is not touched here: fixing the

scripts/manifest.sha256

@@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c  scripts/lib/gp
 d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea  scripts/lib/setup-linux.sh
 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c  scripts/lib/cluster.sh
 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62  scripts/lib/gpu-plugins.sh
-320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a  scripts/lib/install-client-helm.sh
+9a473b2d11545183be0bb24a1446eb8a51bd6a525d5d08dcd09a574f3bd10e06  scripts/lib/install-client-helm.sh
 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a  scripts/lib/install-cli.sh
 ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07  scripts/lib/provision.sh
 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943  scripts/lib/assess.sh
 e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b  scripts/lib/probe.sh
 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd  scripts/lib/summary.sh
 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e  scripts/lib/diagnose.sh
-b2a223a1e819e7056f27707e9e5bd25c6c4bdd830212806240a2559e71f1761d  scripts/install-k8s.ps1
+07722bab739746746f85504867ba3d32d85f9bc39f318b0789880bb7d6c13548  scripts/install-k8s.ps1
 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5  scripts/lib/telemetry.ps1

scripts/tests/control-plane-footprint.bats

@@ -75,7 +75,7 @@ YAML
   # guard hands to backend#2460/#2461.
   #
   # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the
-  # live render the number is 3136 MiB either way -- the chart's init containers
+  # live render the number is the same either way -- the chart's init containers
   # carry no requests -- so the fix is INERT there and a green run proves nothing
   # about the formula. Here the two answers differ, deliberately:
   #
@@ -162,7 +162,13 @@ YAML
 }
 
 @test "guard: RATCHET reddens when the footprint would exceed the ceiling" {
-  TB_CP_FOOTPRINT_MEM_CEIL=3000 run bash "$GUARD"
+  # The ceiling is DERIVED from the live render (one MiB under it), so this test
+  # keeps reddening whatever the chart requests -- a written-down 3000 stopped
+  # breaching once backend#2461 trimmed the render from 3136 to 2336 MiB.
+  local mib
+  mib="$(bash "$GUARD" --print-footprint 2>/dev/null | cut -d' ' -f1)"
+  [[ "$mib" =~ ^[0-9]+$ ]] || { echo "could not derive the render footprint: '$mib'"; return 1; }
+  TB_CP_FOOTPRINT_MEM_CEIL=$(( mib - 1 )) run bash "$GUARD"
   [ "$status" -eq 1 ] || { echo "expected exit 1, got $status: $output"; return 1; }
   printf '%s\n' "$output" | grep -q 'exceed the recorded ceiling' || { echo "$output"; return 1; }
 }