From 583ef3d2c0453a33afd8118d169ad4f5cf9d6e7b Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 15:19:06 +0200 Subject: [PATCH 1/6] fix(chart): interim control-plane trim so the smallest training envelope fits a 2-core / 8 GiB node again (1.9.112) Since chart 1.9.109 the kubelet reservation is honest (#1017) and since client-runtime#544 the jobs-manager's admission subtracts the chart's real requests. On the smallest supported node that made the arithmetic visible: allocatable 1900m / 5538Mi, minus the control plane's requests (800m / 3008Mi without the collector) and k3s's coredns + metrics-server (200m / 140Mi), leaves 900m / 2390Mi -- and the smallest envelope needs 1000m / 3072Mi. Every task on staging was refused NEVER_STEADY_STATE (e2e fleet 34473120598). The requests were never sized from usage: 1050m / 3276Mi requested against 36m / ~1022Mi used. This trims the components whose peaks are measured and takes jobs-manager down modestly, giving back 150m / 800Mi: mysql 250m/1Gi -> 150m/704Mi (peak 0.55x; requests==limits kept) jobs-manager 250m/1Gi -> 250m/768Mi (168Mi idle; restart peak pending) pods-monitor 100m/512Mi -> 100m/320Mi (requests==limits kept) requests-proxy 100m/256Mi -> 50m/128Mi (limits unchanged) egress-proxy 50m/64Mi -> 50m/160Mi (UP: measured 2.45x over) Render: 3136 MiB / 900 m -> 2336 MiB / 750 m. The node above now leaves 1050m / 3190Mi. Interim: the final numbers come from the load run in progress; jobs-manager moves only once its restart peak is measured. Everything that derives from the render moves with it: the footprint ceiling ratchets down, the installer embed (bash + PowerShell) is regenerated, the template fallback defaults match, and the fit tests derive their small/tiny nodes from the footprint instead of restating a machine size -- the 8 GiB reproduction is now pinned as fitting unreduced, and the cpu-only overshoot replays the pre-trim 900 m because no real node can produce it any more. Co-Authored-By: Claude Fable 5.1 --- client/Chart.yaml | 4 +- client/templates/egress-proxy-deployment.yaml | 2 +- client/templates/jobs-manager-deployment.yaml | 8 +- client/templates/mysql-deployment.yaml | 4 +- .../templates/requests-proxy-deployment.yaml | 4 +- client/tests/jobs_manager_test.yaml | 8 +- client/tests/mysql_test.yaml | 6 +- client/tests/requests_proxy_test.yaml | 4 +- client/values.yaml | 56 +++++++++--- scripts/install-k8s.ps1 | 4 +- scripts/lib/install-client-helm.sh | 4 +- scripts/manifest.sha256 | 4 +- scripts/tests/control-plane-footprint.sh | 11 +-- .../envelope-schedulability-mutations.sh | 2 +- scripts/tests/envelope-schedulability.bats | 18 ++-- scripts/tests/envelope-schedulability.sh | 63 +++++++++---- scripts/tests/install-client-helm.bats | 58 ++++++++---- scripts/tests/install-k8s.Tests.ps1 | 89 ++++++++++++------- 18 files changed, 233 insertions(+), 116 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 15c1fd38..14a392bd 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.111 -appVersion: "1.9.111" +version: 1.9.112 +appVersion: "1.9.112" keywords: - tracebloc - kubernetes diff --git a/client/templates/egress-proxy-deployment.yaml b/client/templates/egress-proxy-deployment.yaml index 792d96ab..2080b26d 100644 --- a/client/templates/egress-proxy-deployment.yaml +++ b/client/templates/egress-proxy-deployment.yaml @@ -66,7 +66,7 @@ spec: resources: requests: cpu: {{ $epReq.cpu | default "50m" | quote }} - memory: {{ $epReq.memory | default "64Mi" | quote }} + memory: {{ $epReq.memory | default "160Mi" | quote }} limits: cpu: {{ $epLim.cpu | default "500m" | quote }} memory: {{ $epLim.memory | default "256Mi" | quote }} diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 19543b4a..b724836a 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -339,10 +339,10 @@ spec: resources: requests: cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }} - memory: {{ .Values.resources.jobsManager.requests.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.jobsManager.requests.memory | default "768Mi" | quote }} limits: cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }} - memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.jobsManager.limits.memory | default "768Mi" | quote }} ports: # client-runtime#21: POST /internal/submit-ingestion-run. The # ingestor subchart's post-install hook hits this through the @@ -925,10 +925,10 @@ spec: resources: requests: cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }} - memory: {{ .Values.resources.podsMonitor.requests.memory | default "512Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.requests.memory | default "320Mi" | quote }} limits: cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }} - memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.limits.memory | default "320Mi" | quote }} volumeMounts: - name: logs-volume mountPath: "/data/logs" diff --git a/client/templates/mysql-deployment.yaml b/client/templates/mysql-deployment.yaml index b3f9dbf7..9d29b991 100644 --- a/client/templates/mysql-deployment.yaml +++ b/client/templates/mysql-deployment.yaml @@ -158,10 +158,10 @@ spec: # PodDisruptionBudget cover the eviction angle instead. resources: requests: - memory: {{ .Values.resources.mysql.requests.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }} cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }} limits: - memory: {{ .Values.resources.mysql.limits.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }} {{- /* CPU limit is intentionally unset by default (see comment block above). Schema permits it, so honour an explicit override here diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index a0f94e60..1ef868ce 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -134,8 +134,8 @@ spec: {{- $rpLim := default dict $rp.limits }} resources: requests: - cpu: {{ $rpReq.cpu | default "100m" | quote }} - memory: {{ $rpReq.memory | default "256Mi" | quote }} + cpu: {{ $rpReq.cpu | default "50m" | quote }} + memory: {{ $rpReq.memory | default "128Mi" | quote }} limits: cpu: {{ $rpLim.cpu | default "1000m" | quote }} memory: {{ $rpLim.memory | default "512Mi" | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 964de569..11c9d12f 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -74,16 +74,16 @@ tests: # pods-monitor) to lower oom_score_adj during a mass restart. - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 768Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 1Gi + value: 768Mi - equal: path: spec.template.spec.containers[1].resources.requests.memory - value: 512Mi + value: 320Mi - equal: path: spec.template.spec.containers[1].resources.limits.memory - value: 512Mi + value: 320Mi # SELECTED BY NAME, NOT BY POSITION. This was `env[0]`, and `env[0]` was # CLIENT_ID only for as long as nothing was ever prepended — which stopped being diff --git a/client/tests/mysql_test.yaml b/client/tests/mysql_test.yaml index e0d2ad7b..90808612 100644 --- a/client/tests/mysql_test.yaml +++ b/client/tests/mysql_test.yaml @@ -40,13 +40,13 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m + value: 150m # CPU limit deliberately unset — InnoDB lock-wait timeouts under throttling. - notExists: path: spec.template.spec.containers[0].resources.limits.cpu diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml index 19f66ea8..5161e1d0 100644 --- a/client/tests/requests_proxy_test.yaml +++ b/client/tests/requests_proxy_test.yaml @@ -127,10 +127,10 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 100m + value: 50m - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 256Mi + value: 128Mi - equal: path: spec.template.spec.containers[0].resources.limits.cpu value: 1000m diff --git a/client/values.yaml b/client/values.yaml index 6b22572d..caefc445 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -612,7 +612,7 @@ egressProxy: resources: requests: cpu: "50m" - memory: "64Mi" + memory: "160Mi" limits: cpu: "500m" memory: "256Mi" @@ -1355,35 +1355,65 @@ images: # a mass restart does not throttle their startup. Same requests.memory==limits # philosophy mysql already uses (mysql likewise omits its cpu limit). resources: + # INTERIM TRIM (backend#2461, 2026-09-10) -- read before "fixing" any number back up. + # + # Requests are what the scheduler and the jobs-manager's admission SUBTRACT from + # the node; they are not what the process uses. Measured on an installer edge + # (backend#2461): the control plane requested 1050m / 3276Mi while using + # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest + # (#1017) and since client-runtime#544 the admission subtracts the chart's + # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the + # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests + # (800m / 3008Mi without the collector) and k3s's own coredns + metrics-server + # (200m / 140Mi), LEFT 900m / 2390Mi, and the smallest training envelope needs + # 1000m / 3072Mi. Every task on staging was refused NEVER_STEADY_STATE + # (e2e-test-agent fleet 34473120598, 2026-09-10). + # + # The numbers below give back 150m / 800Mi so that node leaves 1050m / 3190Mi: + # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom, requests==limits kept + # jobs-manager 250m/1Gi -> 250m/768Mi 168Mi idle (0.16x); restart peak NOT yet measured -- 768 keeps 4.5x over idle, cpu untouched + # pods-monitor 100m/512Mi -> 100m/320Mi requests==limits kept (oom_score_adj, #1144) + # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) so bursts still fit + # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) -- it was the one under-provisioned component + # + # What this is NOT: the final trim. backend#2461 sizes requests from p95 under a + # load run (ambient, one cycle, five-queued burst, node restart under 8 GiB) that + # is in progress; jobs-manager's request in particular is justified by the + # restart peak and moves only once that leg is measured. Re-derive from the + # measurement; do not hand-tune here. The guards that read these values: + # scripts/tests/control-plane-footprint.sh (ceiling), the generated embed in + # scripts/lib/install-client-helm.sh (scripts/gen-footprint-embed.sh --check), + # scripts/tests/envelope-schedulability.sh (fit on the golden vectors). mysql: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "704Mi" limits: - memory: "1Gi" + memory: "704Mi" # No cpu limit — see comment above. jobsManager: requests: cpu: "250m" - memory: "1Gi" + memory: "768Mi" limits: cpu: "1000m" - memory: "1Gi" + memory: "768Mi" podsMonitor: requests: cpu: "100m" - memory: "512Mi" + memory: "320Mi" limits: cpu: "500m" - memory: "512Mi" + memory: "320Mi" # requests-proxy serves the Service Bus / backend communication path and - # is mostly idle (a few req/min). 100m/256Mi requests with headroom for - # the occasional burst; revisit if heap profiling shows growth. Schema - # at values.schema.json#resources.requestsProxy validates overrides. + # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi, + # backend#2461) with the limits left at 1000m/512Mi for the occasional burst; + # revisit if heap profiling shows growth. Schema at + # values.schema.json#resources.requestsProxy validates overrides. requestsProxy: requests: - cpu: "100m" - memory: "256Mi" + cpu: "50m" + memory: "128Mi" limits: cpu: "1000m" memory: "512Mi" diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index de49f2e7..9c91ded8 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes = 5368709120 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both # installers' values must equal a fresh render of the chart in the same tree. -$script:TbCpFootprintMemBytes = 3288334336 -$script:TbCpFootprintCpuMilli = 900 +$script:TbCpFootprintMemBytes = 2449473536 +$script:TbCpFootprintCpuMilli = 750 # ── end generated footprint ───────────────────────────────────────────────── # Set by Get-TrainingResources when the machine is readable but below the diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 65c45a07..f4724966 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in # `make drift` (the required Source-of-truth drift job): the value below must # equal a fresh render of the chart in the same tree, or CI reddens. -_TB_CP_FOOTPRINT_MEM_BYTES=3288334336 -_TB_CP_FOOTPRINT_CPU_MILLI=900 +_TB_CP_FOOTPRINT_MEM_BYTES=2449473536 +_TB_CP_FOOTPRINT_CPU_MILLI=750 # ── end generated ─────────────────────────────────────────────────────────── # ── the fallback training envelope (precedence step 4) ────────────────────── diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 9f3572bc..473524e5 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gp d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea scripts/lib/setup-linux.sh 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c scripts/lib/cluster.sh 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62 scripts/lib/gpu-plugins.sh -320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a scripts/lib/install-client-helm.sh +45ad4bf0734ee3c7e71728c1e67caddc999f40caa647997a8fe1aaec4c967328 scripts/lib/install-client-helm.sh 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943 scripts/lib/assess.sh e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b scripts/lib/probe.sh 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd scripts/lib/summary.sh 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e scripts/lib/diagnose.sh -b2a223a1e819e7056f27707e9e5bd25c6c4bdd830212806240a2559e71f1761d scripts/install-k8s.ps1 +7e4fe218cc8373c7798565a761e04bf7ab132f1a5d36ac556b570c4d9bc035db scripts/install-k8s.ps1 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5 scripts/lib/telemetry.ps1 diff --git a/scripts/tests/control-plane-footprint.sh b/scripts/tests/control-plane-footprint.sh index a82ecfa3..2c79e10e 100755 --- a/scripts/tests/control-plane-footprint.sh +++ b/scripts/tests/control-plane-footprint.sh @@ -15,10 +15,11 @@ # footprint`, `control plane requests`: zero hits. The number was invisible. # # Measured here by rendering the chart: the steady-state control plane requests -# ~3136 MiB, already ABOVE the 3 GiB (3072 MiB) the envelope reserves for it. +# ~3136 MiB before backend#2461's interim trim (2026-09-10) -- ABOVE the 3 GiB +# (3072 MiB) the envelope reserved for it; 2336 MiB / 750 m after, under it. # That 64 MiB overshoot is the memory half of the reason a training pod on a # freshly-installed single-node edge can sit `Pending / Insufficient memory` -# (backend#2870). CPU fits: 900m requested against a 1000m reserve. +# (backend#2870). CPU: 900m before the trim, 750m after, against a 1000m reserve. # # WHAT THIS GUARD DOES, AND DELIBERATELY DOES NOT # ----------------------------------------------- @@ -70,8 +71,8 @@ installer="$root/scripts/lib/install-client-helm.sh" # same PR -- which is the moment to weigh whether the training envelope can still # afford it. Overridable so the guard's own test can drive a lower ceiling and # watch a real render breach it. -MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-3136}" -CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-900}" +MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2336}" +CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-750}" command -v helm >/dev/null 2>&1 || { echo "[ERROR] helm is required to render the chart footprint" >&2; exit 3; } command -v python3 >/dev/null 2>&1 || { echo "[ERROR] python3 is required to sum the rendered requests" >&2; exit 3; } @@ -169,7 +170,7 @@ try: # and cpu, so a pod can take its memory from the init side and its cpu # from the app side. # - # THE NUMBER DOES NOT MOVE ON THIS CHART -- 3136 MiB / 900 m either way, + # THE NUMBER DOES NOT MOVE ON THIS CHART -- 2336 MiB / 750 m either way, # because these init containers carry no requests. So the "64 MiB OVER" # finding stands; the method was wrong, the conclusion was not. It also # means the fix is INERT on the real render, which is why the bats diff --git a/scripts/tests/envelope-schedulability-mutations.sh b/scripts/tests/envelope-schedulability-mutations.sh index 89abee65..1012af72 100644 --- a/scripts/tests/envelope-schedulability-mutations.sh +++ b/scripts/tests/envelope-schedulability-mutations.sh @@ -110,7 +110,7 @@ run_case "unreadable cluster writes a carried non-floor size" "$LIB" \ printf '\nthe embed:\n' run_case "embedded footprint drifts from the render" "$LIB" \ - '_TB_CP_FOOTPRINT_MEM_BYTES=3288334336' \ + '_TB_CP_FOOTPRINT_MEM_BYTES=2449473536' \ '_TB_CP_FOOTPRINT_MEM_BYTES=3221225472' 1 printf '\n%s passed, %s failed\n' "$pass" "$fail" diff --git a/scripts/tests/envelope-schedulability.bats b/scripts/tests/envelope-schedulability.bats index 25e1e706..e65449b9 100644 --- a/scripts/tests/envelope-schedulability.bats +++ b/scripts/tests/envelope-schedulability.bats @@ -171,9 +171,12 @@ YAML # It walked the golden vectors -- a guard that passes having checked nothing is # the class this repo catalogues. The table has 13 rows today; assert a floor, # not the exact count, so a vector added upstream does not redden this. - [[ "$output" =~ ok\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } - [ "${BASH_REMATCH[2]}" -ge 10 ] || { echo "only ${BASH_REMATCH[2]} measurable vectors"; return 1; } - [ "${BASH_REMATCH[1]}" -ge 1 ] || return 1 + # Two legitimate shapes: `ok positive control: N/M` while the platform out-requests + # the reserve, or `note positive control: N/M ... does not apply` once it fits + # inside it (backend#2461). Either way the guard must have walked the table. + [[ "$output" =~ (ok|note)\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } + [ "${BASH_REMATCH[3]}" -ge 10 ] || { echo "only ${BASH_REMATCH[3]} measurable vectors"; return 1; } + if [[ "${BASH_REMATCH[1]}" == "ok" ]]; then [ "${BASH_REMATCH[2]}" -ge 1 ] || return 1; fi } @test "envelope-schedulability.sh: CANNOT TELL (exit 2) when the tree has no chart to derive from" { @@ -360,10 +363,11 @@ for key, kind in (("cpu_to_milli", "cpu"), ("mem_to_bytes", "mem")): } helm() { return 1; } unset TRACEBLOC_TRAINING_RESOURCES - # ONE system pod here (70Mi / 100m), so cpu fits (3000 + 900 + 100 = 4000) and - # only memory reduces (5120 + 3136 + 70 > 8192 -> 4 GiB): cpu=3,memory=4Gi. - # Anything else in the capture -- a stray echo -- would corrupt the value. + # ONE system pod here (70Mi / 100m). Since backend#2461's trim the resolver's + # cpu=3,memory=5Gi fits (5120 + 2336 + 70 <= 8192; 3000 + 750 + 100 <= 4000), so + # the fit writes it unchanged. Anything else in the capture -- a stray echo -- + # would corrupt the value, which is what this test is for. local captured captured="$(_resolve_training_size; _fit_training_envelope; printf '%s' "$_TB_TRAINING_SIZE")" - [ "$captured" = "cpu=3,memory=4Gi" ] || { echo "captured '$captured'"; return 1; } + [ "$captured" = "cpu=3,memory=5Gi" ] || { echo "captured '$captured'"; return 1; } } diff --git a/scripts/tests/envelope-schedulability.sh b/scripts/tests/envelope-schedulability.sh index 1e552818..300b3a75 100644 --- a/scripts/tests/envelope-schedulability.sh +++ b/scripts/tests/envelope-schedulability.sh @@ -7,11 +7,15 @@ # --------------- # backend#2870: the installer sized the training envelope as `allocatable - 3 GiB` # and nothing then asked whether that number could be scheduled beside what the -# chart installs. It could not: the control plane requests 3136 MiB (from the -# render) against the 3072 MiB reserve, so every install over-asked and the +# chart installs. It could not: the control plane then requested 3136 MiB (from +# the render) against the 3072 MiB reserve, so every install over-asked and the # training pod sat Pending. No test in this repo could see it, because every # test compared the installers to the CONTRACT and none compared the arithmetic -# to a CLUSTER. +# to a CLUSTER. backend#2461's interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve -- so the over-ask no longer occurs on a +# real node and the positive control below reports that instead of failing; +# the reduce/refuse paths are exercised on nodes DERIVED from the footprint +# (SMALL_MI / TINY_MI), never a written-down machine size. # # WHAT IT ASSERTS, and how it derives rather than restates (backend#1729 rule 1) # ---------------------------------------------------------------------------- @@ -190,32 +194,56 @@ if (( NEED_B > _TB_ENVELOPE_OVERHEAD_MEM_BYTES || NEED_M > _TB_ENVELOPE_OVERHEAD bad "positive control: the platform out-requests the reserve, yet no vector over-asked before the fit -- this guard cannot see the defect it exists for" fi else - echo " note the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve; the fit is a no-op by construction and the positive control does not apply" + echo " note positive control: ${over_before}/${checked} measurable vectors over-asked -- the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve, so the fit is a no-op by construction and the control does not apply (backend#2461)" fi -# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. +# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. Before backend#2461 it over-asked +# (5120 + 3136 + 140 > 8192) and was REDUCED; after the trim it fits unreduced. +# Pinning the fit is the regression guard: a chart change that pushes the +# footprint back over the reserve reddens here first. resolve_and_fit "4 8Gi" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]]; then - ok "8 GiB reproduction: ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" +if [[ "$_TB_FIT_VERDICT" == "fits" && "$_TB_TRAINING_SIZE" == "$BEFORE" ]]; then + ok "8 GiB reproduction: ${BEFORE} fits unreduced beside $(( NEED_B / MIB )) MiB / ${NEED_M} m (backend#2461 closed the over-ask)" +else + bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want fits, unreduced)" +fi + +# 3b. THE REDUCE PATH still has to be exercised, so the node is DERIVED from the +# footprint: below 4 GiB the resolver falls to the 1-core / 2-GiB floor, and +# SMALL_MI leaves room for 1 GiB but not for that floor, so the fit must +# reduce to 1 GiB with the arithmetic on screen. Moves with the render. +SMALL_MI=$(( NEED_B / MIB + 1024 + 64 )) +resolve_and_fit "4 ${SMALL_MI}Mi" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]] && [[ "$_TB_TRAINING_SIZE" == "cpu=1,memory=1Gi" ]]; then + ok "${SMALL_MI} MiB node (derived): ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" printf '%s\n' "$_TB_FIT_LINES" | sed 's/^/ /' else - bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" + bad "${SMALL_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want reduced to cpu=1,memory=1Gi)" fi -# 4. REFUSAL when not even 1 core / 1 GiB fits. -resolve_and_fit "4 4Gi" +# 4. REFUSAL when not even 1 core / 1 GiB fits -- TINY_MI is derived so that +# allocatable minus the platform is 64 MiB short of a 1 GiB run. +TINY_MI=$(( NEED_B / MIB + 1024 - 64 )) +resolve_and_fit "4 ${TINY_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "refused" && "$_TB_FIT_LINES" == *"not even a 1-core / 1-GiB run"* ]]; then - ok "4 GiB node: refused, with the arithmetic" + ok "${TINY_MI} MiB node (derived): refused, with the arithmetic" else - bad "4 GiB node: verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" + bad "${TINY_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" fi -# 5. CPU-ONLY overshoot: memory already fits (the GiB floor absorbed it), cpu does not. +# 5. CPU-ONLY overshoot: memory already fits, cpu does not. The resolver leaves +# one core for the platform and the trimmed platform needs 950 m, so no real +# node can produce this any more -- the pre-trim footprint could. The case +# keeps DoD part 5 covered by REPLAYING that footprint (900 m, the value the +# embed carried before backend#2461) through the real fit; the memory side is +# untouched so only cpu can move. +saved_cpu="$_TB_CP_FOOTPRINT_CPU_MILLI"; _TB_CP_FOOTPRINT_CPU_MILLI=900 resolve_and_fit "8 63928Mi" +_TB_CP_FOOTPRINT_CPU_MILLI="$saved_cpu" a_mem="$(env_mem_b "$_TB_TRAINING_SIZE")"; b_mem="$(env_mem_b "$BEFORE")" a_cpu="$(env_cpu_m "$_TB_TRAINING_SIZE")"; b_cpu="$(env_cpu_m "$BEFORE")" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + NEED_M <= 8000 )); then - ok "cpu-only overshoot: ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + 900 + SYS_M <= 8000 )); then + ok "cpu-only overshoot (pre-trim 900 m replayed): ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" else bad "cpu-only overshoot: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" fi @@ -237,9 +265,10 @@ else bad "measured system pods: $(( ${_TB_SYS_MEM_BYTES:-0} / MIB )) MiB / ${_TB_SYS_CPU_MILLI:-0} m (want ${SYS_MIB} / ${SYS_M}); note: ${_TB_SYS_NOTE:-}" fi -# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces. +# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces on the +# derived SMALL_MI node (the 8 GiB machine no longer needs reducing). PODS_READABLE=0 -resolve_and_fit "4 8Gi" +resolve_and_fit "4 ${SMALL_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "reduced" && "$_TB_FIT_LINES" == *"NOT measured"*"chart derivation only"* ]]; then ok "pod list unreadable: verified against the chart derivation only, and said so (${BEFORE} -> ${_TB_TRAINING_SIZE})" else diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index 0e3a7289..33801007 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -1378,12 +1378,16 @@ _override_release_ctx() { # ── the envelope is checked against what the node can actually schedule (backend#2870) ── # -# The resolver sizes against ALLOCATABLE. The chart's own control plane requests -# 3136 MiB / 900 m (from the render) and k3s's system pods ~140 MiB / 200 m, so -# `allocatable - 3 GiB` over-asked on every machine and the training pod sat -# Pending. These three drive the WHOLE install flow so the written values file -# is what is asserted -- the unit-level arithmetic lives in -# scripts/tests/envelope-schedulability.sh and .bats. +# The resolver sizes against ALLOCATABLE. Before backend#2461 the chart's own +# control plane requested 3136 MiB / 900 m (from the render) and k3s's system +# pods ~140 MiB / 200 m, so `allocatable - 3 GiB` over-asked on every machine and +# the training pod sat Pending. The interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve, so an 8 GiB node now fits unreduced -- and +# the reduce/refuse paths are exercised on nodes DERIVED from the embedded +# footprint (`_TB_CP_FOOTPRINT_MEM_BYTES`, sourced with the library), so these +# tests follow the number instead of restating it. These drive the WHOLE install +# flow so the written values file is what is asserted -- the unit-level +# arithmetic lives in scripts/tests/envelope-schedulability.sh and .bats. _sched_cluster() { # $1 = node line(s) kubectl() { case "$*" in @@ -1395,8 +1399,10 @@ _sched_cluster() { # $1 = node line(s) } _SCHED_NODES="$1" } +# platform = embedded chart footprint + the two canned system pods (140 MiB / 200 m) +_sched_need_mib() { echo $(( _TB_CP_FOOTPRINT_MEM_BYTES / 1048576 + 140 )); } -@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { +@test "install_client_helm: the ticket's 8 GiB node now FITS unreduced (backend#2870 closed by backend#2461)" { HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" _ensure_tracebloc_dirs() { :; } _ensure_release_dirs() { :; } @@ -1405,17 +1411,38 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # The ticket's reproduction: an 8 GiB node. The resolver says cpu=3,memory=5Gi; - # 5120 + 3136 + 140 = 8396 MiB > 8192, and 3000 + 900 + 200 = 4100 m > 4000. + # 5120 + platform <= 8192 MiB and 3000 + 950 <= 4000 m: the resolver's + # cpu=3,memory=5Gi is written as-is, and nothing is reported OVER. _sched_cluster '4 8Gi' run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] || { echo "$output"; return 1; } - grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } - grep -q 'RESOURCE_LIMITS: "memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 + grep -q 'RESOURCE_REQUESTS: "cpu=3,memory=5Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + [[ "$output" != *"OVER"* ]] || { echo "$output"; return 1; } +} + +@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + has() { return 0; } + verify_credentials() { printf valid; } + unset TRACEBLOC_TRAINING_RESOURCES + # A node DERIVED from the footprint: below 4 GiB the resolver falls to the + # 1-core / 2-GiB floor; this node leaves room for 1 GiB but not 2, so the fit + # must reduce to memory=1Gi and print the OVER arithmetic. + local need small over + need="$(_sched_need_mib)"; small=$(( need + 1024 + 64 )); over=$(( 2048 + need - small )) + _sched_cluster "4 ${small}Mi" + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + grep -q 'RESOURCE_REQUESTS: "cpu=1,memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + grep -q 'RESOURCE_LIMITS: "memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 grep -q 'RESOURCE_PROVENANCE: "installer"' "$HOST_DATA_DIR/values.yaml" || return 1 # The arithmetic is on screen, not just the verdict. - [[ "$output" == *"204 MiB OVER"* ]] || { echo "$output"; return 1; } - [[ "$output" == *"reduced cpu=3,memory=5Gi -> cpu=2,memory=4Gi"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"${over} MiB OVER"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"reduced cpu=1,memory=2Gi -> cpu=1,memory=1Gi"* ]] || { echo "$output"; return 1; } } @test "install_client_helm: REFUSES to write an envelope when not even a 1-core/1-GiB run fits (backend#2870)" { @@ -1427,8 +1454,9 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # 4 GiB allocatable: 4096 - 3136 - 140 = 820 MiB is all that is left. - _sched_cluster '4 4Gi' + # Derived: allocatable minus the platform is 64 MiB short of a 1 GiB run. + local tiny; tiny=$(( $(_sched_need_mib) + 1024 - 64 )) + _sched_cluster "4 ${tiny}Mi" run install_client_helm <<< $'myid\nmypw' [ "$status" -ne 0 ] || { echo "$output"; return 1; } [[ "$output" == *"Refusing to write a training envelope"* ]] || { echo "$output"; return 1; } diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 50804852..0b146f13 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9788,33 +9788,52 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } } - It "3. the ticket's reproduction: an 8 GiB node is REDUCED, arithmetic printed" { + It "3. the ticket's reproduction: an 8 GiB node now FITS unreduced (backend#2461 closed the over-ask)" { + # Before the interim trim the platform asked 3136 MiB / 900 m and this node was + # REDUCED to cpu=2,memory=4Gi; at 2336 MiB / 750 m the resolver's cpu=3,memory=5Gi + # fits: 5120 + 2476 <= 8192 and 3000 + 950 <= 4000. Pinning the fit is the + # regression guard for a chart change that pushes the footprint back up. $r = Invoke-FitOn -Nodes @('4 8Gi') + $r.Fit.Verdict | Should -Be 'fits' + $r.Fit.Size | Should -Be $r.Before + } + It "3b. the REDUCE path, on a node DERIVED from the footprint (moves with the render)" { + # Below 4 GiB the resolver falls to the 1-core / 2-GiB floor; this node leaves + # room for 1 GiB but not for that floor, so the fit must reduce to 1 GiB with + # the OVER arithmetic on screen. + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*OVER*' ($r.Fit.Lines -join "`n") | Should -BeLike "*reduced $($r.Before) -> $($r.Fit.Size)*" - # The numbers on this node against the canned cluster: cpu 3000+900+200 = 4100 - # > 4000 -> 2 cores; memory 5120+3136+140 = 8396 > 8192 -> 8192-3276 = 4916 MiB - # -> 4 GiB. Both dimensions over-asked, both reduced. - $r.Fit.Size | Should -Be 'cpu=2,memory=4Gi' + $r.Fit.Size | Should -Be 'cpu=1,memory=1Gi' } - - It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic" { - $r = Invoke-FitOn -Nodes @('4 4Gi') + It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic (node derived)" { + # allocatable minus the platform is 64 MiB short of a 1 GiB run + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $tiny = $needMib + 1024 - 64 + $r = Invoke-FitOn -Nodes @("4 ${tiny}Mi") $r.Fit.Verdict | Should -Be 'refused' ($r.Fit.Lines -join "`n") | Should -BeLike '*not even a 1-core / 1-GiB run*' } - - It "5. cpu-only overshoot reduces cpu alone (memory kept)" { - $r = Invoke-FitOn -Nodes @('8 63928Mi') - $r.Fit.Verdict | Should -Be 'reduced' - (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) - $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) - $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) - $aCpu | Should -BeLessThan $bCpu - ($aCpu + $script:TbCpFootprintCpuMilli + $script:SysM) | Should -BeLessOrEqual 8000 + It "5. cpu-only overshoot reduces cpu alone (memory kept) -- the pre-trim 900 m replayed" { + # The resolver leaves one core for the platform and the trimmed platform needs + # 950 m, so no real node produces a cpu-only overshoot any more; the pre-trim + # footprint (900 m) did. Replaying it through the real fit keeps DoD part 5 + # covered; memory is untouched so only cpu can move. + $saved = $script:TbCpFootprintCpuMilli + try { + $script:TbCpFootprintCpuMilli = 900 + $r = Invoke-FitOn -Nodes @('8 63928Mi') + $r.Fit.Verdict | Should -Be 'reduced' + (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) + $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) + $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) + $aCpu | Should -BeLessThan $bCpu + ($aCpu + 900 + $script:SysM) | Should -BeLessOrEqual 8000 + } finally { $script:TbCpFootprintCpuMilli = $saved } } - It "6. a human's pin is warned, never altered (pinned-over)" { $r = Invoke-FitOn -Nodes @('4 8Gi') -Override 'cpu=4,memory=16Gi' $r.Prov | Should -Be 'user' @@ -9836,7 +9855,10 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "7b. pods unreadable: verified against the chart derivation only, and it SAYS so; still reduces" { - $r = Invoke-FitOn -Nodes @('4 8Gi') -PodsReadable $false + # on the derived small node -- the 8 GiB machine no longer needs reducing + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") -PodsReadable $false $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*NOT measured*chart derivation only*' } @@ -9867,19 +9889,22 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "the reduction below the contract floor flags Undersized, and a fit does not" { - # 4c/5Gi: the resolver leaves cpu=3,memory=2Gi (5120-3072 = 2048 MiB, the - # floor exactly). Memory: 2048+3136+140 > 5120 -> 5120-3276 = 1844 MiB -> 1 GiB, - # below the 2 GiB floor -> Undersized. Cpu: 4000-1100 = 2900 m -> 2 cores, so - # the run is still a requestable shape and is REDUCED, not refused. (On a - # 2-core node the same memory would reduce but cpu would leave 900 m -- under - # one core -- and the verdict is refused; that is case 4's territory.) - $r = Invoke-FitOn -Nodes @('4 5Gi') - $r.Fit.Verdict | Should -Be 'reduced' - $r.Fit.Size | Should -Be 'cpu=2,memory=1Gi' - $r.Fit.Undersized | Should -BeTrue - # A genuine fit: an installer-sized envelope ALWAYS over-asks today (the - # resolver leaves 3 GiB and the platform asks 3136 MiB plus system pods -- the - # positive control above), so use a carried installer size that fits. + # A reduction that lands BELOW the 2 GiB floor needs the resolver to start at + # or above the floor (node >= 5 GiB) AND the platform to push it under -- which + # the trimmed 2336 MiB no longer can (5120 - 2476 = 2644 MiB >= 2 GiB). Replay + # the pre-trim footprint (3136 MiB) through the real fit, as case 5 does for + # cpu: 4c/5Gi -> cpu=3,memory=2Gi; memory 2048+3136+140 > 5120 -> 5120-3276 = + # 1844 MiB -> 1 GiB, below the floor -> Undersized. Cpu: 4000-950 = 3050 m -> + # 3 cores, still requestable, so REDUCED, not refused. + $saved = $script:TbCpFootprintMemBytes + try { + $script:TbCpFootprintMemBytes = [long](3136 * 1MB) + $r = Invoke-FitOn -Nodes @('4 5Gi') + $r.Fit.Verdict | Should -Be 'reduced' + $r.Fit.Size | Should -Be 'cpu=3,memory=1Gi' + $r.Fit.Undersized | Should -BeTrue + } finally { $script:TbCpFootprintMemBytes = $saved } + # A genuine fit, on the real footprint: a carried installer size that fits. $r2 = Invoke-FitOn -Nodes @('16 64Gi') -Carried @{ Size = 'cpu=4,memory=12Gi'; Provenance = 'installer' } $r2.Before | Should -Be 'cpu=4,memory=12Gi' $r2.Fit.Verdict | Should -Be 'fits' From 8f6012684224e4073d1ae261682686b633d2e32d Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 15:24:43 +0200 Subject: [PATCH 2/6] fix(chart): mysql requests.cpu template fallback follows the trimmed 150m (Bugbot on #1032) Co-Authored-By: Claude Fable 5.1 --- client/templates/mysql-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/templates/mysql-deployment.yaml b/client/templates/mysql-deployment.yaml index 9d29b991..4a2cbdc4 100644 --- a/client/templates/mysql-deployment.yaml +++ b/client/templates/mysql-deployment.yaml @@ -159,7 +159,7 @@ spec: resources: requests: memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }} - cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }} + cpu: {{ .Values.resources.mysql.requests.cpu | default "150m" | quote }} limits: memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }} {{- /* From b0aebbfd2fdd136d9ba42af844872c64426965c4 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 15:33:49 +0200 Subject: [PATCH 3/6] test(footprint): derive the ratchet test's ceiling from the render so it keeps reddening after the trim Co-Authored-By: Claude Fable 5.1 --- scripts/tests/control-plane-footprint.bats | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/tests/control-plane-footprint.bats b/scripts/tests/control-plane-footprint.bats index 50c61308..ccb6f0ce 100644 --- a/scripts/tests/control-plane-footprint.bats +++ b/scripts/tests/control-plane-footprint.bats @@ -75,7 +75,7 @@ YAML # guard hands to backend#2460/#2461. # # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the - # live render the number is 3136 MiB either way -- the chart's init containers + # live render the number is 2336 MiB either way (3136 before backend#2461) -- the chart's init containers # carry no requests -- so the fix is INERT there and a green run proves nothing # about the formula. Here the two answers differ, deliberately: # @@ -162,7 +162,13 @@ YAML } @test "guard: RATCHET reddens when the footprint would exceed the ceiling" { - TB_CP_FOOTPRINT_MEM_CEIL=3000 run bash "$GUARD" + # The ceiling is DERIVED from the live render (one MiB under it), so this test + # keeps reddening whatever the chart requests -- a written-down 3000 stopped + # breaching once backend#2461 trimmed the render from 3136 to 2336 MiB. + local mib + mib="$(bash "$GUARD" --print-footprint 2>/dev/null | cut -d' ' -f1)" + [[ "$mib" =~ ^[0-9]+$ ]] || { echo "could not derive the render footprint: '$mib'"; return 1; } + TB_CP_FOOTPRINT_MEM_CEIL=$(( mib - 1 )) run bash "$GUARD" [ "$status" -eq 1 ] || { echo "expected exit 1, got $status: $output"; return 1; } printf '%s\n' "$output" | grep -q 'exceed the recorded ceiling' || { echo "$output"; return 1; } } From fd3b8740cb89447aafb067a99edf6bc85a0bdab3 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 15:52:53 +0200 Subject: [PATCH 4/6] fix(chart): count the collector in the trim and keep the #1144 limits (review on #1032) Review found two holes in the interim trim (backend#2461): 1. The footprint the guard, the embed and the derivation carry INCLUDES the telemetry collector (helm template cannot look up the token Secret, so the DaemonSet renders). With it counted, 750m / 2336Mi left a 2-core / 8 GiB node at 950m / 3062Mi -- 50m and 10Mi short of the smallest envelope. Trim jobs-manager cpu 250m -> 150m (the whole plane idles at 36m) and re-cut the two memory rows below; the render is now 650m / 2272Mi and that node leaves 1050m / 3126Mi. The derivation on values.yaml now says the collector is counted, and why that is the right worst case. 2. pods-monitor was the only row cut without a measurement, and requests == limits made the cut a hard cgroup ceiling. Lower only the REQUESTS on jobs-manager (1Gi -> 640Mi) and pods-monitor (512Mi -> 384Mi) and leave both LIMITS where #1144 put them; the restart peak that justified the limits is exactly the load-run leg not yet measured. Template fallbacks and helm-unittest pins follow. Also fix the four comments the review flagged as stale on the pre-trim numbers (installer fit preamble, schedulability WHAT-IT-ASSERTS list, footprint bats fixture note, Pester positive-control note). Ceilings, embed, mutation needle and manifest regenerated from the render. --- client/templates/jobs-manager-deployment.yaml | 28 +++++---- client/tests/jobs_manager_test.yaml | 16 +++-- client/values.yaml | 63 ++++++++++++------- scripts/install-k8s.ps1 | 4 +- scripts/lib/install-client-helm.sh | 19 +++--- scripts/manifest.sha256 | 4 +- scripts/tests/control-plane-footprint.bats | 2 +- scripts/tests/control-plane-footprint.sh | 24 +++---- .../envelope-schedulability-mutations.sh | 2 +- scripts/tests/envelope-schedulability.sh | 4 +- scripts/tests/install-k8s.Tests.ps1 | 8 ++- 11 files changed, 105 insertions(+), 69 deletions(-) diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index b724836a..ac24ffcc 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -333,16 +333,19 @@ spec: # unresourced-init set, and neither is reachability. # `scripts/tests/qos-reachability.sh` asserts it now. # - # requests.cpu is - # also raised (100m -> 250m) so a mass restart doesn't throttle its - # startup to a crawl (the ~9m recovery seen on lukas-test). + # requests.cpu was raised (100m -> 250m) so a mass restart doesn't throttle + # its startup to a crawl (the ~9m recovery seen on lukas-test); the interim + # trim (backend#2461) brings it to 150m -- the whole plane idles at 36m -- + # and lowers requests.memory to 640Mi while the LIMIT stays 1Gi, so the + # #1144 equality is deliberately broken here: see the derivation on + # values.yaml#resources for why the request moved and the limit did not. resources: requests: - cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }} - memory: {{ .Values.resources.jobsManager.requests.memory | default "768Mi" | quote }} + cpu: {{ .Values.resources.jobsManager.requests.cpu | default "150m" | quote }} + memory: {{ .Values.resources.jobsManager.requests.memory | default "640Mi" | quote }} limits: cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }} - memory: {{ .Values.resources.jobsManager.limits.memory | default "768Mi" | quote }} + memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }} ports: # client-runtime#21: POST /internal/submit-ingestion-run. The # ingestor subchart's post-install hook hits this through the @@ -915,20 +918,23 @@ spec: capabilities: drop: ["ALL"] # #1144: same cluster-recovery treatment as the api container above — - # pods-monitor was also exit-137 killed in the mass restart. Pin + # pods-monitor was also exit-137 killed in the mass restart. #1144 pinned # requests.memory == limits.memory (512Mi) to lower its oom_score_adj # (a less-preferred node OOM victim; the pod stays Burstable — and cpu is # not the only reason: see the init-container note on the api container - # above, backend#2872), and raise + # above, backend#2872), and raised # requests.cpu (50m -> 100m) so its startup is not starved under - # contention. + # contention. The interim trim (backend#2461) lowers the REQUEST to 384Mi + # and keeps the 512Mi LIMIT: nothing has measured this container yet, so + # the ceiling that #1144's restart peak justified stays where it was + # (derivation on values.yaml#resources). resources: requests: cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }} - memory: {{ .Values.resources.podsMonitor.requests.memory | default "320Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.requests.memory | default "384Mi" | quote }} limits: cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }} - memory: {{ .Values.resources.podsMonitor.limits.memory | default "320Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }} volumeMounts: - name: logs-volume mountPath: "/data/logs" diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 11c9d12f..ef289d79 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -70,20 +70,24 @@ tests: path: spec.template.spec.containers[0].resources.requests - isNotEmpty: path: spec.template.spec.containers[0].resources.limits - # #1144: requests.memory pinned == limits.memory (1Gi api / 512Mi - # pods-monitor) to lower oom_score_adj during a mass restart. + # #1144 pinned requests.memory == limits.memory (1Gi api / 512Mi + # pods-monitor) to lower oom_score_adj during a mass restart. The interim + # trim (backend#2461, client#1032) lowers the REQUESTS (640Mi / 384Mi) and + # keeps the LIMITS (1Gi / 512Mi) -- the limit is the hard ceiling the + # restart peak justified and nothing has re-measured it; the request is + # what the scheduler and admission subtract. Derivation: values.yaml#resources. - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 768Mi + value: 640Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 768Mi + value: 1Gi - equal: path: spec.template.spec.containers[1].resources.requests.memory - value: 320Mi + value: 384Mi - equal: path: spec.template.spec.containers[1].resources.limits.memory - value: 320Mi + value: 512Mi # SELECTED BY NAME, NOT BY POSITION. This was `env[0]`, and `env[0]` was # CLIENT_ID only for as long as nothing was ever prepended — which stopped being diff --git a/client/values.yaml b/client/values.yaml index caefc445..4e62469c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -1363,27 +1363,42 @@ resources: # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest # (#1017) and since client-runtime#544 the admission subtracts the chart's # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the - # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests - # (800m / 3008Mi without the collector) and k3s's own coredns + metrics-server - # (200m / 140Mi), LEFT 900m / 2390Mi, and the smallest training envelope needs - # 1000m / 3072Mi. Every task on staging was refused NEVER_STEADY_STATE - # (e2e-test-agent fleet 34473120598, 2026-09-10). + # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests, + # minus k3s's own coredns + metrics-server (200m / 140Mi), left 900m / 2390Mi, + # and the smallest training envelope needs 1000m / 3072Mi. Every task on + # staging was refused NEVER_STEADY_STATE (e2e-test-agent fleet 34473120598). # - # The numbers below give back 150m / 800Mi so that node leaves 1050m / 3190Mi: - # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom, requests==limits kept - # jobs-manager 250m/1Gi -> 250m/768Mi 168Mi idle (0.16x); restart peak NOT yet measured -- 768 keeps 4.5x over idle, cpu untouched - # pods-monitor 100m/512Mi -> 100m/320Mi requests==limits kept (oom_score_adj, #1144) - # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) so bursts still fit - # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) -- it was the one under-provisioned component + # THE FOOTPRINT COUNTS THE COLLECTOR. `helm template` cannot look up the token + # Secret, so the render -- and therefore control-plane-footprint.sh, the + # installer embed and this derivation -- carries the telemetry collector's + # 100m / 128Mi. That is the right worst case: on a real edge the collector IS + # rendered once jobs-manager has written the token Secret (backend#2274), so an + # edge that trains on install must still train after that. With it counted the + # render is 650m / 2272Mi, and the node above leaves 1050m / 3126Mi (was + # 900m / 2390Mi): 50m and 54Mi over the envelope. Thin, and honest. # - # What this is NOT: the final trim. backend#2461 sizes requests from p95 under a - # load run (ambient, one cycle, five-queued burst, node restart under 8 GiB) that - # is in progress; jobs-manager's request in particular is justified by the - # restart peak and moves only once that leg is measured. Re-derive from the - # measurement; do not hand-tune here. The guards that read these values: - # scripts/tests/control-plane-footprint.sh (ceiling), the generated embed in - # scripts/lib/install-client-helm.sh (scripts/gen-footprint-embed.sh --check), - # scripts/tests/envelope-schedulability.sh (fit on the golden vectors). + # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom; requests==limits kept + # jobs-manager 250m/1Gi -> 150m/640Mi 168Mi idle (0.16x); LIMIT STAYS 1Gi (see below); cpu: the whole plane idles at 36m + # pods-monitor 100m/512Mi -> 100m/384Mi no measurement yet; LIMIT STAYS 512Mi (see below) + # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) + # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) + # collector 100m/128Mi unchanged unmeasured (distroless, no readable cgroup) -- not trimmed on a guess + # + # WHY jobs-manager AND pods-monitor NOW HAVE requests < limits. #1144 pinned + # requests.memory == limits.memory on both so a cluster-recovery mass restart + # would not pick them as OOM victims; the PODS were Burstable then and still + # are (the unresourced init containers, see above). The trim lowers only the + # REQUEST -- what the scheduler and admission subtract -- and leaves the LIMIT + # where it was, because the limit is the hard cgroup ceiling and the restart + # peak that justifies it is exactly the leg backend#2461's load run has not + # produced yet (review on client#1032). The cost is a higher oom_score_adj + # under node pressure; the alternative was a hard ceiling nothing has measured. + # When the restart leg lands, set BOTH from its p95 and restore the equality. + # + # What this is NOT: the final trim. Re-derive from the load run; do not + # hand-tune here. Guards that read these values: control-plane-footprint.sh + # (ceiling), the generated embed in scripts/lib/install-client-helm.sh + # (scripts/gen-footprint-embed.sh --check), envelope-schedulability.sh. mysql: requests: cpu: "150m" @@ -1393,18 +1408,18 @@ resources: # No cpu limit — see comment above. jobsManager: requests: - cpu: "250m" - memory: "768Mi" + cpu: "150m" + memory: "640Mi" limits: cpu: "1000m" - memory: "768Mi" + memory: "1Gi" podsMonitor: requests: cpu: "100m" - memory: "320Mi" + memory: "384Mi" limits: cpu: "500m" - memory: "320Mi" + memory: "512Mi" # requests-proxy serves the Service Bus / backend communication path and # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi, # backend#2461) with the limits left at 1000m/512Mi for the occasional burst; diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index 9c91ded8..b37b578c 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes = 5368709120 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both # installers' values must equal a fresh render of the chart in the same tree. -$script:TbCpFootprintMemBytes = 2449473536 -$script:TbCpFootprintCpuMilli = 750 +$script:TbCpFootprintMemBytes = 2382364672 +$script:TbCpFootprintCpuMilli = 650 # ── end generated footprint ───────────────────────────────────────────────── # Set by Get-TrainingResources when the machine is readable but below the diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index f4724966..5e55501b 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in # `make drift` (the required Source-of-truth drift job): the value below must # equal a fresh render of the chart in the same tree, or CI reddens. -_TB_CP_FOOTPRINT_MEM_BYTES=2449473536 -_TB_CP_FOOTPRINT_CPU_MILLI=750 +_TB_CP_FOOTPRINT_MEM_BYTES=2382364672 +_TB_CP_FOOTPRINT_CPU_MILLI=650 # ── end generated ─────────────────────────────────────────────────────────── # ── the fallback training envelope (precedence step 4) ────────────────────── @@ -724,12 +724,17 @@ _training_provenance() { # # Everything above sizes the envelope against ALLOCATABLE and stops. Nothing then # asked whether the number written can be scheduled beside what else runs on the -# node: the chart's own control plane (3136 MiB / 900 m, from the render) plus -# the distribution's system pods (~140 MiB / 200 m of coredns and metrics-server -# on k3s). Measured on a real single-node install, the platform out-requests the -# 3 GiB reserve the envelope subtracts, so `allocatable - 3 GiB` over-asked at -# EVERY machine size and the training pod sat `Pending / Insufficient memory` +# node: the chart's own control plane (the embedded _TB_CP_FOOTPRINT_* constants, +# generated from the render by scripts/gen-footprint-embed.sh) plus the +# distribution's system pods (~140 MiB / 200 m of coredns and metrics-server on +# k3s). When this was written the chart requested 3136 MiB / 900 m -- MORE than +# the 3 GiB reserve the envelope subtracts -- so `allocatable - 3 GiB` over-asked +# at EVERY machine size and the training pod sat `Pending / Insufficient memory` # while the installer printed `Training size: ...` and cli doctor said OK. +# backend#2461's interim trim (chart 1.9.112) brought the control plane under the +# reserve, so today the fit below is a no-op on every golden vector; it stays +# because the constants are generated, not promised, and the next chart that +# out-requests the reserve should be reduced here rather than sit Pending again. # # The two functions below make the installer own that question at the moment it # writes the envelope. The 3 GiB constant itself is not touched here: fixing the diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 473524e5..22c38328 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gp d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea scripts/lib/setup-linux.sh 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c scripts/lib/cluster.sh 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62 scripts/lib/gpu-plugins.sh -45ad4bf0734ee3c7e71728c1e67caddc999f40caa647997a8fe1aaec4c967328 scripts/lib/install-client-helm.sh +9a473b2d11545183be0bb24a1446eb8a51bd6a525d5d08dcd09a574f3bd10e06 scripts/lib/install-client-helm.sh 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943 scripts/lib/assess.sh e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b scripts/lib/probe.sh 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd scripts/lib/summary.sh 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e scripts/lib/diagnose.sh -7e4fe218cc8373c7798565a761e04bf7ab132f1a5d36ac556b570c4d9bc035db scripts/install-k8s.ps1 +07722bab739746746f85504867ba3d32d85f9bc39f318b0789880bb7d6c13548 scripts/install-k8s.ps1 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5 scripts/lib/telemetry.ps1 diff --git a/scripts/tests/control-plane-footprint.bats b/scripts/tests/control-plane-footprint.bats index ccb6f0ce..f65020ee 100644 --- a/scripts/tests/control-plane-footprint.bats +++ b/scripts/tests/control-plane-footprint.bats @@ -75,7 +75,7 @@ YAML # guard hands to backend#2460/#2461. # # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the - # live render the number is 2336 MiB either way (3136 before backend#2461) -- the chart's init containers + # live render the number is the same either way -- the chart's init containers # carry no requests -- so the fix is INERT there and a green run proves nothing # about the formula. Here the two answers differ, deliberately: # diff --git a/scripts/tests/control-plane-footprint.sh b/scripts/tests/control-plane-footprint.sh index 2c79e10e..037b9f4e 100755 --- a/scripts/tests/control-plane-footprint.sh +++ b/scripts/tests/control-plane-footprint.sh @@ -14,12 +14,14 @@ # constant still covers it. Grep the three consumers for `3008`, `platform # footprint`, `control plane requests`: zero hits. The number was invisible. # -# Measured here by rendering the chart: the steady-state control plane requests -# ~3136 MiB before backend#2461's interim trim (2026-09-10) -- ABOVE the 3 GiB -# (3072 MiB) the envelope reserved for it; 2336 MiB / 750 m after, under it. -# That 64 MiB overshoot is the memory half of the reason a training pod on a -# freshly-installed single-node edge can sit `Pending / Insufficient memory` -# (backend#2870). CPU: 900m before the trim, 750m after, against a 1000m reserve. +# Measured here by rendering the chart: the steady-state control plane requested +# ~3136 MiB / 900 m before backend#2461's interim trim (2026-09-10) -- ABOVE the +# 3 GiB (3072 MiB) the envelope reserved for it; 2272 MiB / 650 m after, under it. +# That 64 MiB overshoot was the memory half of the reason a training pod on a +# freshly-installed single-node edge could sit `Pending / Insufficient memory` +# (backend#2870). The render COUNTS THE TELEMETRY COLLECTOR (helm template cannot +# look up the token Secret, so the collector renders): that is the worst case a +# real edge reaches once the Secret exists, and the one the ceiling must hold. # # WHAT THIS GUARD DOES, AND DELIBERATELY DOES NOT # ----------------------------------------------- @@ -71,8 +73,8 @@ installer="$root/scripts/lib/install-client-helm.sh" # same PR -- which is the moment to weigh whether the training envelope can still # afford it. Overridable so the guard's own test can drive a lower ceiling and # watch a real render breach it. -MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2336}" -CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-750}" +MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2272}" +CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-650}" command -v helm >/dev/null 2>&1 || { echo "[ERROR] helm is required to render the chart footprint" >&2; exit 3; } command -v python3 >/dev/null 2>&1 || { echo "[ERROR] python3 is required to sum the rendered requests" >&2; exit 3; } @@ -170,9 +172,9 @@ try: # and cpu, so a pod can take its memory from the init side and its cpu # from the app side. # - # THE NUMBER DOES NOT MOVE ON THIS CHART -- 2336 MiB / 750 m either way, - # because these init containers carry no requests. So the "64 MiB OVER" - # finding stands; the method was wrong, the conclusion was not. It also + # THE NUMBER DOES NOT MOVE ON THIS CHART -- the render is the same + # either way, because these init containers carry no requests. So the + # original "64 MiB OVER" finding stood; the method was wrong, the conclusion was not. It also # means the fix is INERT on the real render, which is why the bats # fixture is built so the two formulas disagree. def _req(c, key): diff --git a/scripts/tests/envelope-schedulability-mutations.sh b/scripts/tests/envelope-schedulability-mutations.sh index 1012af72..09600db6 100644 --- a/scripts/tests/envelope-schedulability-mutations.sh +++ b/scripts/tests/envelope-schedulability-mutations.sh @@ -110,7 +110,7 @@ run_case "unreadable cluster writes a carried non-floor size" "$LIB" \ printf '\nthe embed:\n' run_case "embedded footprint drifts from the render" "$LIB" \ - '_TB_CP_FOOTPRINT_MEM_BYTES=2449473536' \ + '_TB_CP_FOOTPRINT_MEM_BYTES=2382364672' \ '_TB_CP_FOOTPRINT_MEM_BYTES=3221225472' 1 printf '\n%s passed, %s failed\n' "$pass" "$fail" diff --git a/scripts/tests/envelope-schedulability.sh b/scripts/tests/envelope-schedulability.sh index 300b3a75..d22e04c1 100644 --- a/scripts/tests/envelope-schedulability.sh +++ b/scripts/tests/envelope-schedulability.sh @@ -27,7 +27,9 @@ # 1. the installer's embedded footprint constants equal that render; # 2. for EVERY single-node golden vector in the contract table, the envelope # written + chart footprint + system pods <= allocatable, on memory AND cpu; -# 3. the ticket's 8 GiB reproduction is REDUCED, with the arithmetic; +# 3. the ticket's 8 GiB reproduction now FITS unreduced (backend#2461 brought +# the footprint under the reserve), and a node sized from the live +# footprint to over-ask is REDUCED, with the arithmetic (3b); # 4. a machine where not even a 1-core / 1-GiB run fits is REFUSED; # 5. a cpu-only overshoot reduces cpu alone (DoD part 5: cover cpu too); # 6. a human's pin is never altered, only warned; diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 0b146f13..bb7b9707 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9780,9 +9780,11 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } $failures -join "`n" | Should -BeNullOrEmpty $checked | Should -BeGreaterOrEqual 10 -Because "a guard that checked almost nothing proves almost nothing" - # POSITIVE CONTROL: the platform out-requests the reserve today, so at least - # one vector must over-ask BEFORE the fit -- else this suite could never have - # seen the defect it exists for. + # POSITIVE CONTROL: while the platform out-requested the reserve (before + # backend#2461's trim) at least one vector had to over-ask BEFORE the fit -- + # else this suite could never have seen the defect it exists for. The guard is + # conditional on the embedded footprint, so it re-arms by itself the day a + # chart out-requests the reserve again. if ($needB -gt $script:TbEnvelopeOverheadMemBytes -or $needM -gt $script:TbEnvelopeOverheadCpuMilli) { $overBefore | Should -BeGreaterThan 0 -Because "the platform out-requests the reserve, yet no vector over-asked before the fit" } From 4fa0d7d956d88abbdd6ccba96d2a917835daa544 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 16:21:44 +0200 Subject: [PATCH 5/6] test(pester): keep the REDUCE case's claim phrase so mutation-check attributes both REDUCE mutations to it mutation-check.ps1 expects 'the chart footprint drops out of the sum' and 'the reduction is computed but the original size is written' to redden the test named 'is REDUCED, arithmetic printed'. The trim renamed that case to 3b (node derived from the footprint) and dropped the phrase, so both mutations reddened the suite but read MISATTRIBUTED. Put the phrase back in the 3b name; the body is unchanged. --- scripts/tests/install-k8s.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index bb7b9707..14c131c1 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9799,7 +9799,7 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client $r.Fit.Verdict | Should -Be 'fits' $r.Fit.Size | Should -Be $r.Before } - It "3b. the REDUCE path, on a node DERIVED from the footprint (moves with the render)" { + It "3b. a node DERIVED from the footprint (moves with the render) is REDUCED, arithmetic printed" { # Below 4 GiB the resolver falls to the 1-core / 2-GiB floor; this node leaves # room for 1 GiB but not for that floor, so the fit must reduce to 1 GiB with # the OVER arithmetic on screen. From ac90616575024d55aced723f87b7b0d34ba9e215 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 10 Sep 2026 16:23:25 +0200 Subject: [PATCH 6/6] test(pester): write the pre-trim footprint replay as an Int64 literal Bugbot flagged [long](3136 * 1MB) twice as an Int32 overflow. PowerShell promotes the overflowing product to Double rather than wrapping, so the finding was a false positive on both 5.1 and 7 -- but 3136L * 1MB is Int64 from the first operand and leaves nothing to argue about. --- scripts/tests/install-k8s.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 14c131c1..bfbd2aee 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9900,7 +9900,7 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client # 3 cores, still requestable, so REDUCED, not refused. $saved = $script:TbCpFootprintMemBytes try { - $script:TbCpFootprintMemBytes = [long](3136 * 1MB) + $script:TbCpFootprintMemBytes = 3136L * 1MB # Int64 literal: no Int32 product to argue about (Bugbot, twice) $r = Invoke-FitOn -Nodes @('4 5Gi') $r.Fit.Verdict | Should -Be 'reduced' $r.Fit.Size | Should -Be 'cpu=3,memory=1Gi'