diff --git a/client/Chart.yaml b/client/Chart.yaml index 15c1fd38..14a392bd 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.111 -appVersion: "1.9.111" +version: 1.9.112 +appVersion: "1.9.112" keywords: - tracebloc - kubernetes diff --git a/client/templates/egress-proxy-deployment.yaml b/client/templates/egress-proxy-deployment.yaml index 792d96ab..2080b26d 100644 --- a/client/templates/egress-proxy-deployment.yaml +++ b/client/templates/egress-proxy-deployment.yaml @@ -66,7 +66,7 @@ spec: resources: requests: cpu: {{ $epReq.cpu | default "50m" | quote }} - memory: {{ $epReq.memory | default "64Mi" | quote }} + memory: {{ $epReq.memory | default "160Mi" | quote }} limits: cpu: {{ $epLim.cpu | default "500m" | quote }} memory: {{ $epLim.memory | default "256Mi" | quote }} diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 19543b4a..ac24ffcc 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -333,13 +333,16 @@ spec: # unresourced-init set, and neither is reachability. # `scripts/tests/qos-reachability.sh` asserts it now. # - # requests.cpu is - # also raised (100m -> 250m) so a mass restart doesn't throttle its - # startup to a crawl (the ~9m recovery seen on lukas-test). + # requests.cpu was raised (100m -> 250m) so a mass restart doesn't throttle + # its startup to a crawl (the ~9m recovery seen on lukas-test); the interim + # trim (backend#2461) brings it to 150m -- the whole plane idles at 36m -- + # and lowers requests.memory to 640Mi while the LIMIT stays 1Gi, so the + # #1144 equality is deliberately broken here: see the derivation on + # values.yaml#resources for why the request moved and the limit did not. resources: requests: - cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }} - memory: {{ .Values.resources.jobsManager.requests.memory | default "1Gi" | quote }} + cpu: {{ .Values.resources.jobsManager.requests.cpu | default "150m" | quote }} + memory: {{ .Values.resources.jobsManager.requests.memory | default "640Mi" | quote }} limits: cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }} memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }} @@ -915,17 +918,20 @@ spec: capabilities: drop: ["ALL"] # #1144: same cluster-recovery treatment as the api container above — - # pods-monitor was also exit-137 killed in the mass restart. Pin + # pods-monitor was also exit-137 killed in the mass restart. #1144 pinned # requests.memory == limits.memory (512Mi) to lower its oom_score_adj # (a less-preferred node OOM victim; the pod stays Burstable — and cpu is # not the only reason: see the init-container note on the api container - # above, backend#2872), and raise + # above, backend#2872), and raised # requests.cpu (50m -> 100m) so its startup is not starved under - # contention. + # contention. The interim trim (backend#2461) lowers the REQUEST to 384Mi + # and keeps the 512Mi LIMIT: nothing has measured this container yet, so + # the ceiling that #1144's restart peak justified stays where it was + # (derivation on values.yaml#resources). resources: requests: cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }} - memory: {{ .Values.resources.podsMonitor.requests.memory | default "512Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.requests.memory | default "384Mi" | quote }} limits: cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }} memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }} diff --git a/client/templates/mysql-deployment.yaml b/client/templates/mysql-deployment.yaml index b3f9dbf7..4a2cbdc4 100644 --- a/client/templates/mysql-deployment.yaml +++ b/client/templates/mysql-deployment.yaml @@ -158,10 +158,10 @@ spec: # PodDisruptionBudget cover the eviction angle instead. resources: requests: - memory: {{ .Values.resources.mysql.requests.memory | default "1Gi" | quote }} - cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }} + memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }} + cpu: {{ .Values.resources.mysql.requests.cpu | default "150m" | quote }} limits: - memory: {{ .Values.resources.mysql.limits.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }} {{- /* CPU limit is intentionally unset by default (see comment block above). Schema permits it, so honour an explicit override here diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index a0f94e60..1ef868ce 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -134,8 +134,8 @@ spec: {{- $rpLim := default dict $rp.limits }} resources: requests: - cpu: {{ $rpReq.cpu | default "100m" | quote }} - memory: {{ $rpReq.memory | default "256Mi" | quote }} + cpu: {{ $rpReq.cpu | default "50m" | quote }} + memory: {{ $rpReq.memory | default "128Mi" | quote }} limits: cpu: {{ $rpLim.cpu | default "1000m" | quote }} memory: {{ $rpLim.memory | default "512Mi" | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 964de569..ef289d79 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -70,17 +70,21 @@ tests: path: spec.template.spec.containers[0].resources.requests - isNotEmpty: path: spec.template.spec.containers[0].resources.limits - # #1144: requests.memory pinned == limits.memory (1Gi api / 512Mi - # pods-monitor) to lower oom_score_adj during a mass restart. + # #1144 pinned requests.memory == limits.memory (1Gi api / 512Mi + # pods-monitor) to lower oom_score_adj during a mass restart. The interim + # trim (backend#2461, client#1032) lowers the REQUESTS (640Mi / 384Mi) and + # keeps the LIMITS (1Gi / 512Mi) -- the limit is the hard ceiling the + # restart peak justified and nothing has re-measured it; the request is + # what the scheduler and admission subtract. Derivation: values.yaml#resources. - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 640Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory value: 1Gi - equal: path: spec.template.spec.containers[1].resources.requests.memory - value: 512Mi + value: 384Mi - equal: path: spec.template.spec.containers[1].resources.limits.memory value: 512Mi diff --git a/client/tests/mysql_test.yaml b/client/tests/mysql_test.yaml index e0d2ad7b..90808612 100644 --- a/client/tests/mysql_test.yaml +++ b/client/tests/mysql_test.yaml @@ -40,13 +40,13 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m + value: 150m # CPU limit deliberately unset — InnoDB lock-wait timeouts under throttling. - notExists: path: spec.template.spec.containers[0].resources.limits.cpu diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml index 19f66ea8..5161e1d0 100644 --- a/client/tests/requests_proxy_test.yaml +++ b/client/tests/requests_proxy_test.yaml @@ -127,10 +127,10 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 100m + value: 50m - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 256Mi + value: 128Mi - equal: path: spec.template.spec.containers[0].resources.limits.cpu value: 1000m diff --git a/client/values.yaml b/client/values.yaml index 6b22572d..4e62469c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -612,7 +612,7 @@ egressProxy: resources: requests: cpu: "50m" - memory: "64Mi" + memory: "160Mi" limits: cpu: "500m" memory: "256Mi" @@ -1355,35 +1355,80 @@ images: # a mass restart does not throttle their startup. Same requests.memory==limits # philosophy mysql already uses (mysql likewise omits its cpu limit). resources: + # INTERIM TRIM (backend#2461, 2026-09-10) -- read before "fixing" any number back up. + # + # Requests are what the scheduler and the jobs-manager's admission SUBTRACT from + # the node; they are not what the process uses. Measured on an installer edge + # (backend#2461): the control plane requested 1050m / 3276Mi while using + # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest + # (#1017) and since client-runtime#544 the admission subtracts the chart's + # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the + # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests, + # minus k3s's own coredns + metrics-server (200m / 140Mi), left 900m / 2390Mi, + # and the smallest training envelope needs 1000m / 3072Mi. Every task on + # staging was refused NEVER_STEADY_STATE (e2e-test-agent fleet 34473120598). + # + # THE FOOTPRINT COUNTS THE COLLECTOR. `helm template` cannot look up the token + # Secret, so the render -- and therefore control-plane-footprint.sh, the + # installer embed and this derivation -- carries the telemetry collector's + # 100m / 128Mi. That is the right worst case: on a real edge the collector IS + # rendered once jobs-manager has written the token Secret (backend#2274), so an + # edge that trains on install must still train after that. With it counted the + # render is 650m / 2272Mi, and the node above leaves 1050m / 3126Mi (was + # 900m / 2390Mi): 50m and 54Mi over the envelope. Thin, and honest. + # + # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom; requests==limits kept + # jobs-manager 250m/1Gi -> 150m/640Mi 168Mi idle (0.16x); LIMIT STAYS 1Gi (see below); cpu: the whole plane idles at 36m + # pods-monitor 100m/512Mi -> 100m/384Mi no measurement yet; LIMIT STAYS 512Mi (see below) + # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) + # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) + # collector 100m/128Mi unchanged unmeasured (distroless, no readable cgroup) -- not trimmed on a guess + # + # WHY jobs-manager AND pods-monitor NOW HAVE requests < limits. #1144 pinned + # requests.memory == limits.memory on both so a cluster-recovery mass restart + # would not pick them as OOM victims; the PODS were Burstable then and still + # are (the unresourced init containers, see above). The trim lowers only the + # REQUEST -- what the scheduler and admission subtract -- and leaves the LIMIT + # where it was, because the limit is the hard cgroup ceiling and the restart + # peak that justifies it is exactly the leg backend#2461's load run has not + # produced yet (review on client#1032). The cost is a higher oom_score_adj + # under node pressure; the alternative was a hard ceiling nothing has measured. + # When the restart leg lands, set BOTH from its p95 and restore the equality. + # + # What this is NOT: the final trim. Re-derive from the load run; do not + # hand-tune here. Guards that read these values: control-plane-footprint.sh + # (ceiling), the generated embed in scripts/lib/install-client-helm.sh + # (scripts/gen-footprint-embed.sh --check), envelope-schedulability.sh. mysql: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "704Mi" limits: - memory: "1Gi" + memory: "704Mi" # No cpu limit — see comment above. jobsManager: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "640Mi" limits: cpu: "1000m" memory: "1Gi" podsMonitor: requests: cpu: "100m" - memory: "512Mi" + memory: "384Mi" limits: cpu: "500m" memory: "512Mi" # requests-proxy serves the Service Bus / backend communication path and - # is mostly idle (a few req/min). 100m/256Mi requests with headroom for - # the occasional burst; revisit if heap profiling shows growth. Schema - # at values.schema.json#resources.requestsProxy validates overrides. + # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi, + # backend#2461) with the limits left at 1000m/512Mi for the occasional burst; + # revisit if heap profiling shows growth. Schema at + # values.schema.json#resources.requestsProxy validates overrides. requestsProxy: requests: - cpu: "100m" - memory: "256Mi" + cpu: "50m" + memory: "128Mi" limits: cpu: "1000m" memory: "512Mi" diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index de49f2e7..b37b578c 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes = 5368709120 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both # installers' values must equal a fresh render of the chart in the same tree. -$script:TbCpFootprintMemBytes = 3288334336 -$script:TbCpFootprintCpuMilli = 900 +$script:TbCpFootprintMemBytes = 2382364672 +$script:TbCpFootprintCpuMilli = 650 # ── end generated footprint ───────────────────────────────────────────────── # Set by Get-TrainingResources when the machine is readable but below the diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 65c45a07..5e55501b 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in # `make drift` (the required Source-of-truth drift job): the value below must # equal a fresh render of the chart in the same tree, or CI reddens. -_TB_CP_FOOTPRINT_MEM_BYTES=3288334336 -_TB_CP_FOOTPRINT_CPU_MILLI=900 +_TB_CP_FOOTPRINT_MEM_BYTES=2382364672 +_TB_CP_FOOTPRINT_CPU_MILLI=650 # ── end generated ─────────────────────────────────────────────────────────── # ── the fallback training envelope (precedence step 4) ────────────────────── @@ -724,12 +724,17 @@ _training_provenance() { # # Everything above sizes the envelope against ALLOCATABLE and stops. Nothing then # asked whether the number written can be scheduled beside what else runs on the -# node: the chart's own control plane (3136 MiB / 900 m, from the render) plus -# the distribution's system pods (~140 MiB / 200 m of coredns and metrics-server -# on k3s). Measured on a real single-node install, the platform out-requests the -# 3 GiB reserve the envelope subtracts, so `allocatable - 3 GiB` over-asked at -# EVERY machine size and the training pod sat `Pending / Insufficient memory` +# node: the chart's own control plane (the embedded _TB_CP_FOOTPRINT_* constants, +# generated from the render by scripts/gen-footprint-embed.sh) plus the +# distribution's system pods (~140 MiB / 200 m of coredns and metrics-server on +# k3s). When this was written the chart requested 3136 MiB / 900 m -- MORE than +# the 3 GiB reserve the envelope subtracts -- so `allocatable - 3 GiB` over-asked +# at EVERY machine size and the training pod sat `Pending / Insufficient memory` # while the installer printed `Training size: ...` and cli doctor said OK. +# backend#2461's interim trim (chart 1.9.112) brought the control plane under the +# reserve, so today the fit below is a no-op on every golden vector; it stays +# because the constants are generated, not promised, and the next chart that +# out-requests the reserve should be reduced here rather than sit Pending again. # # The two functions below make the installer own that question at the moment it # writes the envelope. The 3 GiB constant itself is not touched here: fixing the diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 9f3572bc..22c38328 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gp d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea scripts/lib/setup-linux.sh 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c scripts/lib/cluster.sh 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62 scripts/lib/gpu-plugins.sh -320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a scripts/lib/install-client-helm.sh +9a473b2d11545183be0bb24a1446eb8a51bd6a525d5d08dcd09a574f3bd10e06 scripts/lib/install-client-helm.sh 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943 scripts/lib/assess.sh e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b scripts/lib/probe.sh 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd scripts/lib/summary.sh 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e scripts/lib/diagnose.sh -b2a223a1e819e7056f27707e9e5bd25c6c4bdd830212806240a2559e71f1761d scripts/install-k8s.ps1 +07722bab739746746f85504867ba3d32d85f9bc39f318b0789880bb7d6c13548 scripts/install-k8s.ps1 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5 scripts/lib/telemetry.ps1 diff --git a/scripts/tests/control-plane-footprint.bats b/scripts/tests/control-plane-footprint.bats index 50c61308..f65020ee 100644 --- a/scripts/tests/control-plane-footprint.bats +++ b/scripts/tests/control-plane-footprint.bats @@ -75,7 +75,7 @@ YAML # guard hands to backend#2460/#2461. # # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the - # live render the number is 3136 MiB either way -- the chart's init containers + # live render the number is the same either way -- the chart's init containers # carry no requests -- so the fix is INERT there and a green run proves nothing # about the formula. Here the two answers differ, deliberately: # @@ -162,7 +162,13 @@ YAML } @test "guard: RATCHET reddens when the footprint would exceed the ceiling" { - TB_CP_FOOTPRINT_MEM_CEIL=3000 run bash "$GUARD" + # The ceiling is DERIVED from the live render (one MiB under it), so this test + # keeps reddening whatever the chart requests -- a written-down 3000 stopped + # breaching once backend#2461 trimmed the render from 3136 to 2336 MiB. + local mib + mib="$(bash "$GUARD" --print-footprint 2>/dev/null | cut -d' ' -f1)" + [[ "$mib" =~ ^[0-9]+$ ]] || { echo "could not derive the render footprint: '$mib'"; return 1; } + TB_CP_FOOTPRINT_MEM_CEIL=$(( mib - 1 )) run bash "$GUARD" [ "$status" -eq 1 ] || { echo "expected exit 1, got $status: $output"; return 1; } printf '%s\n' "$output" | grep -q 'exceed the recorded ceiling' || { echo "$output"; return 1; } } diff --git a/scripts/tests/control-plane-footprint.sh b/scripts/tests/control-plane-footprint.sh index a82ecfa3..037b9f4e 100755 --- a/scripts/tests/control-plane-footprint.sh +++ b/scripts/tests/control-plane-footprint.sh @@ -14,11 +14,14 @@ # constant still covers it. Grep the three consumers for `3008`, `platform # footprint`, `control plane requests`: zero hits. The number was invisible. # -# Measured here by rendering the chart: the steady-state control plane requests -# ~3136 MiB, already ABOVE the 3 GiB (3072 MiB) the envelope reserves for it. -# That 64 MiB overshoot is the memory half of the reason a training pod on a -# freshly-installed single-node edge can sit `Pending / Insufficient memory` -# (backend#2870). CPU fits: 900m requested against a 1000m reserve. +# Measured here by rendering the chart: the steady-state control plane requested +# ~3136 MiB / 900 m before backend#2461's interim trim (2026-09-10) -- ABOVE the +# 3 GiB (3072 MiB) the envelope reserved for it; 2272 MiB / 650 m after, under it. +# That 64 MiB overshoot was the memory half of the reason a training pod on a +# freshly-installed single-node edge could sit `Pending / Insufficient memory` +# (backend#2870). The render COUNTS THE TELEMETRY COLLECTOR (helm template cannot +# look up the token Secret, so the collector renders): that is the worst case a +# real edge reaches once the Secret exists, and the one the ceiling must hold. # # WHAT THIS GUARD DOES, AND DELIBERATELY DOES NOT # ----------------------------------------------- @@ -70,8 +73,8 @@ installer="$root/scripts/lib/install-client-helm.sh" # same PR -- which is the moment to weigh whether the training envelope can still # afford it. Overridable so the guard's own test can drive a lower ceiling and # watch a real render breach it. -MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-3136}" -CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-900}" +MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2272}" +CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-650}" command -v helm >/dev/null 2>&1 || { echo "[ERROR] helm is required to render the chart footprint" >&2; exit 3; } command -v python3 >/dev/null 2>&1 || { echo "[ERROR] python3 is required to sum the rendered requests" >&2; exit 3; } @@ -169,9 +172,9 @@ try: # and cpu, so a pod can take its memory from the init side and its cpu # from the app side. # - # THE NUMBER DOES NOT MOVE ON THIS CHART -- 3136 MiB / 900 m either way, - # because these init containers carry no requests. So the "64 MiB OVER" - # finding stands; the method was wrong, the conclusion was not. It also + # THE NUMBER DOES NOT MOVE ON THIS CHART -- the render is the same + # either way, because these init containers carry no requests. So the + # original "64 MiB OVER" finding stood; the method was wrong, the conclusion was not. It also # means the fix is INERT on the real render, which is why the bats # fixture is built so the two formulas disagree. def _req(c, key): diff --git a/scripts/tests/envelope-schedulability-mutations.sh b/scripts/tests/envelope-schedulability-mutations.sh index 89abee65..09600db6 100644 --- a/scripts/tests/envelope-schedulability-mutations.sh +++ b/scripts/tests/envelope-schedulability-mutations.sh @@ -110,7 +110,7 @@ run_case "unreadable cluster writes a carried non-floor size" "$LIB" \ printf '\nthe embed:\n' run_case "embedded footprint drifts from the render" "$LIB" \ - '_TB_CP_FOOTPRINT_MEM_BYTES=3288334336' \ + '_TB_CP_FOOTPRINT_MEM_BYTES=2382364672' \ '_TB_CP_FOOTPRINT_MEM_BYTES=3221225472' 1 printf '\n%s passed, %s failed\n' "$pass" "$fail" diff --git a/scripts/tests/envelope-schedulability.bats b/scripts/tests/envelope-schedulability.bats index 25e1e706..e65449b9 100644 --- a/scripts/tests/envelope-schedulability.bats +++ b/scripts/tests/envelope-schedulability.bats @@ -171,9 +171,12 @@ YAML # It walked the golden vectors -- a guard that passes having checked nothing is # the class this repo catalogues. The table has 13 rows today; assert a floor, # not the exact count, so a vector added upstream does not redden this. - [[ "$output" =~ ok\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } - [ "${BASH_REMATCH[2]}" -ge 10 ] || { echo "only ${BASH_REMATCH[2]} measurable vectors"; return 1; } - [ "${BASH_REMATCH[1]}" -ge 1 ] || return 1 + # Two legitimate shapes: `ok positive control: N/M` while the platform out-requests + # the reserve, or `note positive control: N/M ... does not apply` once it fits + # inside it (backend#2461). Either way the guard must have walked the table. + [[ "$output" =~ (ok|note)\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } + [ "${BASH_REMATCH[3]}" -ge 10 ] || { echo "only ${BASH_REMATCH[3]} measurable vectors"; return 1; } + if [[ "${BASH_REMATCH[1]}" == "ok" ]]; then [ "${BASH_REMATCH[2]}" -ge 1 ] || return 1; fi } @test "envelope-schedulability.sh: CANNOT TELL (exit 2) when the tree has no chart to derive from" { @@ -360,10 +363,11 @@ for key, kind in (("cpu_to_milli", "cpu"), ("mem_to_bytes", "mem")): } helm() { return 1; } unset TRACEBLOC_TRAINING_RESOURCES - # ONE system pod here (70Mi / 100m), so cpu fits (3000 + 900 + 100 = 4000) and - # only memory reduces (5120 + 3136 + 70 > 8192 -> 4 GiB): cpu=3,memory=4Gi. - # Anything else in the capture -- a stray echo -- would corrupt the value. + # ONE system pod here (70Mi / 100m). Since backend#2461's trim the resolver's + # cpu=3,memory=5Gi fits (5120 + 2336 + 70 <= 8192; 3000 + 750 + 100 <= 4000), so + # the fit writes it unchanged. Anything else in the capture -- a stray echo -- + # would corrupt the value, which is what this test is for. local captured captured="$(_resolve_training_size; _fit_training_envelope; printf '%s' "$_TB_TRAINING_SIZE")" - [ "$captured" = "cpu=3,memory=4Gi" ] || { echo "captured '$captured'"; return 1; } + [ "$captured" = "cpu=3,memory=5Gi" ] || { echo "captured '$captured'"; return 1; } } diff --git a/scripts/tests/envelope-schedulability.sh b/scripts/tests/envelope-schedulability.sh index 1e552818..d22e04c1 100644 --- a/scripts/tests/envelope-schedulability.sh +++ b/scripts/tests/envelope-schedulability.sh @@ -7,11 +7,15 @@ # --------------- # backend#2870: the installer sized the training envelope as `allocatable - 3 GiB` # and nothing then asked whether that number could be scheduled beside what the -# chart installs. It could not: the control plane requests 3136 MiB (from the -# render) against the 3072 MiB reserve, so every install over-asked and the +# chart installs. It could not: the control plane then requested 3136 MiB (from +# the render) against the 3072 MiB reserve, so every install over-asked and the # training pod sat Pending. No test in this repo could see it, because every # test compared the installers to the CONTRACT and none compared the arithmetic -# to a CLUSTER. +# to a CLUSTER. backend#2461's interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve -- so the over-ask no longer occurs on a +# real node and the positive control below reports that instead of failing; +# the reduce/refuse paths are exercised on nodes DERIVED from the footprint +# (SMALL_MI / TINY_MI), never a written-down machine size. # # WHAT IT ASSERTS, and how it derives rather than restates (backend#1729 rule 1) # ---------------------------------------------------------------------------- @@ -23,7 +27,9 @@ # 1. the installer's embedded footprint constants equal that render; # 2. for EVERY single-node golden vector in the contract table, the envelope # written + chart footprint + system pods <= allocatable, on memory AND cpu; -# 3. the ticket's 8 GiB reproduction is REDUCED, with the arithmetic; +# 3. the ticket's 8 GiB reproduction now FITS unreduced (backend#2461 brought +# the footprint under the reserve), and a node sized from the live +# footprint to over-ask is REDUCED, with the arithmetic (3b); # 4. a machine where not even a 1-core / 1-GiB run fits is REFUSED; # 5. a cpu-only overshoot reduces cpu alone (DoD part 5: cover cpu too); # 6. a human's pin is never altered, only warned; @@ -190,32 +196,56 @@ if (( NEED_B > _TB_ENVELOPE_OVERHEAD_MEM_BYTES || NEED_M > _TB_ENVELOPE_OVERHEAD bad "positive control: the platform out-requests the reserve, yet no vector over-asked before the fit -- this guard cannot see the defect it exists for" fi else - echo " note the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve; the fit is a no-op by construction and the positive control does not apply" + echo " note positive control: ${over_before}/${checked} measurable vectors over-asked -- the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve, so the fit is a no-op by construction and the control does not apply (backend#2461)" fi -# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. +# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. Before backend#2461 it over-asked +# (5120 + 3136 + 140 > 8192) and was REDUCED; after the trim it fits unreduced. +# Pinning the fit is the regression guard: a chart change that pushes the +# footprint back over the reserve reddens here first. resolve_and_fit "4 8Gi" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]]; then - ok "8 GiB reproduction: ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" +if [[ "$_TB_FIT_VERDICT" == "fits" && "$_TB_TRAINING_SIZE" == "$BEFORE" ]]; then + ok "8 GiB reproduction: ${BEFORE} fits unreduced beside $(( NEED_B / MIB )) MiB / ${NEED_M} m (backend#2461 closed the over-ask)" +else + bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want fits, unreduced)" +fi + +# 3b. THE REDUCE PATH still has to be exercised, so the node is DERIVED from the +# footprint: below 4 GiB the resolver falls to the 1-core / 2-GiB floor, and +# SMALL_MI leaves room for 1 GiB but not for that floor, so the fit must +# reduce to 1 GiB with the arithmetic on screen. Moves with the render. +SMALL_MI=$(( NEED_B / MIB + 1024 + 64 )) +resolve_and_fit "4 ${SMALL_MI}Mi" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]] && [[ "$_TB_TRAINING_SIZE" == "cpu=1,memory=1Gi" ]]; then + ok "${SMALL_MI} MiB node (derived): ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" printf '%s\n' "$_TB_FIT_LINES" | sed 's/^/ /' else - bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" + bad "${SMALL_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want reduced to cpu=1,memory=1Gi)" fi -# 4. REFUSAL when not even 1 core / 1 GiB fits. -resolve_and_fit "4 4Gi" +# 4. REFUSAL when not even 1 core / 1 GiB fits -- TINY_MI is derived so that +# allocatable minus the platform is 64 MiB short of a 1 GiB run. +TINY_MI=$(( NEED_B / MIB + 1024 - 64 )) +resolve_and_fit "4 ${TINY_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "refused" && "$_TB_FIT_LINES" == *"not even a 1-core / 1-GiB run"* ]]; then - ok "4 GiB node: refused, with the arithmetic" + ok "${TINY_MI} MiB node (derived): refused, with the arithmetic" else - bad "4 GiB node: verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" + bad "${TINY_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" fi -# 5. CPU-ONLY overshoot: memory already fits (the GiB floor absorbed it), cpu does not. +# 5. CPU-ONLY overshoot: memory already fits, cpu does not. The resolver leaves +# one core for the platform and the trimmed platform needs 950 m, so no real +# node can produce this any more -- the pre-trim footprint could. The case +# keeps DoD part 5 covered by REPLAYING that footprint (900 m, the value the +# embed carried before backend#2461) through the real fit; the memory side is +# untouched so only cpu can move. +saved_cpu="$_TB_CP_FOOTPRINT_CPU_MILLI"; _TB_CP_FOOTPRINT_CPU_MILLI=900 resolve_and_fit "8 63928Mi" +_TB_CP_FOOTPRINT_CPU_MILLI="$saved_cpu" a_mem="$(env_mem_b "$_TB_TRAINING_SIZE")"; b_mem="$(env_mem_b "$BEFORE")" a_cpu="$(env_cpu_m "$_TB_TRAINING_SIZE")"; b_cpu="$(env_cpu_m "$BEFORE")" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + NEED_M <= 8000 )); then - ok "cpu-only overshoot: ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + 900 + SYS_M <= 8000 )); then + ok "cpu-only overshoot (pre-trim 900 m replayed): ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" else bad "cpu-only overshoot: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" fi @@ -237,9 +267,10 @@ else bad "measured system pods: $(( ${_TB_SYS_MEM_BYTES:-0} / MIB )) MiB / ${_TB_SYS_CPU_MILLI:-0} m (want ${SYS_MIB} / ${SYS_M}); note: ${_TB_SYS_NOTE:-}" fi -# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces. +# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces on the +# derived SMALL_MI node (the 8 GiB machine no longer needs reducing). PODS_READABLE=0 -resolve_and_fit "4 8Gi" +resolve_and_fit "4 ${SMALL_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "reduced" && "$_TB_FIT_LINES" == *"NOT measured"*"chart derivation only"* ]]; then ok "pod list unreadable: verified against the chart derivation only, and said so (${BEFORE} -> ${_TB_TRAINING_SIZE})" else diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index 0e3a7289..33801007 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -1378,12 +1378,16 @@ _override_release_ctx() { # ── the envelope is checked against what the node can actually schedule (backend#2870) ── # -# The resolver sizes against ALLOCATABLE. The chart's own control plane requests -# 3136 MiB / 900 m (from the render) and k3s's system pods ~140 MiB / 200 m, so -# `allocatable - 3 GiB` over-asked on every machine and the training pod sat -# Pending. These three drive the WHOLE install flow so the written values file -# is what is asserted -- the unit-level arithmetic lives in -# scripts/tests/envelope-schedulability.sh and .bats. +# The resolver sizes against ALLOCATABLE. Before backend#2461 the chart's own +# control plane requested 3136 MiB / 900 m (from the render) and k3s's system +# pods ~140 MiB / 200 m, so `allocatable - 3 GiB` over-asked on every machine and +# the training pod sat Pending. The interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve, so an 8 GiB node now fits unreduced -- and +# the reduce/refuse paths are exercised on nodes DERIVED from the embedded +# footprint (`_TB_CP_FOOTPRINT_MEM_BYTES`, sourced with the library), so these +# tests follow the number instead of restating it. These drive the WHOLE install +# flow so the written values file is what is asserted -- the unit-level +# arithmetic lives in scripts/tests/envelope-schedulability.sh and .bats. _sched_cluster() { # $1 = node line(s) kubectl() { case "$*" in @@ -1395,8 +1399,10 @@ _sched_cluster() { # $1 = node line(s) } _SCHED_NODES="$1" } +# platform = embedded chart footprint + the two canned system pods (140 MiB / 200 m) +_sched_need_mib() { echo $(( _TB_CP_FOOTPRINT_MEM_BYTES / 1048576 + 140 )); } -@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { +@test "install_client_helm: the ticket's 8 GiB node now FITS unreduced (backend#2870 closed by backend#2461)" { HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" _ensure_tracebloc_dirs() { :; } _ensure_release_dirs() { :; } @@ -1405,17 +1411,38 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # The ticket's reproduction: an 8 GiB node. The resolver says cpu=3,memory=5Gi; - # 5120 + 3136 + 140 = 8396 MiB > 8192, and 3000 + 900 + 200 = 4100 m > 4000. + # 5120 + platform <= 8192 MiB and 3000 + 950 <= 4000 m: the resolver's + # cpu=3,memory=5Gi is written as-is, and nothing is reported OVER. _sched_cluster '4 8Gi' run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] || { echo "$output"; return 1; } - grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } - grep -q 'RESOURCE_LIMITS: "memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 + grep -q 'RESOURCE_REQUESTS: "cpu=3,memory=5Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + [[ "$output" != *"OVER"* ]] || { echo "$output"; return 1; } +} + +@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + has() { return 0; } + verify_credentials() { printf valid; } + unset TRACEBLOC_TRAINING_RESOURCES + # A node DERIVED from the footprint: below 4 GiB the resolver falls to the + # 1-core / 2-GiB floor; this node leaves room for 1 GiB but not 2, so the fit + # must reduce to memory=1Gi and print the OVER arithmetic. + local need small over + need="$(_sched_need_mib)"; small=$(( need + 1024 + 64 )); over=$(( 2048 + need - small )) + _sched_cluster "4 ${small}Mi" + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + grep -q 'RESOURCE_REQUESTS: "cpu=1,memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + grep -q 'RESOURCE_LIMITS: "memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 grep -q 'RESOURCE_PROVENANCE: "installer"' "$HOST_DATA_DIR/values.yaml" || return 1 # The arithmetic is on screen, not just the verdict. - [[ "$output" == *"204 MiB OVER"* ]] || { echo "$output"; return 1; } - [[ "$output" == *"reduced cpu=3,memory=5Gi -> cpu=2,memory=4Gi"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"${over} MiB OVER"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"reduced cpu=1,memory=2Gi -> cpu=1,memory=1Gi"* ]] || { echo "$output"; return 1; } } @test "install_client_helm: REFUSES to write an envelope when not even a 1-core/1-GiB run fits (backend#2870)" { @@ -1427,8 +1454,9 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # 4 GiB allocatable: 4096 - 3136 - 140 = 820 MiB is all that is left. - _sched_cluster '4 4Gi' + # Derived: allocatable minus the platform is 64 MiB short of a 1 GiB run. + local tiny; tiny=$(( $(_sched_need_mib) + 1024 - 64 )) + _sched_cluster "4 ${tiny}Mi" run install_client_helm <<< $'myid\nmypw' [ "$status" -ne 0 ] || { echo "$output"; return 1; } [[ "$output" == *"Refusing to write a training envelope"* ]] || { echo "$output"; return 1; } diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 50804852..bfbd2aee 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9780,41 +9780,62 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } $failures -join "`n" | Should -BeNullOrEmpty $checked | Should -BeGreaterOrEqual 10 -Because "a guard that checked almost nothing proves almost nothing" - # POSITIVE CONTROL: the platform out-requests the reserve today, so at least - # one vector must over-ask BEFORE the fit -- else this suite could never have - # seen the defect it exists for. + # POSITIVE CONTROL: while the platform out-requested the reserve (before + # backend#2461's trim) at least one vector had to over-ask BEFORE the fit -- + # else this suite could never have seen the defect it exists for. The guard is + # conditional on the embedded footprint, so it re-arms by itself the day a + # chart out-requests the reserve again. if ($needB -gt $script:TbEnvelopeOverheadMemBytes -or $needM -gt $script:TbEnvelopeOverheadCpuMilli) { $overBefore | Should -BeGreaterThan 0 -Because "the platform out-requests the reserve, yet no vector over-asked before the fit" } } - It "3. the ticket's reproduction: an 8 GiB node is REDUCED, arithmetic printed" { + It "3. the ticket's reproduction: an 8 GiB node now FITS unreduced (backend#2461 closed the over-ask)" { + # Before the interim trim the platform asked 3136 MiB / 900 m and this node was + # REDUCED to cpu=2,memory=4Gi; at 2336 MiB / 750 m the resolver's cpu=3,memory=5Gi + # fits: 5120 + 2476 <= 8192 and 3000 + 950 <= 4000. Pinning the fit is the + # regression guard for a chart change that pushes the footprint back up. $r = Invoke-FitOn -Nodes @('4 8Gi') + $r.Fit.Verdict | Should -Be 'fits' + $r.Fit.Size | Should -Be $r.Before + } + It "3b. a node DERIVED from the footprint (moves with the render) is REDUCED, arithmetic printed" { + # Below 4 GiB the resolver falls to the 1-core / 2-GiB floor; this node leaves + # room for 1 GiB but not for that floor, so the fit must reduce to 1 GiB with + # the OVER arithmetic on screen. + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*OVER*' ($r.Fit.Lines -join "`n") | Should -BeLike "*reduced $($r.Before) -> $($r.Fit.Size)*" - # The numbers on this node against the canned cluster: cpu 3000+900+200 = 4100 - # > 4000 -> 2 cores; memory 5120+3136+140 = 8396 > 8192 -> 8192-3276 = 4916 MiB - # -> 4 GiB. Both dimensions over-asked, both reduced. - $r.Fit.Size | Should -Be 'cpu=2,memory=4Gi' + $r.Fit.Size | Should -Be 'cpu=1,memory=1Gi' } - - It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic" { - $r = Invoke-FitOn -Nodes @('4 4Gi') + It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic (node derived)" { + # allocatable minus the platform is 64 MiB short of a 1 GiB run + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $tiny = $needMib + 1024 - 64 + $r = Invoke-FitOn -Nodes @("4 ${tiny}Mi") $r.Fit.Verdict | Should -Be 'refused' ($r.Fit.Lines -join "`n") | Should -BeLike '*not even a 1-core / 1-GiB run*' } - - It "5. cpu-only overshoot reduces cpu alone (memory kept)" { - $r = Invoke-FitOn -Nodes @('8 63928Mi') - $r.Fit.Verdict | Should -Be 'reduced' - (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) - $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) - $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) - $aCpu | Should -BeLessThan $bCpu - ($aCpu + $script:TbCpFootprintCpuMilli + $script:SysM) | Should -BeLessOrEqual 8000 + It "5. cpu-only overshoot reduces cpu alone (memory kept) -- the pre-trim 900 m replayed" { + # The resolver leaves one core for the platform and the trimmed platform needs + # 950 m, so no real node produces a cpu-only overshoot any more; the pre-trim + # footprint (900 m) did. Replaying it through the real fit keeps DoD part 5 + # covered; memory is untouched so only cpu can move. + $saved = $script:TbCpFootprintCpuMilli + try { + $script:TbCpFootprintCpuMilli = 900 + $r = Invoke-FitOn -Nodes @('8 63928Mi') + $r.Fit.Verdict | Should -Be 'reduced' + (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) + $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) + $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) + $aCpu | Should -BeLessThan $bCpu + ($aCpu + 900 + $script:SysM) | Should -BeLessOrEqual 8000 + } finally { $script:TbCpFootprintCpuMilli = $saved } } - It "6. a human's pin is warned, never altered (pinned-over)" { $r = Invoke-FitOn -Nodes @('4 8Gi') -Override 'cpu=4,memory=16Gi' $r.Prov | Should -Be 'user' @@ -9836,7 +9857,10 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "7b. pods unreadable: verified against the chart derivation only, and it SAYS so; still reduces" { - $r = Invoke-FitOn -Nodes @('4 8Gi') -PodsReadable $false + # on the derived small node -- the 8 GiB machine no longer needs reducing + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") -PodsReadable $false $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*NOT measured*chart derivation only*' } @@ -9867,19 +9891,22 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "the reduction below the contract floor flags Undersized, and a fit does not" { - # 4c/5Gi: the resolver leaves cpu=3,memory=2Gi (5120-3072 = 2048 MiB, the - # floor exactly). Memory: 2048+3136+140 > 5120 -> 5120-3276 = 1844 MiB -> 1 GiB, - # below the 2 GiB floor -> Undersized. Cpu: 4000-1100 = 2900 m -> 2 cores, so - # the run is still a requestable shape and is REDUCED, not refused. (On a - # 2-core node the same memory would reduce but cpu would leave 900 m -- under - # one core -- and the verdict is refused; that is case 4's territory.) - $r = Invoke-FitOn -Nodes @('4 5Gi') - $r.Fit.Verdict | Should -Be 'reduced' - $r.Fit.Size | Should -Be 'cpu=2,memory=1Gi' - $r.Fit.Undersized | Should -BeTrue - # A genuine fit: an installer-sized envelope ALWAYS over-asks today (the - # resolver leaves 3 GiB and the platform asks 3136 MiB plus system pods -- the - # positive control above), so use a carried installer size that fits. + # A reduction that lands BELOW the 2 GiB floor needs the resolver to start at + # or above the floor (node >= 5 GiB) AND the platform to push it under -- which + # the trimmed 2336 MiB no longer can (5120 - 2476 = 2644 MiB >= 2 GiB). Replay + # the pre-trim footprint (3136 MiB) through the real fit, as case 5 does for + # cpu: 4c/5Gi -> cpu=3,memory=2Gi; memory 2048+3136+140 > 5120 -> 5120-3276 = + # 1844 MiB -> 1 GiB, below the floor -> Undersized. Cpu: 4000-950 = 3050 m -> + # 3 cores, still requestable, so REDUCED, not refused. + $saved = $script:TbCpFootprintMemBytes + try { + $script:TbCpFootprintMemBytes = 3136L * 1MB # Int64 literal: no Int32 product to argue about (Bugbot, twice) + $r = Invoke-FitOn -Nodes @('4 5Gi') + $r.Fit.Verdict | Should -Be 'reduced' + $r.Fit.Size | Should -Be 'cpu=3,memory=1Gi' + $r.Fit.Undersized | Should -BeTrue + } finally { $script:TbCpFootprintMemBytes = $saved } + # A genuine fit, on the real footprint: a carried installer size that fits. $r2 = Invoke-FitOn -Nodes @('16 64Gi') -Carried @{ Size = 'cpu=4,memory=12Gi'; Provenance = 'installer' } $r2.Before | Should -Be 'cpu=4,memory=12Gi' $r2.Fit.Verdict | Should -Be 'fits'