From d3531129d18d3314860745a802f7dd3262937852 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:30:22 +0200 Subject: [PATCH 1/3] feat(chart): one registry helper for the tracebloc control-plane images (images.traceblocRegistry) (#1028) * feat(chart): one registry helper for the tracebloc control-plane images (images.traceblocRegistry) The four tracebloc-published control-plane image sites (jobs-manager init+api, pods-monitor, requests-proxy, resource-monitor), the image-refresh CronJob and NOTES.txt each derived "which registry" on their own, all as a docker.io literal behind the global.imageRegistry dig. Route them through ONE helper, tracebloc.tbRegistry: global.imageRegistry (mirror) wins, then the new tracebloc-only knob images.traceblocRegistry, then the chart default. The default is unchanged (docker.io): default installs render byte-identically (696 existing tests untouched). The knob is the per-edge opt-in to the ghcr.io copies -- the images are published to both registries at the same digests -- and the per-edge rollback once the default moves in a follow-up. image-refresh now resolves digests on the registry the pods pull from instead of a docker.io literal, and its "can I reconcile here" guard reads a chart-rendered verdict (IMAGE_REGISTRY_RESOLVABLE) derived from the same declaration (tracebloc.imageRefreshResolvableRegistries: docker.io ghcr.io) that flips the pods to IfNotPresent -- so the pull policy and the script's inert path cannot disagree. Not routed on purpose: tracebloc/mysql-client (frozen, digest-pinned, Docker Hub only), busybox and the other third-party images, and JOB_IMAGE_HOST (the training images move in their own step). 32 new helm unit tests (tests/tracebloc_registry_test.yaml), mutation-proved against seven breakages; schema entry; docs. * test(image-refresh): the stale-pin bats harness supplies IMAGE_REGISTRY, which the branch now resolves on The branch reads $IMAGE_REGISTRY instead of a docker.io literal, and the harness runs under set -eu with the pod's env hand-supplied -- so the whole suite (14 tests) went red in CI while the two refresh-script suites run locally stayed green. Run every file that exercises the script, derived by grep, not the two that came to mind. --- client/Chart.yaml | 4 +- client/templates/NOTES.txt | 2 +- client/templates/_helpers.tpl | 80 +++- client/templates/image-refresh-cronjob.yaml | 77 +-- client/templates/jobs-manager-deployment.yaml | 6 +- .../templates/requests-proxy-deployment.yaml | 2 +- .../templates/resource-monitor-daemonset.yaml | 2 +- client/tests/image_refresh_test.yaml | 8 +- client/tests/tracebloc_registry_test.yaml | 439 ++++++++++++++++++ client/values.schema.json | 5 + client/values.yaml | 57 ++- docs/INSTALL.md | 4 +- scripts/tests/image-refresh-skip-streak.bats | 7 +- scripts/tests/image-refresh-stale-pin.bats | 3 + 14 files changed, 631 insertions(+), 65 deletions(-) create mode 100644 client/tests/tracebloc_registry_test.yaml diff --git a/client/Chart.yaml b/client/Chart.yaml index de42aa09..15c1fd38 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.110 -appVersion: "1.9.110" +version: 1.9.111 +appVersion: "1.9.111" keywords: - tracebloc - kubernetes diff --git a/client/templates/NOTES.txt b/client/templates/NOTES.txt index 4b2472f8..c06bfad3 100644 --- a/client/templates/NOTES.txt +++ b/client/templates/NOTES.txt @@ -13,7 +13,7 @@ {{ "\033[1;34m" }}Service Account:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.serviceAccountName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}Secret:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.secretName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}RBAC scope:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ if ne .Values.clusterScope false }}Cluster{{ else }}Namespace{{ end }}{{ "\033[0m" }} - {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}docker.io{{ "\033[0m" }} + {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}{{ include "tracebloc.tbRegistry" . }}{{ "\033[0m" }} (tracebloc control-plane images) {{- if (default dict .Values.hostPath).enabled }} {{ "\033[1;34m" }}Storage:{{ "\033[0m" }} {{ "\033[0;33m" }}hostPath (bare-metal){{ "\033[0m" }} {{ "\033[1;34m" }}Host dirs:{{ "\033[0m" }} {{ "\033[0;33m" }}/tracebloc/data, /tracebloc/logs, /tracebloc/mysql (on the node){{ "\033[0m" }} diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index ed7d5292..4b8e2e69 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -463,10 +463,12 @@ true the kubelet pulls a digest it has never seen. Two conditions: * the CronJob renders at all (`imageRefresh.enabled`, and not every refreshed image already pinned), and - * images come from docker.io. Under a `global.imageRegistry` mirror the - script goes inert by design — it resolves digests from docker.io, and - pinning one onto a mirrored reference could pin an image the mirror - does not hold. + * the images come from a registry the script can resolve digests on + anonymously (tracebloc.imageRefreshResolvableRegistries: docker.io, + ghcr.io — wherever tracebloc.tbRegistry points). Under a + `global.imageRegistry` mirror the script goes inert by design — it + cannot resolve there, and pinning a digest resolved elsewhere onto a + mirrored reference could pin an image the mirror does not hold. 3. Otherwise -> Always. No reconcile, no pin, so a floating tag plus a restart is the ONLY way that edge can ever move. This is exactly the pre-#569 behaviour, kept for exactly the edges that still depend on it: @@ -483,10 +485,9 @@ true Usage: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" $d "root" $) }} */}} {{- define "tracebloc.controlPlanePullPolicy" -}} -{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} {{- if .digest -}} IfNotPresent -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (include "tracebloc.imageRefreshResolvable" .root) -}} IfNotPresent {{- else -}} Always @@ -584,6 +585,73 @@ with the ROOT context (e.g. `include "tracebloc.mirrorPrefix" $`). {{- with (dig "imageRegistry" "" (.Values.global | default dict)) }}{{ . }}/{{ end -}} {{- end -}} +{{/* +tracebloc.imageRefreshResolvableRegistries — the registries the image-refresh +script can resolve a floating tag to a digest on WITHOUT a credential: the +public token endpoints it knows (`get_token` / `get_latest_digest` in +image-refresh-cronjob.yaml carry one arm per entry). Space-separated, ONE +declaration: `tracebloc.imageRefreshResolvable` (below) and the +`tracebloc.controlPlanePullPolicy` decision both read it, and the CronJob +renders the verdict into the script's env as IMAGE_REGISTRY_RESOLVABLE — so the +pods' pull policy and the script's "can I reconcile here" guard cannot disagree. +Adding a registry here without a matching token arm in the script would make +the pods IfNotPresent while every tick WARNs "unknown registry"; the unit tests +pin the arms to this list. +*/}} +{{- define "tracebloc.imageRefreshResolvableRegistries" -}} +docker.io ghcr.io +{{- end -}} + +{{/* +tracebloc.tbRegistry — the registry the tracebloc-PUBLISHED control-plane images +(tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor, and +the requests-proxy, which runs the jobs-manager image) are pulled from. + +ONE precedence chain, so the four call sites, the image-refresh CronJob and +NOTES.txt cannot disagree about where those images live: + + 1. `global.imageRegistry` — a private mirror re-homes EVERY image the + chart pulls (#585), tracebloc/* included. + It always wins. + 2. `images.traceblocRegistry` — the tracebloc-only knob: moves just the + tracebloc-published images, leaving busybox, + squid, alpine/*, the device plugins and the + ingestor where they are. Also the per-edge + rollback: set it to the previous registry. + 3. "docker.io" — the chart default while the images are + published to Docker Hub. + +NOT routed through here, on purpose: `tracebloc/mysql-client` (frozen, +digest-pinned, published only to Docker Hub — see images.mysqlClient), the +third-party images (each has its own `registry` key), and — for now — the +training-image host JOB_IMAGE_HOST, which moves in its own step. + +Every read is nil-guarded and `| default`-chained: values.yaml ships +`global.imageRegistry: ""` (the key EXISTS, so `dig`'s own fallback never +applies — the trap image_refresh_test.yaml pins), and an edge upgrading with +`--reuse-values` from before `images.traceblocRegistry` existed has no such key +at all. Both must render the default, never "". + +Call with the ROOT context: {{ include "tracebloc.tbRegistry" . }} +*/}} +{{- define "tracebloc.tbRegistry" -}} +{{- $mirror := dig "imageRegistry" "" (.Values.global | default dict) -}} +{{- $own := dig "traceblocRegistry" "" (.Values.images | default dict) -}} +{{- $mirror | default ($own | default "docker.io") -}} +{{- end -}} + +{{/* +tracebloc.imageRefreshResolvable — "true" when the image-refresh script can +reconcile on this edge: the registry tracebloc.tbRegistry resolves to is one of +tracebloc.imageRefreshResolvableRegistries. Empty otherwise (a private mirror, +or a registry the script has no token arm for). Call with the ROOT context. +*/}} +{{- define "tracebloc.imageRefreshResolvable" -}} +{{- if has (include "tracebloc.tbRegistry" .) (splitList " " (include "tracebloc.imageRefreshResolvableRegistries" .)) -}} +true +{{- end -}} +{{- end -}} + {{/* tracebloc.ingestorDigest — the ONE effective digest for the spawned ingestor image. Renders the digest to pin to, or nothing at all to float on diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index d664eab1..d57bdaae 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -12,8 +12,10 @@ data: image-refresh.sh: | #!/bin/sh # Polls the registry for a new manifest digest and reconciles it onto the - # long-lived control-plane workloads, on docker.io under the floating - # CLIENT_ENV tag (#154, extended by #569). + # long-lived control-plane workloads, on IMAGE_REGISTRY (the registry the + # chart pulls the tracebloc-published images from -- see the + # tracebloc.tbRegistry helper) under the floating CLIENT_ENV tag (#154, + # extended by #569). # # Action on change: `kubectl set image =repo@digest`. # It used to be `kubectl rollout restart`. #569 changed the mechanism so @@ -180,16 +182,24 @@ data: SKIP_KEY="tracebloc.io/refresh-skip-streak" log "release=$RELEASE_NAME namespace=$RELEASE_NAMESPACE deployment=$DEPLOYMENT_NAME" - log " client images: tracebloc/{jobs-manager,pods-monitor,resource-monitor} on docker.io under tag=$IMAGE_TAG" + log " client images: tracebloc/{jobs-manager,pods-monitor,resource-monitor} on ${IMAGE_REGISTRY} under tag=$IMAGE_TAG" log " also reconciling: deployment/${REQUESTS_PROXY_DEPLOYMENT} (jobs-manager image), daemonset/${RESOURCE_MONITOR_DAEMONSET} in ${NODE_AGENTS_NAMESPACE}" # #569: a private-mirror install (global.imageRegistry) renders every pod - # image as /tracebloc/... , but the digests resolved below come from - # docker.io. Writing a docker.io digest onto a mirror reference is not just - # wrong, it is BREAKING: the mirror may not hold that digest at all, so - # `set image` would pin the control plane to an unpullable ref. Under the - # old `rollout restart` this mismatch was merely useless (it restarted pods - # to re-pull the mirror's own tag); with `set image` it must fail closed. + # image as /tracebloc/... , but this script can only resolve a tag + # to a digest on the public registries it has a token arm for (get_token + # below: docker.io, ghcr.io). Writing a digest resolved elsewhere onto a + # mirror reference is not just wrong, it is BREAKING: the mirror may not + # hold that digest at all, so `set image` would pin the control plane to an + # unpullable ref. Under the old `rollout restart` this mismatch was merely + # useless (it restarted pods to re-pull the mirror's own tag); with + # `set image` it must fail closed. + # + # The verdict is RENDERED by the chart (IMAGE_REGISTRY_RESOLVABLE, from the + # same tracebloc.imageRefreshResolvableRegistries declaration that decides + # the pods' imagePullPolicy) rather than re-derived here, so the guard and + # the pull policy cannot disagree about which edges the reconcile runs on. + # Absent or anything but "true" is refused: not knowing is not permission. # # Exit 0, not 1: a mirror install has deliberately taken over image # distribution, so "nothing to reconcile" is the correct steady state, not @@ -198,9 +208,9 @@ data: # The CronJob is still RENDERED under a mirror (the alpine/k8s runner image # is re-homed onto it, per global_image_registry_test.yaml); only the # reconcile is inert. - if [ "$IMAGE_REGISTRY" != "docker.io" ]; then - log "NOTE: images are served from private mirror '${IMAGE_REGISTRY}' (global.imageRegistry), but this script resolves digests from docker.io." - log " Pinning a docker.io digest onto a mirrored reference could pin an image the mirror does not hold, so auto-refresh is INERT on this edge." + if [ "${IMAGE_REGISTRY_RESOLVABLE:-}" != "true" ]; then + log "NOTE: images are served from '${IMAGE_REGISTRY}', a registry this script cannot resolve digests on anonymously (it knows docker.io and ghcr.io)." + log " Pinning a digest resolved elsewhere onto that reference could pin an image the registry does not hold, so auto-refresh is INERT on this edge." log " Update the control-plane images by syncing your mirror and restarting the workloads, or pin images..digest explicitly." log " (That restart DOES re-pull: because this reconcile is inert here, the chart renders imagePullPolicy=Always on the unpinned control-plane images rather than IfNotPresent -- see the tracebloc.controlPlanePullPolicy helper. Offline-restart safety is traded away on this edge precisely so it is not left with no update path at all.)" log "tick complete" @@ -547,7 +557,7 @@ data: # it changes no behaviour: `continue` still ends this image's tick either # way. Only the reporting is new. stale_key="tracebloc.io/stale-pin-${repo#*/}" - pin_latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "docker.io" || true)" + pin_latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)" if [ -z "$pin_latest" ]; then log " pinned by digest in values; skipping (could not resolve ${IMAGE_TAG} to compare -- not knowing is a finding, not agreement)" log " leaving any existing ${stale_key} as the last KNOWN state rather than asserting agreement" @@ -587,7 +597,7 @@ data: annotate_args="$annotate_args ${unpinned_stale_key}-" fi - latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "docker.io" || true)" + latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)" if [ -z "$latest" ]; then log " WARN: could not resolve latest digest (rate-limited or transient); skipping this tick" continue @@ -726,9 +736,10 @@ data: annotate_args="$annotate_args ${key}=${latest}" restart_needed=1 - # The reference we pin. IMAGE_REGISTRY is docker.io here — a mirror - # exits the tick far above — but render it from the variable so this - # stays honest if that guard is ever relaxed. + # The reference we pin: the SAME registry the digest above was resolved + # on, so a copy of this image on another registry can never be pinned by + # a digest it does not hold. A registry the script cannot resolve on + # exited the tick far above (IMAGE_REGISTRY_RESOLVABLE). ref="${IMAGE_REGISTRY}/${repo}@${latest}" # Map the image onto the workload+container(s) that run it. Container @@ -990,21 +1001,25 @@ spec: value: {{ include "tracebloc.resourceMonitorName" . | quote }} - name: NODE_AGENTS_NAMESPACE value: {{ .Values.nodeAgents.namespace.name | quote }} - # The registry the chart renders into every pod image. The - # script resolves digests from docker.io only, so it goes inert - # when this is a private mirror rather than pinning a digest the - # mirror may not hold (see the guard at the top of the script). - # - # `| default "docker.io"` is NOT redundant with dig's default: - # values.yaml ships `global.imageRegistry: ""`, so the key EXISTS - # and dig returns the empty string rather than its fallback. - # Without the trailing default this renders "", the mirror guard - # below sees "" != "docker.io" and goes inert on every DEFAULT - # install — silently disabling refresh fleet-wide. The - # `tracebloc.image` helper handles the same trap with its own - # `| default "docker.io"`. + # The registry the chart renders into the control-plane pod + # images -- ONE helper (tracebloc.tbRegistry: global mirror, else + # images.traceblocRegistry, else the chart default), so the + # script resolves digests on exactly the registry the pods pull + # from. The helper owns the `| default` chain: values.yaml ships + # `global.imageRegistry: ""`, so the key EXISTS and a bare dig + # returns "" rather than its fallback; rendering "" here would + # make the reconcile go inert on every DEFAULT install (the + # "IMAGE_REGISTRY defaults to ..., not the empty string" test). - name: IMAGE_REGISTRY - value: {{ dig "imageRegistry" "docker.io" (.Values.global | default dict) | default "docker.io" | quote }} + value: {{ include "tracebloc.tbRegistry" . | quote }} + # Whether the script can resolve digests on IMAGE_REGISTRY + # anonymously -- derived from the same declaration + # (tracebloc.imageRefreshResolvableRegistries) that switches the + # pods to IfNotPresent, so the guard at the top of the script and + # the pull policy agree by construction. "false" makes the tick + # log and exit 0 (a mirror install has taken over distribution). + - name: IMAGE_REGISTRY_RESOLVABLE + value: {{ include "tracebloc.imageRefreshResolvable" . | default "false" | quote }} - name: IMAGE_TAG # RESOLVED, not raw. This is the tag the refresh pulls, so a # documented alias here freezes the edge on a tag the publish diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 81007109..19543b4a 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -100,7 +100,7 @@ spec: # # It also removes an image: busybox was a second pull on every edge, for a # TCP connect the app image can already do. - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} securityContext: # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's # `restricted` SCC assigns a UID from the project's range, and the pod @@ -265,7 +265,7 @@ spec: {{- end }} containers: - name: api - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile. `Always` forced a # registry round-trip on every (re)start, so an offline Docker Desktop / @@ -906,7 +906,7 @@ spec: {{- end }} {{- end }} - name: pods-monitor-container - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: same policy as the api container above -- see # tracebloc.controlPlanePullPolicy. imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.podsMonitor.digest "root" $) }} diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index 0ac6505d..a0f94e60 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -41,7 +41,7 @@ spec: these keys. */}} {{- $rpDigest := (dig "requestsProxy" "digest" "" $rpImages) | default (dig "jobsManager" "digest" "" $rpImages) }} - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile (see # tracebloc.controlPlanePullPolicy; it stays Always on an edge the diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index fca159ff..81bf11c4 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -136,7 +136,7 @@ spec: entry). `dig` is not usable here — it rejects chartutil.Values. */}} {{- $rmDigest := (default (dict) (default (dict) .Values.images).resourceMonitor).digest | default "" }} - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # see tracebloc.controlPlanePullPolicy. `Always` made every node's pod # re-pull on restart, so an offline restart wedged the DaemonSet in diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index 3787fbc7..e1f8ddda 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -770,11 +770,13 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: 'kubectl set image -n "\$NODE_AGENTS_NAMESPACE"' - # Mirror guard: a docker.io digest must never be pinned onto a mirrored - # reference the mirror may not hold. + # Mirror guard: a digest resolved on a public registry must never be pinned + # onto a mirrored reference the mirror may not hold. The verdict is the + # chart-rendered IMAGE_REGISTRY_RESOLVABLE (tracebloc_registry_test.yaml + # covers its values); anything but "true" -- including absent -- refuses. - matchRegex: path: data["image-refresh.sh"] - pattern: 'if \[ "\$IMAGE_REGISTRY" != "docker.io" \]; then' + pattern: 'if \[ "\$\{IMAGE_REGISTRY_RESOLVABLE:-\}" != "true" \]; then' # The pods-monitor container name is contractual with the deployment. - matchRegex: path: data["image-refresh.sh"] diff --git a/client/tests/tracebloc_registry_test.yaml b/client/tests/tracebloc_registry_test.yaml new file mode 100644 index 00000000..a3386a61 --- /dev/null +++ b/client/tests/tracebloc_registry_test.yaml @@ -0,0 +1,439 @@ +suite: images.traceblocRegistry — the tracebloc-only registry knob (tracebloc.tbRegistry) +# The tracebloc-PUBLISHED control-plane images (jobs-manager, pods-monitor, +# resource-monitor, and the requests-proxy that runs the jobs-manager image) are +# published to both docker.io and ghcr.io at the same digests. ONE helper, +# tracebloc.tbRegistry, decides which registry the chart pulls them from, as one +# precedence chain: global.imageRegistry (a mirror re-homes everything) wins, +# then images.traceblocRegistry, then the chart default. +# +# Pinned here, each of them a machine check for a sentence in values.yaml: +# 1. The knob moves EXACTLY the tracebloc control-plane sites: busybox, the +# ingestor repo and JOB_IMAGE_HOST do not follow it, and mysql-client stays +# on docker.io (frozen, digest-pinned, published nowhere else). +# 2. The mirror wins over the knob; an empty knob renders the default, not "". +# 3. The image-refresh CronJob resolves digests on the same registry the pods +# pull from, and its reconcile verdict (IMAGE_REGISTRY_RESOLVABLE) and the +# pods' imagePullPolicy are derived from ONE declaration +# (tracebloc.imageRefreshResolvableRegistries) -- so a registry the script +# has no token arm for makes the pods Always AND the tick inert, together. +# 4. NOTES.txt reports the registry the pods actually pull from. +# helm-unittest 0.5.2 applies asserts to EVERY template a test names, so each +# test below names exactly one. +templates: + - templates/jobs-manager-deployment.yaml + - templates/requests-proxy-deployment.yaml + - templates/resource-monitor-daemonset.yaml + - templates/mysql-deployment.yaml + - templates/image-refresh-cronjob.yaml + - templates/NOTES.txt +release: + name: t + namespace: tracebloc +set: + clientId: "test-id" + clientPassword: "test" +tests: + - it: the chart default pulls the jobs-manager Deployment's images from docker.io + template: templates/jobs-manager-deployment.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[1].image + pattern: "^docker\\.io/tracebloc/pods-monitor:" + + - it: the chart default pulls the requests-proxy image from docker.io + template: templates/requests-proxy-deployment.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + + - it: the chart default pulls the resource-monitor image from docker.io + template: templates/resource-monitor-daemonset.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/resource-monitor:" + + - it: an EMPTY knob renders the default registry on the pods, not an empty one (the dig-empty trap) + # values.yaml ships every registry key as a string, so `dig` finds the key + # and returns "" instead of its fallback; an edge that clears the knob, or a + # --reuse-values replay from before it existed, must still get a registry. + # The pods are guarded twice (tracebloc.image applies its own default to an + # empty registry), so this test alone would stay green if the helper lost + # its `| default` chain -- the IMAGE_REGISTRY test right below is the one + # with no second net, and it is what reddens under that mutation. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + + - it: an EMPTY knob renders the default registry into IMAGE_REGISTRY, not an empty one + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: "" + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "docker.io" + + - it: images.traceblocRegistry re-homes the jobs-manager Deployment (init, api, pods-monitor) + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[1].image + pattern: "^ghcr\\.io/tracebloc/pods-monitor:" + + - it: images.traceblocRegistry re-homes the requests-proxy + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + + - it: images.traceblocRegistry re-homes the resource-monitor + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/resource-monitor:" + + - it: a digest pin keeps the knob's registry (repo@digest, not repo:tag) + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "ghcr.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + - it: the knob does NOT move mysql-client (frozen, digest-pinned, docker.io only) + # The open question on mysql-client, recorded on the epic: it stays on + # Docker Hub. This is that decision as a machine check -- routing the mysql + # site through tracebloc.tbRegistry would make this test red. + template: templates/mysql-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/mysql-client@sha256:" + + - it: the knob does NOT move busybox + # busybox keeps its own precedence (global mirror, else docker.io). + template: templates/mysql-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: "^docker\\.io/library/busybox:" + + - it: the knob does NOT move the ingestor repository or JOB_IMAGE_HOST + # The ingestor is already on ghcr.io by repository; the training-image host + # moves in its own step, so it must still read docker.io/ here. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_REPOSITORY + value: "ghcr.io/tracebloc/ingestor" + - contains: + path: spec.template.spec.containers[0].env + content: + name: JOB_IMAGE_HOST + value: "docker.io/" + + - it: global.imageRegistry wins over images.traceblocRegistry on the jobs-manager + template: templates/jobs-manager-deployment.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^mirror\\.corp\\.example/tracebloc/jobs-manager:" + + - it: global.imageRegistry wins over images.traceblocRegistry on the resource-monitor + template: templates/resource-monitor-daemonset.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^mirror\\.corp\\.example/tracebloc/resource-monitor:" + + - it: global.imageRegistry wins over images.traceblocRegistry in IMAGE_REGISTRY + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "mirror.corp.example" + + - it: image-refresh resolves digests on the registry the pods pull from, never a docker.io literal + # The pinned reference is built from the same variable the digest was + # resolved on, so a copy on another registry can never be pinned by a digest + # it does not hold. + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'pin_latest="\$\(get_latest_digest "\$repo" "\$IMAGE_TAG" "\$IMAGE_REGISTRY" \|\| true\)"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'latest="\$\(get_latest_digest "\$repo" "\$IMAGE_TAG" "\$IMAGE_REGISTRY" \|\| true\)"' + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'get_latest_digest "\$repo" "\$IMAGE_TAG" "docker\.io"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'ref="\$\{IMAGE_REGISTRY\}/\$\{repo\}@\$\{latest\}"' + + - it: the script carries a token arm for every registry the chart declares resolvable + # tracebloc.imageRefreshResolvableRegistries is "docker.io ghcr.io". A + # registry declared there without a get_token / get_latest_digest arm would + # make the pods IfNotPresent while every tick WARNs "unknown registry". + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["image-refresh.sh"] + pattern: '_url="https://auth\.docker\.io/token\?service=registry\.docker\.io&scope=repository:\$\{_repo\}:pull"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: '_url="https://ghcr\.io/token\?service=ghcr\.io&scope=repository:\$\{_repo\}:pull"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'docker\.io\) _host="registry-1\.docker\.io" ;;' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'ghcr\.io\) _host="ghcr\.io" ;;' + + - it: on docker.io the reconcile is resolvable + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "true" + + - it: on docker.io the pods are IfNotPresent + template: templates/jobs-manager-deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the reconcile is resolvable and IMAGE_REGISTRY follows + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "ghcr.io" + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "true" + + - it: on ghcr.io the jobs-manager pods are IfNotPresent + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + - equal: + path: spec.template.spec.containers[1].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the requests-proxy is IfNotPresent + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the resource-monitor is IfNotPresent + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: a registry the script has no token arm for makes the tick inert + # The two consumers of tracebloc.imageRefreshResolvableRegistries must move + # as one: IfNotPresent with an inert reconcile is a frozen control plane + # with a green CronJob (#569, Bugbot High). + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: quay.example + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "quay.example" + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "false" + + - it: a registry the script has no token arm for makes the jobs-manager pods Always + # The two consumers of tracebloc.imageRefreshResolvableRegistries must move + # as one: IfNotPresent with an inert reconcile is a frozen control plane + # with a green CronJob (#569, Bugbot High). + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + - equal: + path: spec.template.spec.containers[1].imagePullPolicy + value: Always + + - it: a registry the script has no token arm for makes the requests-proxy Always + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: a registry the script has no token arm for makes the resource-monitor Always + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: a global mirror is not resolvable even when the knob names a public registry + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "false" + + - it: a global mirror keeps the pods Always even when the knob names a public registry + template: templates/jobs-manager-deployment.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: NOTES prints the default registry + template: templates/NOTES.txt + asserts: + - matchRegexRaw: + pattern: "Image registry:.*docker\\.io.*\\(tracebloc control-plane images\\)" + + - it: NOTES follows images.traceblocRegistry + template: templates/NOTES.txt + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegexRaw: + pattern: "Image registry:.*ghcr\\.io" + - notMatchRegexRaw: + pattern: "Image registry:.*docker\\.io" + + - it: NOTES follows the global mirror over the knob + template: templates/NOTES.txt + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegexRaw: + pattern: "Image registry:.*mirror\\.corp\\.example" + diff --git a/client/values.schema.json b/client/values.schema.json index 1a5abdaf..c50422fc 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -791,6 +791,11 @@ "type": "object", "description": "Container image pinning. Prefer digest over tag for immutability.", "properties": { + "traceblocRegistry": { + "type": "string", + "pattern": "^([A-Za-z0-9.-]+(:[0-9]+)?)?$", + "description": "Registry the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) are pulled from. A bare host, no scheme; empty renders the chart default. global.imageRegistry always wins when set. Moves ONLY the tracebloc images (mysql-client stays on docker.io; third-party images keep their own registry keys; JOB_IMAGE_HOST is not routed through it yet). Image-refresh resolves digests on this registry and can only do so anonymously on docker.io and ghcr.io -- any other value makes the reconcile inert and the pods imagePullPolicy=Always, like a mirror." + }, "jobsManager": { "type": "object", "properties": { diff --git a/client/values.yaml b/client/values.yaml index 564c0962..6b22572d 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -12,10 +12,13 @@ global: # When set, EVERY image the chart pulls is re-homed onto this host — the # tracebloc services, the spawned ingestor + training-job images, and the # alpine/*, ubuntu/squid, busybox, curl helper images — which must already host - # the same repositories + tags/digests. Unset (the default) pulls tracebloc/* - # and the helper images from docker.io and the ingestor from ghcr.io. If the - # mirror needs authentication, also set dockerRegistry below. The standalone - # installer writes this from TRACEBLOC_IMAGE_REGISTRY. See docs/INSTALL.md. + # the same repositories + tags/digests. Unset (the default) pulls the + # tracebloc-published control-plane images from images.traceblocRegistry + # (below), the helper images from docker.io and the ingestor from ghcr.io. If + # the mirror needs authentication, also set dockerRegistry below. The + # standalone installer writes this from TRACEBLOC_IMAGE_REGISTRY. See + # docs/INSTALL.md. To move ONLY the tracebloc images, leave this unset and set + # images.traceblocRegistry instead. imageRegistry: "" # -- Environment variables passed to containers @@ -906,15 +909,39 @@ telemetryCollector: # * neither -> Always (floating tag + restart is the # only update path there is) # -# The third case is real, not theoretical: a `global.imageRegistry` mirror makes -# the reconcile inert (it resolves digests from docker.io, which the mirror may -# not hold), and `imageRefresh.enabled: false` disables it outright. Those edges +# The third case is real, not theoretical: a registry the reconcile cannot +# resolve digests on anonymously -- a `global.imageRegistry` mirror, or an +# `images.traceblocRegistry` other than docker.io / ghcr.io -- makes it inert +# (a digest resolved elsewhere may not exist there), and +# `imageRefresh.enabled: false` disables it outright. Those edges # keep the pre-#569 behaviour deliberately — a frozen control plane with a green # CronJob and no signal is worse than a restart that needs the network. # # Setting `digest` here is still meaningful: it is an explicit operator pin # that also opts the image OUT of auto-refresh. images: + # Registry the tracebloc-PUBLISHED control-plane images are pulled from: + # tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor + # and the requests-proxy (which runs the jobs-manager image). A bare host, no + # scheme. Resolved by the tracebloc.tbRegistry helper as ONE precedence chain + # -- `global.imageRegistry` (a mirror re-homes everything) always wins, then + # this, then the chart default -- so the pods, the image-refresh CronJob and + # NOTES.txt cannot disagree about where those images live. + # + # This knob moves ONLY the tracebloc images: busybox, squid, alpine/*, the + # device plugins and the ingestor keep their own registries, and + # tracebloc/mysql-client stays on docker.io (frozen, digest-pinned, published + # nowhere else -- see images.mysqlClient). The training-image host + # (JOB_IMAGE_HOST) is not routed through it yet; that moves in its own step. + # + # The tracebloc images are published to BOTH docker.io and ghcr.io, at the + # same digests. Setting this to "ghcr.io" on one edge is how to try the + # ghcr.io copies before the chart default moves; setting it back is the + # per-edge rollback once it has. Image-refresh follows it: the CronJob + # resolves digests on this registry (it can do so anonymously on docker.io + # and ghcr.io; any other value makes the reconcile inert and the pods + # imagePullPolicy=Always, exactly as a global.imageRegistry mirror does). + traceblocRegistry: "docker.io" jobsManager: digest: "" # -- TRAINING IMAGES: the digest-pinned spawn path (RFC-1246 P2, landed for @@ -1730,10 +1757,13 @@ autoUpgrade: # prefix rewrite is not mistaken for a revert). Steady state is one HEAD # per image per tick, well under Docker Hub's 100/6h anonymous # pull-rate limit. -# - Private mirrors (global.imageRegistry): the script resolves digests -# from docker.io, so pinning one onto a mirrored reference could pin -# an image the mirror does not hold. It logs and goes INERT instead. -# Under the old `rollout restart` the mismatch was merely useless; with +# - The script resolves digests on the SAME registry the pods pull the +# tracebloc images from (images.traceblocRegistry, or the +# global.imageRegistry mirror when set), and it can only do so +# anonymously on docker.io and ghcr.io. Anywhere else -- a private +# mirror -- pinning a digest resolved elsewhere could pin an image the +# registry does not hold, so it logs and goes INERT instead. Under the +# old `rollout restart` the mismatch was merely useless; with # `set image` it has to fail closed. # # Pinned-digest interaction: if images.jobsManager.digest, @@ -1756,8 +1786,9 @@ autoUpgrade: # requires) are namespace-wide and read-only. Still far narrower than # autoUpgrade, which holds cluster-admin. # -# The script reaches out only to docker.io's public registry endpoints; if -# your cluster requires egress through a proxy, set HTTPS_PROXY via env +# The script reaches out only to the public token + manifest endpoints of +# the registry the tracebloc images are pulled from (docker.io or ghcr.io); +# if your cluster requires egress through a proxy, set HTTPS_PROXY via env # or disable this CronJob. # # Disabled-on-this-cluster check after install: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 498f760c..0480884b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -34,7 +34,7 @@ The standalone installer runs a **preflight** check that verifies this connectiv | Host | Why | |---|---| | `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox + the tracebloc client images | -| `ghcr.io` | k3d node images + the ingestor image | +| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | k3d node images + the ingestor image; the tracebloc control-plane images when `images.traceblocRegistry` points there | | `api.tracebloc.io` (`dev-api`/`stg-api` for non-prod) | client credential check + the running client's platform connection | | `tracebloc.github.io` | the tracebloc Helm chart repository | @@ -55,6 +55,8 @@ Some sites hard-block Docker Hub / GHCR outright — the images aren't reachable The chart follows the **`global.imageRegistry`** convention: set it once and **every** image the chart pulls — the tracebloc services, the spawned ingestor, the training-job images, and the `alpine/*`, `ubuntu/squid`, `busybox`, `curl` helper images — is re-homed onto your registry. No per-image overrides. +**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. To point just the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) at a different registry — they are published to both `docker.io` and `ghcr.io` at the same digests — set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset. `global.imageRegistry` always wins when both are set. The image-refresh CronJob follows the same value and can resolve digests anonymously on `docker.io` and `ghcr.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the training-image host yet. + **1. A private/mirror registry your site *can* reach.** First mirror the images into it. List exactly what to copy (tags/digests stay authoritative across chart versions) with: diff --git a/scripts/tests/image-refresh-skip-streak.bats b/scripts/tests/image-refresh-skip-streak.bats index 9fb7ad3e..f14fc9b9 100644 --- a/scripts/tests/image-refresh-skip-streak.bats +++ b/scripts/tests/image-refresh-skip-streak.bats @@ -107,15 +107,16 @@ exit 6 CURLSTUB chmod +x "$STUB_DIR/curl" - # The env the pod supplies. IMAGE_REGISTRY must be docker.io or the mirror - # guard exits 0 long before the settled guard is reached. + # The env the pod supplies. IMAGE_REGISTRY_RESOLVABLE is the chart-rendered + # verdict the mirror guard reads (tracebloc.imageRefreshResolvable); anything + # but "true" exits 0 long before the settled guard is reached. export PATH="$STUB_DIR:$PATH" export RELEASE_NAME=stg RELEASE_NAMESPACE=tracebloc export DEPLOYMENT_NAME=stg-jobs-manager export REQUESTS_PROXY_DEPLOYMENT=stg-requests-proxy export RESOURCE_MONITOR_DAEMONSET=stg-resource-monitor export NODE_AGENTS_NAMESPACE=tracebloc-node-agents - export IMAGE_REGISTRY=docker.io IMAGE_TAG=stg + export IMAGE_REGISTRY=docker.io IMAGE_REGISTRY_RESOLVABLE=true IMAGE_TAG=stg export ROLLOUT_TIMEOUT=300s # The script builds an explicit in-cluster kubeconfig from these (#634) and # runs under `set -u`, so they are not optional. diff --git a/scripts/tests/image-refresh-stale-pin.bats b/scripts/tests/image-refresh-stale-pin.bats index 8b84e958..32bbb310 100644 --- a/scripts/tests/image-refresh-stale-pin.bats +++ b/scripts/tests/image-refresh-stale-pin.bats @@ -75,6 +75,9 @@ run_branch() { cat > "$TMP/harness.sh" < Date: Thu, 10 Sep 2026 14:19:13 +0200 Subject: [PATCH 2/3] ci(standard-checks): drop the runner image's Google Chrome apt source before installing bats (#1029) * ci(standard-checks): drop the runner image's Google Chrome apt source before installing bats dl.google.com served a Hash Sum mismatch for ~10 minutes on 2026-09-09 and all three bounded bats-install attempts lost on main; nothing in this job reads that repository. Co-Authored-By: Claude Fable 5.1 * ci: drop every Google Chrome apt source file, deb822 .sources included, not only .list Bugbot (Medium): current ubuntu-latest images write google-chrome.sources, which the .list glob left in place, so apt-get update could still reach dl.google.com. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .github/workflows/standard-checks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/standard-checks.yml b/.github/workflows/standard-checks.yml index d67907bd..8d59b4f2 100644 --- a/.github/workflows/standard-checks.yml +++ b/.github/workflows/standard-checks.yml @@ -187,6 +187,12 @@ jobs: # `bash -e`, so a bare test returning false on the last iteration would # abort the script before the ::error:: line ever printed. run: | + # ubuntu-latest pre-bakes google-chrome.list (and, on current images, the deb822 google-chrome.sources) into the apt sources, so + # every `apt-get update` here depends on dl.google.com serving + # consistent index metadata. On 2026-09-09 it did not ("Hash Sum + # mismatch", ~10 minutes) and all three attempts below lost on main for + # a repository this job never reads. Drop it before the first update. + sudo rm -f /etc/apt/sources.list.d/google-chrome* for i in 1 2 3; do if sudo timeout 60 apt-get update -qq \ && sudo timeout 60 apt-get install -y bats; then From b3c66bd8601e8580fe2169b860f05370e5d0ba59 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:47:08 +0200 Subject: [PATCH 3/3] fix(chart): interim control-plane trim so the smallest training envelope fits a 2-core / 8 GiB node again (1.9.112) (#1032) * fix(chart): interim control-plane trim so the smallest training envelope fits a 2-core / 8 GiB node again (1.9.112) Since chart 1.9.109 the kubelet reservation is honest (#1017) and since client-runtime#544 the jobs-manager's admission subtracts the chart's real requests. On the smallest supported node that made the arithmetic visible: allocatable 1900m / 5538Mi, minus the control plane's requests (800m / 3008Mi without the collector) and k3s's coredns + metrics-server (200m / 140Mi), leaves 900m / 2390Mi -- and the smallest envelope needs 1000m / 3072Mi. Every task on staging was refused NEVER_STEADY_STATE (e2e fleet 34473120598). The requests were never sized from usage: 1050m / 3276Mi requested against 36m / ~1022Mi used. This trims the components whose peaks are measured and takes jobs-manager down modestly, giving back 150m / 800Mi: mysql 250m/1Gi -> 150m/704Mi (peak 0.55x; requests==limits kept) jobs-manager 250m/1Gi -> 250m/768Mi (168Mi idle; restart peak pending) pods-monitor 100m/512Mi -> 100m/320Mi (requests==limits kept) requests-proxy 100m/256Mi -> 50m/128Mi (limits unchanged) egress-proxy 50m/64Mi -> 50m/160Mi (UP: measured 2.45x over) Render: 3136 MiB / 900 m -> 2336 MiB / 750 m. The node above now leaves 1050m / 3190Mi. Interim: the final numbers come from the load run in progress; jobs-manager moves only once its restart peak is measured. Everything that derives from the render moves with it: the footprint ceiling ratchets down, the installer embed (bash + PowerShell) is regenerated, the template fallback defaults match, and the fit tests derive their small/tiny nodes from the footprint instead of restating a machine size -- the 8 GiB reproduction is now pinned as fitting unreduced, and the cpu-only overshoot replays the pre-trim 900 m because no real node can produce it any more. Co-Authored-By: Claude Fable 5.1 * fix(chart): mysql requests.cpu template fallback follows the trimmed 150m (Bugbot on #1032) Co-Authored-By: Claude Fable 5.1 * test(footprint): derive the ratchet test's ceiling from the render so it keeps reddening after the trim Co-Authored-By: Claude Fable 5.1 * fix(chart): count the collector in the trim and keep the #1144 limits (review on #1032) Review found two holes in the interim trim (backend#2461): 1. The footprint the guard, the embed and the derivation carry INCLUDES the telemetry collector (helm template cannot look up the token Secret, so the DaemonSet renders). With it counted, 750m / 2336Mi left a 2-core / 8 GiB node at 950m / 3062Mi -- 50m and 10Mi short of the smallest envelope. Trim jobs-manager cpu 250m -> 150m (the whole plane idles at 36m) and re-cut the two memory rows below; the render is now 650m / 2272Mi and that node leaves 1050m / 3126Mi. The derivation on values.yaml now says the collector is counted, and why that is the right worst case. 2. pods-monitor was the only row cut without a measurement, and requests == limits made the cut a hard cgroup ceiling. Lower only the REQUESTS on jobs-manager (1Gi -> 640Mi) and pods-monitor (512Mi -> 384Mi) and leave both LIMITS where #1144 put them; the restart peak that justified the limits is exactly the load-run leg not yet measured. Template fallbacks and helm-unittest pins follow. Also fix the four comments the review flagged as stale on the pre-trim numbers (installer fit preamble, schedulability WHAT-IT-ASSERTS list, footprint bats fixture note, Pester positive-control note). Ceilings, embed, mutation needle and manifest regenerated from the render. * test(pester): keep the REDUCE case's claim phrase so mutation-check attributes both REDUCE mutations to it mutation-check.ps1 expects 'the chart footprint drops out of the sum' and 'the reduction is computed but the original size is written' to redden the test named 'is REDUCED, arithmetic printed'. The trim renamed that case to 3b (node derived from the footprint) and dropped the phrase, so both mutations reddened the suite but read MISATTRIBUTED. Put the phrase back in the 3b name; the body is unchanged. * test(pester): write the pre-trim footprint replay as an Int64 literal Bugbot flagged [long](3136 * 1MB) twice as an Int32 overflow. PowerShell promotes the overflowing product to Double rather than wrapping, so the finding was a false positive on both 5.1 and 7 -- but 3136L * 1MB is Int64 from the first operand and leaves nothing to argue about. --------- Co-authored-by: Claude Fable 5.1 --- client/Chart.yaml | 4 +- client/templates/egress-proxy-deployment.yaml | 2 +- client/templates/jobs-manager-deployment.yaml | 24 +++-- client/templates/mysql-deployment.yaml | 6 +- .../templates/requests-proxy-deployment.yaml | 4 +- client/tests/jobs_manager_test.yaml | 12 ++- client/tests/mysql_test.yaml | 6 +- client/tests/requests_proxy_test.yaml | 4 +- client/values.yaml | 69 ++++++++++--- scripts/install-k8s.ps1 | 4 +- scripts/lib/install-client-helm.sh | 19 ++-- scripts/manifest.sha256 | 4 +- scripts/tests/control-plane-footprint.bats | 10 +- scripts/tests/control-plane-footprint.sh | 23 +++-- .../envelope-schedulability-mutations.sh | 2 +- scripts/tests/envelope-schedulability.bats | 18 ++-- scripts/tests/envelope-schedulability.sh | 67 +++++++++---- scripts/tests/install-client-helm.bats | 58 ++++++++--- scripts/tests/install-k8s.Tests.ps1 | 97 ++++++++++++------- 19 files changed, 296 insertions(+), 137 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 15c1fd38..14a392bd 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.111 -appVersion: "1.9.111" +version: 1.9.112 +appVersion: "1.9.112" keywords: - tracebloc - kubernetes diff --git a/client/templates/egress-proxy-deployment.yaml b/client/templates/egress-proxy-deployment.yaml index 792d96ab..2080b26d 100644 --- a/client/templates/egress-proxy-deployment.yaml +++ b/client/templates/egress-proxy-deployment.yaml @@ -66,7 +66,7 @@ spec: resources: requests: cpu: {{ $epReq.cpu | default "50m" | quote }} - memory: {{ $epReq.memory | default "64Mi" | quote }} + memory: {{ $epReq.memory | default "160Mi" | quote }} limits: cpu: {{ $epLim.cpu | default "500m" | quote }} memory: {{ $epLim.memory | default "256Mi" | quote }} diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 19543b4a..ac24ffcc 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -333,13 +333,16 @@ spec: # unresourced-init set, and neither is reachability. # `scripts/tests/qos-reachability.sh` asserts it now. # - # requests.cpu is - # also raised (100m -> 250m) so a mass restart doesn't throttle its - # startup to a crawl (the ~9m recovery seen on lukas-test). + # requests.cpu was raised (100m -> 250m) so a mass restart doesn't throttle + # its startup to a crawl (the ~9m recovery seen on lukas-test); the interim + # trim (backend#2461) brings it to 150m -- the whole plane idles at 36m -- + # and lowers requests.memory to 640Mi while the LIMIT stays 1Gi, so the + # #1144 equality is deliberately broken here: see the derivation on + # values.yaml#resources for why the request moved and the limit did not. resources: requests: - cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }} - memory: {{ .Values.resources.jobsManager.requests.memory | default "1Gi" | quote }} + cpu: {{ .Values.resources.jobsManager.requests.cpu | default "150m" | quote }} + memory: {{ .Values.resources.jobsManager.requests.memory | default "640Mi" | quote }} limits: cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }} memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }} @@ -915,17 +918,20 @@ spec: capabilities: drop: ["ALL"] # #1144: same cluster-recovery treatment as the api container above — - # pods-monitor was also exit-137 killed in the mass restart. Pin + # pods-monitor was also exit-137 killed in the mass restart. #1144 pinned # requests.memory == limits.memory (512Mi) to lower its oom_score_adj # (a less-preferred node OOM victim; the pod stays Burstable — and cpu is # not the only reason: see the init-container note on the api container - # above, backend#2872), and raise + # above, backend#2872), and raised # requests.cpu (50m -> 100m) so its startup is not starved under - # contention. + # contention. The interim trim (backend#2461) lowers the REQUEST to 384Mi + # and keeps the 512Mi LIMIT: nothing has measured this container yet, so + # the ceiling that #1144's restart peak justified stays where it was + # (derivation on values.yaml#resources). resources: requests: cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }} - memory: {{ .Values.resources.podsMonitor.requests.memory | default "512Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.requests.memory | default "384Mi" | quote }} limits: cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }} memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }} diff --git a/client/templates/mysql-deployment.yaml b/client/templates/mysql-deployment.yaml index b3f9dbf7..4a2cbdc4 100644 --- a/client/templates/mysql-deployment.yaml +++ b/client/templates/mysql-deployment.yaml @@ -158,10 +158,10 @@ spec: # PodDisruptionBudget cover the eviction angle instead. resources: requests: - memory: {{ .Values.resources.mysql.requests.memory | default "1Gi" | quote }} - cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }} + memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }} + cpu: {{ .Values.resources.mysql.requests.cpu | default "150m" | quote }} limits: - memory: {{ .Values.resources.mysql.limits.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }} {{- /* CPU limit is intentionally unset by default (see comment block above). Schema permits it, so honour an explicit override here diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index a0f94e60..1ef868ce 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -134,8 +134,8 @@ spec: {{- $rpLim := default dict $rp.limits }} resources: requests: - cpu: {{ $rpReq.cpu | default "100m" | quote }} - memory: {{ $rpReq.memory | default "256Mi" | quote }} + cpu: {{ $rpReq.cpu | default "50m" | quote }} + memory: {{ $rpReq.memory | default "128Mi" | quote }} limits: cpu: {{ $rpLim.cpu | default "1000m" | quote }} memory: {{ $rpLim.memory | default "512Mi" | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 964de569..ef289d79 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -70,17 +70,21 @@ tests: path: spec.template.spec.containers[0].resources.requests - isNotEmpty: path: spec.template.spec.containers[0].resources.limits - # #1144: requests.memory pinned == limits.memory (1Gi api / 512Mi - # pods-monitor) to lower oom_score_adj during a mass restart. + # #1144 pinned requests.memory == limits.memory (1Gi api / 512Mi + # pods-monitor) to lower oom_score_adj during a mass restart. The interim + # trim (backend#2461, client#1032) lowers the REQUESTS (640Mi / 384Mi) and + # keeps the LIMITS (1Gi / 512Mi) -- the limit is the hard ceiling the + # restart peak justified and nothing has re-measured it; the request is + # what the scheduler and admission subtract. Derivation: values.yaml#resources. - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 640Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory value: 1Gi - equal: path: spec.template.spec.containers[1].resources.requests.memory - value: 512Mi + value: 384Mi - equal: path: spec.template.spec.containers[1].resources.limits.memory value: 512Mi diff --git a/client/tests/mysql_test.yaml b/client/tests/mysql_test.yaml index e0d2ad7b..90808612 100644 --- a/client/tests/mysql_test.yaml +++ b/client/tests/mysql_test.yaml @@ -40,13 +40,13 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m + value: 150m # CPU limit deliberately unset — InnoDB lock-wait timeouts under throttling. - notExists: path: spec.template.spec.containers[0].resources.limits.cpu diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml index 19f66ea8..5161e1d0 100644 --- a/client/tests/requests_proxy_test.yaml +++ b/client/tests/requests_proxy_test.yaml @@ -127,10 +127,10 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 100m + value: 50m - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 256Mi + value: 128Mi - equal: path: spec.template.spec.containers[0].resources.limits.cpu value: 1000m diff --git a/client/values.yaml b/client/values.yaml index 6b22572d..4e62469c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -612,7 +612,7 @@ egressProxy: resources: requests: cpu: "50m" - memory: "64Mi" + memory: "160Mi" limits: cpu: "500m" memory: "256Mi" @@ -1355,35 +1355,80 @@ images: # a mass restart does not throttle their startup. Same requests.memory==limits # philosophy mysql already uses (mysql likewise omits its cpu limit). resources: + # INTERIM TRIM (backend#2461, 2026-09-10) -- read before "fixing" any number back up. + # + # Requests are what the scheduler and the jobs-manager's admission SUBTRACT from + # the node; they are not what the process uses. Measured on an installer edge + # (backend#2461): the control plane requested 1050m / 3276Mi while using + # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest + # (#1017) and since client-runtime#544 the admission subtracts the chart's + # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the + # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests, + # minus k3s's own coredns + metrics-server (200m / 140Mi), left 900m / 2390Mi, + # and the smallest training envelope needs 1000m / 3072Mi. Every task on + # staging was refused NEVER_STEADY_STATE (e2e-test-agent fleet 34473120598). + # + # THE FOOTPRINT COUNTS THE COLLECTOR. `helm template` cannot look up the token + # Secret, so the render -- and therefore control-plane-footprint.sh, the + # installer embed and this derivation -- carries the telemetry collector's + # 100m / 128Mi. That is the right worst case: on a real edge the collector IS + # rendered once jobs-manager has written the token Secret (backend#2274), so an + # edge that trains on install must still train after that. With it counted the + # render is 650m / 2272Mi, and the node above leaves 1050m / 3126Mi (was + # 900m / 2390Mi): 50m and 54Mi over the envelope. Thin, and honest. + # + # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom; requests==limits kept + # jobs-manager 250m/1Gi -> 150m/640Mi 168Mi idle (0.16x); LIMIT STAYS 1Gi (see below); cpu: the whole plane idles at 36m + # pods-monitor 100m/512Mi -> 100m/384Mi no measurement yet; LIMIT STAYS 512Mi (see below) + # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) + # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) + # collector 100m/128Mi unchanged unmeasured (distroless, no readable cgroup) -- not trimmed on a guess + # + # WHY jobs-manager AND pods-monitor NOW HAVE requests < limits. #1144 pinned + # requests.memory == limits.memory on both so a cluster-recovery mass restart + # would not pick them as OOM victims; the PODS were Burstable then and still + # are (the unresourced init containers, see above). The trim lowers only the + # REQUEST -- what the scheduler and admission subtract -- and leaves the LIMIT + # where it was, because the limit is the hard cgroup ceiling and the restart + # peak that justifies it is exactly the leg backend#2461's load run has not + # produced yet (review on client#1032). The cost is a higher oom_score_adj + # under node pressure; the alternative was a hard ceiling nothing has measured. + # When the restart leg lands, set BOTH from its p95 and restore the equality. + # + # What this is NOT: the final trim. Re-derive from the load run; do not + # hand-tune here. Guards that read these values: control-plane-footprint.sh + # (ceiling), the generated embed in scripts/lib/install-client-helm.sh + # (scripts/gen-footprint-embed.sh --check), envelope-schedulability.sh. mysql: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "704Mi" limits: - memory: "1Gi" + memory: "704Mi" # No cpu limit — see comment above. jobsManager: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "640Mi" limits: cpu: "1000m" memory: "1Gi" podsMonitor: requests: cpu: "100m" - memory: "512Mi" + memory: "384Mi" limits: cpu: "500m" memory: "512Mi" # requests-proxy serves the Service Bus / backend communication path and - # is mostly idle (a few req/min). 100m/256Mi requests with headroom for - # the occasional burst; revisit if heap profiling shows growth. Schema - # at values.schema.json#resources.requestsProxy validates overrides. + # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi, + # backend#2461) with the limits left at 1000m/512Mi for the occasional burst; + # revisit if heap profiling shows growth. Schema at + # values.schema.json#resources.requestsProxy validates overrides. requestsProxy: requests: - cpu: "100m" - memory: "256Mi" + cpu: "50m" + memory: "128Mi" limits: cpu: "1000m" memory: "512Mi" diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index de49f2e7..b37b578c 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes = 5368709120 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both # installers' values must equal a fresh render of the chart in the same tree. -$script:TbCpFootprintMemBytes = 3288334336 -$script:TbCpFootprintCpuMilli = 900 +$script:TbCpFootprintMemBytes = 2382364672 +$script:TbCpFootprintCpuMilli = 650 # ── end generated footprint ───────────────────────────────────────────────── # Set by Get-TrainingResources when the machine is readable but below the diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 65c45a07..5e55501b 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in # `make drift` (the required Source-of-truth drift job): the value below must # equal a fresh render of the chart in the same tree, or CI reddens. -_TB_CP_FOOTPRINT_MEM_BYTES=3288334336 -_TB_CP_FOOTPRINT_CPU_MILLI=900 +_TB_CP_FOOTPRINT_MEM_BYTES=2382364672 +_TB_CP_FOOTPRINT_CPU_MILLI=650 # ── end generated ─────────────────────────────────────────────────────────── # ── the fallback training envelope (precedence step 4) ────────────────────── @@ -724,12 +724,17 @@ _training_provenance() { # # Everything above sizes the envelope against ALLOCATABLE and stops. Nothing then # asked whether the number written can be scheduled beside what else runs on the -# node: the chart's own control plane (3136 MiB / 900 m, from the render) plus -# the distribution's system pods (~140 MiB / 200 m of coredns and metrics-server -# on k3s). Measured on a real single-node install, the platform out-requests the -# 3 GiB reserve the envelope subtracts, so `allocatable - 3 GiB` over-asked at -# EVERY machine size and the training pod sat `Pending / Insufficient memory` +# node: the chart's own control plane (the embedded _TB_CP_FOOTPRINT_* constants, +# generated from the render by scripts/gen-footprint-embed.sh) plus the +# distribution's system pods (~140 MiB / 200 m of coredns and metrics-server on +# k3s). When this was written the chart requested 3136 MiB / 900 m -- MORE than +# the 3 GiB reserve the envelope subtracts -- so `allocatable - 3 GiB` over-asked +# at EVERY machine size and the training pod sat `Pending / Insufficient memory` # while the installer printed `Training size: ...` and cli doctor said OK. +# backend#2461's interim trim (chart 1.9.112) brought the control plane under the +# reserve, so today the fit below is a no-op on every golden vector; it stays +# because the constants are generated, not promised, and the next chart that +# out-requests the reserve should be reduced here rather than sit Pending again. # # The two functions below make the installer own that question at the moment it # writes the envelope. The 3 GiB constant itself is not touched here: fixing the diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 9f3572bc..22c38328 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gp d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea scripts/lib/setup-linux.sh 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c scripts/lib/cluster.sh 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62 scripts/lib/gpu-plugins.sh -320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a scripts/lib/install-client-helm.sh +9a473b2d11545183be0bb24a1446eb8a51bd6a525d5d08dcd09a574f3bd10e06 scripts/lib/install-client-helm.sh 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943 scripts/lib/assess.sh e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b scripts/lib/probe.sh 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd scripts/lib/summary.sh 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e scripts/lib/diagnose.sh -b2a223a1e819e7056f27707e9e5bd25c6c4bdd830212806240a2559e71f1761d scripts/install-k8s.ps1 +07722bab739746746f85504867ba3d32d85f9bc39f318b0789880bb7d6c13548 scripts/install-k8s.ps1 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5 scripts/lib/telemetry.ps1 diff --git a/scripts/tests/control-plane-footprint.bats b/scripts/tests/control-plane-footprint.bats index 50c61308..f65020ee 100644 --- a/scripts/tests/control-plane-footprint.bats +++ b/scripts/tests/control-plane-footprint.bats @@ -75,7 +75,7 @@ YAML # guard hands to backend#2460/#2461. # # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the - # live render the number is 3136 MiB either way -- the chart's init containers + # live render the number is the same either way -- the chart's init containers # carry no requests -- so the fix is INERT there and a green run proves nothing # about the formula. Here the two answers differ, deliberately: # @@ -162,7 +162,13 @@ YAML } @test "guard: RATCHET reddens when the footprint would exceed the ceiling" { - TB_CP_FOOTPRINT_MEM_CEIL=3000 run bash "$GUARD" + # The ceiling is DERIVED from the live render (one MiB under it), so this test + # keeps reddening whatever the chart requests -- a written-down 3000 stopped + # breaching once backend#2461 trimmed the render from 3136 to 2336 MiB. + local mib + mib="$(bash "$GUARD" --print-footprint 2>/dev/null | cut -d' ' -f1)" + [[ "$mib" =~ ^[0-9]+$ ]] || { echo "could not derive the render footprint: '$mib'"; return 1; } + TB_CP_FOOTPRINT_MEM_CEIL=$(( mib - 1 )) run bash "$GUARD" [ "$status" -eq 1 ] || { echo "expected exit 1, got $status: $output"; return 1; } printf '%s\n' "$output" | grep -q 'exceed the recorded ceiling' || { echo "$output"; return 1; } } diff --git a/scripts/tests/control-plane-footprint.sh b/scripts/tests/control-plane-footprint.sh index a82ecfa3..037b9f4e 100755 --- a/scripts/tests/control-plane-footprint.sh +++ b/scripts/tests/control-plane-footprint.sh @@ -14,11 +14,14 @@ # constant still covers it. Grep the three consumers for `3008`, `platform # footprint`, `control plane requests`: zero hits. The number was invisible. # -# Measured here by rendering the chart: the steady-state control plane requests -# ~3136 MiB, already ABOVE the 3 GiB (3072 MiB) the envelope reserves for it. -# That 64 MiB overshoot is the memory half of the reason a training pod on a -# freshly-installed single-node edge can sit `Pending / Insufficient memory` -# (backend#2870). CPU fits: 900m requested against a 1000m reserve. +# Measured here by rendering the chart: the steady-state control plane requested +# ~3136 MiB / 900 m before backend#2461's interim trim (2026-09-10) -- ABOVE the +# 3 GiB (3072 MiB) the envelope reserved for it; 2272 MiB / 650 m after, under it. +# That 64 MiB overshoot was the memory half of the reason a training pod on a +# freshly-installed single-node edge could sit `Pending / Insufficient memory` +# (backend#2870). The render COUNTS THE TELEMETRY COLLECTOR (helm template cannot +# look up the token Secret, so the collector renders): that is the worst case a +# real edge reaches once the Secret exists, and the one the ceiling must hold. # # WHAT THIS GUARD DOES, AND DELIBERATELY DOES NOT # ----------------------------------------------- @@ -70,8 +73,8 @@ installer="$root/scripts/lib/install-client-helm.sh" # same PR -- which is the moment to weigh whether the training envelope can still # afford it. Overridable so the guard's own test can drive a lower ceiling and # watch a real render breach it. -MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-3136}" -CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-900}" +MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2272}" +CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-650}" command -v helm >/dev/null 2>&1 || { echo "[ERROR] helm is required to render the chart footprint" >&2; exit 3; } command -v python3 >/dev/null 2>&1 || { echo "[ERROR] python3 is required to sum the rendered requests" >&2; exit 3; } @@ -169,9 +172,9 @@ try: # and cpu, so a pod can take its memory from the init side and its cpu # from the app side. # - # THE NUMBER DOES NOT MOVE ON THIS CHART -- 3136 MiB / 900 m either way, - # because these init containers carry no requests. So the "64 MiB OVER" - # finding stands; the method was wrong, the conclusion was not. It also + # THE NUMBER DOES NOT MOVE ON THIS CHART -- the render is the same + # either way, because these init containers carry no requests. So the + # original "64 MiB OVER" finding stood; the method was wrong, the conclusion was not. It also # means the fix is INERT on the real render, which is why the bats # fixture is built so the two formulas disagree. def _req(c, key): diff --git a/scripts/tests/envelope-schedulability-mutations.sh b/scripts/tests/envelope-schedulability-mutations.sh index 89abee65..09600db6 100644 --- a/scripts/tests/envelope-schedulability-mutations.sh +++ b/scripts/tests/envelope-schedulability-mutations.sh @@ -110,7 +110,7 @@ run_case "unreadable cluster writes a carried non-floor size" "$LIB" \ printf '\nthe embed:\n' run_case "embedded footprint drifts from the render" "$LIB" \ - '_TB_CP_FOOTPRINT_MEM_BYTES=3288334336' \ + '_TB_CP_FOOTPRINT_MEM_BYTES=2382364672' \ '_TB_CP_FOOTPRINT_MEM_BYTES=3221225472' 1 printf '\n%s passed, %s failed\n' "$pass" "$fail" diff --git a/scripts/tests/envelope-schedulability.bats b/scripts/tests/envelope-schedulability.bats index 25e1e706..e65449b9 100644 --- a/scripts/tests/envelope-schedulability.bats +++ b/scripts/tests/envelope-schedulability.bats @@ -171,9 +171,12 @@ YAML # It walked the golden vectors -- a guard that passes having checked nothing is # the class this repo catalogues. The table has 13 rows today; assert a floor, # not the exact count, so a vector added upstream does not redden this. - [[ "$output" =~ ok\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } - [ "${BASH_REMATCH[2]}" -ge 10 ] || { echo "only ${BASH_REMATCH[2]} measurable vectors"; return 1; } - [ "${BASH_REMATCH[1]}" -ge 1 ] || return 1 + # Two legitimate shapes: `ok positive control: N/M` while the platform out-requests + # the reserve, or `note positive control: N/M ... does not apply` once it fits + # inside it (backend#2461). Either way the guard must have walked the table. + [[ "$output" =~ (ok|note)\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } + [ "${BASH_REMATCH[3]}" -ge 10 ] || { echo "only ${BASH_REMATCH[3]} measurable vectors"; return 1; } + if [[ "${BASH_REMATCH[1]}" == "ok" ]]; then [ "${BASH_REMATCH[2]}" -ge 1 ] || return 1; fi } @test "envelope-schedulability.sh: CANNOT TELL (exit 2) when the tree has no chart to derive from" { @@ -360,10 +363,11 @@ for key, kind in (("cpu_to_milli", "cpu"), ("mem_to_bytes", "mem")): } helm() { return 1; } unset TRACEBLOC_TRAINING_RESOURCES - # ONE system pod here (70Mi / 100m), so cpu fits (3000 + 900 + 100 = 4000) and - # only memory reduces (5120 + 3136 + 70 > 8192 -> 4 GiB): cpu=3,memory=4Gi. - # Anything else in the capture -- a stray echo -- would corrupt the value. + # ONE system pod here (70Mi / 100m). Since backend#2461's trim the resolver's + # cpu=3,memory=5Gi fits (5120 + 2336 + 70 <= 8192; 3000 + 750 + 100 <= 4000), so + # the fit writes it unchanged. Anything else in the capture -- a stray echo -- + # would corrupt the value, which is what this test is for. local captured captured="$(_resolve_training_size; _fit_training_envelope; printf '%s' "$_TB_TRAINING_SIZE")" - [ "$captured" = "cpu=3,memory=4Gi" ] || { echo "captured '$captured'"; return 1; } + [ "$captured" = "cpu=3,memory=5Gi" ] || { echo "captured '$captured'"; return 1; } } diff --git a/scripts/tests/envelope-schedulability.sh b/scripts/tests/envelope-schedulability.sh index 1e552818..d22e04c1 100644 --- a/scripts/tests/envelope-schedulability.sh +++ b/scripts/tests/envelope-schedulability.sh @@ -7,11 +7,15 @@ # --------------- # backend#2870: the installer sized the training envelope as `allocatable - 3 GiB` # and nothing then asked whether that number could be scheduled beside what the -# chart installs. It could not: the control plane requests 3136 MiB (from the -# render) against the 3072 MiB reserve, so every install over-asked and the +# chart installs. It could not: the control plane then requested 3136 MiB (from +# the render) against the 3072 MiB reserve, so every install over-asked and the # training pod sat Pending. No test in this repo could see it, because every # test compared the installers to the CONTRACT and none compared the arithmetic -# to a CLUSTER. +# to a CLUSTER. backend#2461's interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve -- so the over-ask no longer occurs on a +# real node and the positive control below reports that instead of failing; +# the reduce/refuse paths are exercised on nodes DERIVED from the footprint +# (SMALL_MI / TINY_MI), never a written-down machine size. # # WHAT IT ASSERTS, and how it derives rather than restates (backend#1729 rule 1) # ---------------------------------------------------------------------------- @@ -23,7 +27,9 @@ # 1. the installer's embedded footprint constants equal that render; # 2. for EVERY single-node golden vector in the contract table, the envelope # written + chart footprint + system pods <= allocatable, on memory AND cpu; -# 3. the ticket's 8 GiB reproduction is REDUCED, with the arithmetic; +# 3. the ticket's 8 GiB reproduction now FITS unreduced (backend#2461 brought +# the footprint under the reserve), and a node sized from the live +# footprint to over-ask is REDUCED, with the arithmetic (3b); # 4. a machine where not even a 1-core / 1-GiB run fits is REFUSED; # 5. a cpu-only overshoot reduces cpu alone (DoD part 5: cover cpu too); # 6. a human's pin is never altered, only warned; @@ -190,32 +196,56 @@ if (( NEED_B > _TB_ENVELOPE_OVERHEAD_MEM_BYTES || NEED_M > _TB_ENVELOPE_OVERHEAD bad "positive control: the platform out-requests the reserve, yet no vector over-asked before the fit -- this guard cannot see the defect it exists for" fi else - echo " note the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve; the fit is a no-op by construction and the positive control does not apply" + echo " note positive control: ${over_before}/${checked} measurable vectors over-asked -- the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve, so the fit is a no-op by construction and the control does not apply (backend#2461)" fi -# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. +# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. Before backend#2461 it over-asked +# (5120 + 3136 + 140 > 8192) and was REDUCED; after the trim it fits unreduced. +# Pinning the fit is the regression guard: a chart change that pushes the +# footprint back over the reserve reddens here first. resolve_and_fit "4 8Gi" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]]; then - ok "8 GiB reproduction: ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" +if [[ "$_TB_FIT_VERDICT" == "fits" && "$_TB_TRAINING_SIZE" == "$BEFORE" ]]; then + ok "8 GiB reproduction: ${BEFORE} fits unreduced beside $(( NEED_B / MIB )) MiB / ${NEED_M} m (backend#2461 closed the over-ask)" +else + bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want fits, unreduced)" +fi + +# 3b. THE REDUCE PATH still has to be exercised, so the node is DERIVED from the +# footprint: below 4 GiB the resolver falls to the 1-core / 2-GiB floor, and +# SMALL_MI leaves room for 1 GiB but not for that floor, so the fit must +# reduce to 1 GiB with the arithmetic on screen. Moves with the render. +SMALL_MI=$(( NEED_B / MIB + 1024 + 64 )) +resolve_and_fit "4 ${SMALL_MI}Mi" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]] && [[ "$_TB_TRAINING_SIZE" == "cpu=1,memory=1Gi" ]]; then + ok "${SMALL_MI} MiB node (derived): ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" printf '%s\n' "$_TB_FIT_LINES" | sed 's/^/ /' else - bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" + bad "${SMALL_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want reduced to cpu=1,memory=1Gi)" fi -# 4. REFUSAL when not even 1 core / 1 GiB fits. -resolve_and_fit "4 4Gi" +# 4. REFUSAL when not even 1 core / 1 GiB fits -- TINY_MI is derived so that +# allocatable minus the platform is 64 MiB short of a 1 GiB run. +TINY_MI=$(( NEED_B / MIB + 1024 - 64 )) +resolve_and_fit "4 ${TINY_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "refused" && "$_TB_FIT_LINES" == *"not even a 1-core / 1-GiB run"* ]]; then - ok "4 GiB node: refused, with the arithmetic" + ok "${TINY_MI} MiB node (derived): refused, with the arithmetic" else - bad "4 GiB node: verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" + bad "${TINY_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" fi -# 5. CPU-ONLY overshoot: memory already fits (the GiB floor absorbed it), cpu does not. +# 5. CPU-ONLY overshoot: memory already fits, cpu does not. The resolver leaves +# one core for the platform and the trimmed platform needs 950 m, so no real +# node can produce this any more -- the pre-trim footprint could. The case +# keeps DoD part 5 covered by REPLAYING that footprint (900 m, the value the +# embed carried before backend#2461) through the real fit; the memory side is +# untouched so only cpu can move. +saved_cpu="$_TB_CP_FOOTPRINT_CPU_MILLI"; _TB_CP_FOOTPRINT_CPU_MILLI=900 resolve_and_fit "8 63928Mi" +_TB_CP_FOOTPRINT_CPU_MILLI="$saved_cpu" a_mem="$(env_mem_b "$_TB_TRAINING_SIZE")"; b_mem="$(env_mem_b "$BEFORE")" a_cpu="$(env_cpu_m "$_TB_TRAINING_SIZE")"; b_cpu="$(env_cpu_m "$BEFORE")" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + NEED_M <= 8000 )); then - ok "cpu-only overshoot: ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + 900 + SYS_M <= 8000 )); then + ok "cpu-only overshoot (pre-trim 900 m replayed): ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" else bad "cpu-only overshoot: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" fi @@ -237,9 +267,10 @@ else bad "measured system pods: $(( ${_TB_SYS_MEM_BYTES:-0} / MIB )) MiB / ${_TB_SYS_CPU_MILLI:-0} m (want ${SYS_MIB} / ${SYS_M}); note: ${_TB_SYS_NOTE:-}" fi -# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces. +# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces on the +# derived SMALL_MI node (the 8 GiB machine no longer needs reducing). PODS_READABLE=0 -resolve_and_fit "4 8Gi" +resolve_and_fit "4 ${SMALL_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "reduced" && "$_TB_FIT_LINES" == *"NOT measured"*"chart derivation only"* ]]; then ok "pod list unreadable: verified against the chart derivation only, and said so (${BEFORE} -> ${_TB_TRAINING_SIZE})" else diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index 0e3a7289..33801007 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -1378,12 +1378,16 @@ _override_release_ctx() { # ── the envelope is checked against what the node can actually schedule (backend#2870) ── # -# The resolver sizes against ALLOCATABLE. The chart's own control plane requests -# 3136 MiB / 900 m (from the render) and k3s's system pods ~140 MiB / 200 m, so -# `allocatable - 3 GiB` over-asked on every machine and the training pod sat -# Pending. These three drive the WHOLE install flow so the written values file -# is what is asserted -- the unit-level arithmetic lives in -# scripts/tests/envelope-schedulability.sh and .bats. +# The resolver sizes against ALLOCATABLE. Before backend#2461 the chart's own +# control plane requested 3136 MiB / 900 m (from the render) and k3s's system +# pods ~140 MiB / 200 m, so `allocatable - 3 GiB` over-asked on every machine and +# the training pod sat Pending. The interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve, so an 8 GiB node now fits unreduced -- and +# the reduce/refuse paths are exercised on nodes DERIVED from the embedded +# footprint (`_TB_CP_FOOTPRINT_MEM_BYTES`, sourced with the library), so these +# tests follow the number instead of restating it. These drive the WHOLE install +# flow so the written values file is what is asserted -- the unit-level +# arithmetic lives in scripts/tests/envelope-schedulability.sh and .bats. _sched_cluster() { # $1 = node line(s) kubectl() { case "$*" in @@ -1395,8 +1399,10 @@ _sched_cluster() { # $1 = node line(s) } _SCHED_NODES="$1" } +# platform = embedded chart footprint + the two canned system pods (140 MiB / 200 m) +_sched_need_mib() { echo $(( _TB_CP_FOOTPRINT_MEM_BYTES / 1048576 + 140 )); } -@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { +@test "install_client_helm: the ticket's 8 GiB node now FITS unreduced (backend#2870 closed by backend#2461)" { HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" _ensure_tracebloc_dirs() { :; } _ensure_release_dirs() { :; } @@ -1405,17 +1411,38 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # The ticket's reproduction: an 8 GiB node. The resolver says cpu=3,memory=5Gi; - # 5120 + 3136 + 140 = 8396 MiB > 8192, and 3000 + 900 + 200 = 4100 m > 4000. + # 5120 + platform <= 8192 MiB and 3000 + 950 <= 4000 m: the resolver's + # cpu=3,memory=5Gi is written as-is, and nothing is reported OVER. _sched_cluster '4 8Gi' run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] || { echo "$output"; return 1; } - grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } - grep -q 'RESOURCE_LIMITS: "memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 + grep -q 'RESOURCE_REQUESTS: "cpu=3,memory=5Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + [[ "$output" != *"OVER"* ]] || { echo "$output"; return 1; } +} + +@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + has() { return 0; } + verify_credentials() { printf valid; } + unset TRACEBLOC_TRAINING_RESOURCES + # A node DERIVED from the footprint: below 4 GiB the resolver falls to the + # 1-core / 2-GiB floor; this node leaves room for 1 GiB but not 2, so the fit + # must reduce to memory=1Gi and print the OVER arithmetic. + local need small over + need="$(_sched_need_mib)"; small=$(( need + 1024 + 64 )); over=$(( 2048 + need - small )) + _sched_cluster "4 ${small}Mi" + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + grep -q 'RESOURCE_REQUESTS: "cpu=1,memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + grep -q 'RESOURCE_LIMITS: "memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 grep -q 'RESOURCE_PROVENANCE: "installer"' "$HOST_DATA_DIR/values.yaml" || return 1 # The arithmetic is on screen, not just the verdict. - [[ "$output" == *"204 MiB OVER"* ]] || { echo "$output"; return 1; } - [[ "$output" == *"reduced cpu=3,memory=5Gi -> cpu=2,memory=4Gi"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"${over} MiB OVER"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"reduced cpu=1,memory=2Gi -> cpu=1,memory=1Gi"* ]] || { echo "$output"; return 1; } } @test "install_client_helm: REFUSES to write an envelope when not even a 1-core/1-GiB run fits (backend#2870)" { @@ -1427,8 +1454,9 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # 4 GiB allocatable: 4096 - 3136 - 140 = 820 MiB is all that is left. - _sched_cluster '4 4Gi' + # Derived: allocatable minus the platform is 64 MiB short of a 1 GiB run. + local tiny; tiny=$(( $(_sched_need_mib) + 1024 - 64 )) + _sched_cluster "4 ${tiny}Mi" run install_client_helm <<< $'myid\nmypw' [ "$status" -ne 0 ] || { echo "$output"; return 1; } [[ "$output" == *"Refusing to write a training envelope"* ]] || { echo "$output"; return 1; } diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 50804852..bfbd2aee 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9780,41 +9780,62 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } $failures -join "`n" | Should -BeNullOrEmpty $checked | Should -BeGreaterOrEqual 10 -Because "a guard that checked almost nothing proves almost nothing" - # POSITIVE CONTROL: the platform out-requests the reserve today, so at least - # one vector must over-ask BEFORE the fit -- else this suite could never have - # seen the defect it exists for. + # POSITIVE CONTROL: while the platform out-requested the reserve (before + # backend#2461's trim) at least one vector had to over-ask BEFORE the fit -- + # else this suite could never have seen the defect it exists for. The guard is + # conditional on the embedded footprint, so it re-arms by itself the day a + # chart out-requests the reserve again. if ($needB -gt $script:TbEnvelopeOverheadMemBytes -or $needM -gt $script:TbEnvelopeOverheadCpuMilli) { $overBefore | Should -BeGreaterThan 0 -Because "the platform out-requests the reserve, yet no vector over-asked before the fit" } } - It "3. the ticket's reproduction: an 8 GiB node is REDUCED, arithmetic printed" { + It "3. the ticket's reproduction: an 8 GiB node now FITS unreduced (backend#2461 closed the over-ask)" { + # Before the interim trim the platform asked 3136 MiB / 900 m and this node was + # REDUCED to cpu=2,memory=4Gi; at 2336 MiB / 750 m the resolver's cpu=3,memory=5Gi + # fits: 5120 + 2476 <= 8192 and 3000 + 950 <= 4000. Pinning the fit is the + # regression guard for a chart change that pushes the footprint back up. $r = Invoke-FitOn -Nodes @('4 8Gi') + $r.Fit.Verdict | Should -Be 'fits' + $r.Fit.Size | Should -Be $r.Before + } + It "3b. a node DERIVED from the footprint (moves with the render) is REDUCED, arithmetic printed" { + # Below 4 GiB the resolver falls to the 1-core / 2-GiB floor; this node leaves + # room for 1 GiB but not for that floor, so the fit must reduce to 1 GiB with + # the OVER arithmetic on screen. + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*OVER*' ($r.Fit.Lines -join "`n") | Should -BeLike "*reduced $($r.Before) -> $($r.Fit.Size)*" - # The numbers on this node against the canned cluster: cpu 3000+900+200 = 4100 - # > 4000 -> 2 cores; memory 5120+3136+140 = 8396 > 8192 -> 8192-3276 = 4916 MiB - # -> 4 GiB. Both dimensions over-asked, both reduced. - $r.Fit.Size | Should -Be 'cpu=2,memory=4Gi' + $r.Fit.Size | Should -Be 'cpu=1,memory=1Gi' } - - It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic" { - $r = Invoke-FitOn -Nodes @('4 4Gi') + It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic (node derived)" { + # allocatable minus the platform is 64 MiB short of a 1 GiB run + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $tiny = $needMib + 1024 - 64 + $r = Invoke-FitOn -Nodes @("4 ${tiny}Mi") $r.Fit.Verdict | Should -Be 'refused' ($r.Fit.Lines -join "`n") | Should -BeLike '*not even a 1-core / 1-GiB run*' } - - It "5. cpu-only overshoot reduces cpu alone (memory kept)" { - $r = Invoke-FitOn -Nodes @('8 63928Mi') - $r.Fit.Verdict | Should -Be 'reduced' - (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) - $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) - $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) - $aCpu | Should -BeLessThan $bCpu - ($aCpu + $script:TbCpFootprintCpuMilli + $script:SysM) | Should -BeLessOrEqual 8000 + It "5. cpu-only overshoot reduces cpu alone (memory kept) -- the pre-trim 900 m replayed" { + # The resolver leaves one core for the platform and the trimmed platform needs + # 950 m, so no real node produces a cpu-only overshoot any more; the pre-trim + # footprint (900 m) did. Replaying it through the real fit keeps DoD part 5 + # covered; memory is untouched so only cpu can move. + $saved = $script:TbCpFootprintCpuMilli + try { + $script:TbCpFootprintCpuMilli = 900 + $r = Invoke-FitOn -Nodes @('8 63928Mi') + $r.Fit.Verdict | Should -Be 'reduced' + (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) + $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) + $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) + $aCpu | Should -BeLessThan $bCpu + ($aCpu + 900 + $script:SysM) | Should -BeLessOrEqual 8000 + } finally { $script:TbCpFootprintCpuMilli = $saved } } - It "6. a human's pin is warned, never altered (pinned-over)" { $r = Invoke-FitOn -Nodes @('4 8Gi') -Override 'cpu=4,memory=16Gi' $r.Prov | Should -Be 'user' @@ -9836,7 +9857,10 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "7b. pods unreadable: verified against the chart derivation only, and it SAYS so; still reduces" { - $r = Invoke-FitOn -Nodes @('4 8Gi') -PodsReadable $false + # on the derived small node -- the 8 GiB machine no longer needs reducing + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") -PodsReadable $false $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*NOT measured*chart derivation only*' } @@ -9867,19 +9891,22 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "the reduction below the contract floor flags Undersized, and a fit does not" { - # 4c/5Gi: the resolver leaves cpu=3,memory=2Gi (5120-3072 = 2048 MiB, the - # floor exactly). Memory: 2048+3136+140 > 5120 -> 5120-3276 = 1844 MiB -> 1 GiB, - # below the 2 GiB floor -> Undersized. Cpu: 4000-1100 = 2900 m -> 2 cores, so - # the run is still a requestable shape and is REDUCED, not refused. (On a - # 2-core node the same memory would reduce but cpu would leave 900 m -- under - # one core -- and the verdict is refused; that is case 4's territory.) - $r = Invoke-FitOn -Nodes @('4 5Gi') - $r.Fit.Verdict | Should -Be 'reduced' - $r.Fit.Size | Should -Be 'cpu=2,memory=1Gi' - $r.Fit.Undersized | Should -BeTrue - # A genuine fit: an installer-sized envelope ALWAYS over-asks today (the - # resolver leaves 3 GiB and the platform asks 3136 MiB plus system pods -- the - # positive control above), so use a carried installer size that fits. + # A reduction that lands BELOW the 2 GiB floor needs the resolver to start at + # or above the floor (node >= 5 GiB) AND the platform to push it under -- which + # the trimmed 2336 MiB no longer can (5120 - 2476 = 2644 MiB >= 2 GiB). Replay + # the pre-trim footprint (3136 MiB) through the real fit, as case 5 does for + # cpu: 4c/5Gi -> cpu=3,memory=2Gi; memory 2048+3136+140 > 5120 -> 5120-3276 = + # 1844 MiB -> 1 GiB, below the floor -> Undersized. Cpu: 4000-950 = 3050 m -> + # 3 cores, still requestable, so REDUCED, not refused. + $saved = $script:TbCpFootprintMemBytes + try { + $script:TbCpFootprintMemBytes = 3136L * 1MB # Int64 literal: no Int32 product to argue about (Bugbot, twice) + $r = Invoke-FitOn -Nodes @('4 5Gi') + $r.Fit.Verdict | Should -Be 'reduced' + $r.Fit.Size | Should -Be 'cpu=3,memory=1Gi' + $r.Fit.Undersized | Should -BeTrue + } finally { $script:TbCpFootprintMemBytes = $saved } + # A genuine fit, on the real footprint: a carried installer size that fits. $r2 = Invoke-FitOn -Nodes @('16 64Gi') -Carried @{ Size = 'cpu=4,memory=12Gi'; Provenance = 'installer' } $r2.Before | Should -Be 'cpu=4,memory=12Gi' $r2.Fit.Verdict | Should -Be 'fits'