diff --git a/.github/workflows/standard-checks.yml b/.github/workflows/standard-checks.yml index d67907bd..8d59b4f2 100644 --- a/.github/workflows/standard-checks.yml +++ b/.github/workflows/standard-checks.yml @@ -187,6 +187,12 @@ jobs: # `bash -e`, so a bare test returning false on the last iteration would # abort the script before the ::error:: line ever printed. run: | + # ubuntu-latest pre-bakes google-chrome.list (and, on current images, the deb822 google-chrome.sources) into the apt sources, so + # every `apt-get update` here depends on dl.google.com serving + # consistent index metadata. On 2026-09-09 it did not ("Hash Sum + # mismatch", ~10 minutes) and all three attempts below lost on main for + # a repository this job never reads. Drop it before the first update. + sudo rm -f /etc/apt/sources.list.d/google-chrome* for i in 1 2 3; do if sudo timeout 60 apt-get update -qq \ && sudo timeout 60 apt-get install -y bats; then diff --git a/client/Chart.yaml b/client/Chart.yaml index de42aa09..14a392bd 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.110 -appVersion: "1.9.110" +version: 1.9.112 +appVersion: "1.9.112" keywords: - tracebloc - kubernetes diff --git a/client/templates/NOTES.txt b/client/templates/NOTES.txt index 4b2472f8..c06bfad3 100644 --- a/client/templates/NOTES.txt +++ b/client/templates/NOTES.txt @@ -13,7 +13,7 @@ {{ "\033[1;34m" }}Service Account:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.serviceAccountName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}Secret:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.secretName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}RBAC scope:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ if ne .Values.clusterScope false }}Cluster{{ else }}Namespace{{ end }}{{ "\033[0m" }} - {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}docker.io{{ "\033[0m" }} + {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}{{ include "tracebloc.tbRegistry" . }}{{ "\033[0m" }} (tracebloc control-plane images) {{- if (default dict .Values.hostPath).enabled }} {{ "\033[1;34m" }}Storage:{{ "\033[0m" }} {{ "\033[0;33m" }}hostPath (bare-metal){{ "\033[0m" }} {{ "\033[1;34m" }}Host dirs:{{ "\033[0m" }} {{ "\033[0;33m" }}/tracebloc/data, /tracebloc/logs, /tracebloc/mysql (on the node){{ "\033[0m" }} diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index ed7d5292..4b8e2e69 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -463,10 +463,12 @@ true the kubelet pulls a digest it has never seen. Two conditions: * the CronJob renders at all (`imageRefresh.enabled`, and not every refreshed image already pinned), and - * images come from docker.io. Under a `global.imageRegistry` mirror the - script goes inert by design — it resolves digests from docker.io, and - pinning one onto a mirrored reference could pin an image the mirror - does not hold. + * the images come from a registry the script can resolve digests on + anonymously (tracebloc.imageRefreshResolvableRegistries: docker.io, + ghcr.io — wherever tracebloc.tbRegistry points). Under a + `global.imageRegistry` mirror the script goes inert by design — it + cannot resolve there, and pinning a digest resolved elsewhere onto a + mirrored reference could pin an image the mirror does not hold. 3. Otherwise -> Always. No reconcile, no pin, so a floating tag plus a restart is the ONLY way that edge can ever move. This is exactly the pre-#569 behaviour, kept for exactly the edges that still depend on it: @@ -483,10 +485,9 @@ true Usage: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" $d "root" $) }} */}} {{- define "tracebloc.controlPlanePullPolicy" -}} -{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} {{- if .digest -}} IfNotPresent -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (include "tracebloc.imageRefreshResolvable" .root) -}} IfNotPresent {{- else -}} Always @@ -584,6 +585,73 @@ with the ROOT context (e.g. `include "tracebloc.mirrorPrefix" $`). {{- with (dig "imageRegistry" "" (.Values.global | default dict)) }}{{ . }}/{{ end -}} {{- end -}} +{{/* +tracebloc.imageRefreshResolvableRegistries — the registries the image-refresh +script can resolve a floating tag to a digest on WITHOUT a credential: the +public token endpoints it knows (`get_token` / `get_latest_digest` in +image-refresh-cronjob.yaml carry one arm per entry). Space-separated, ONE +declaration: `tracebloc.imageRefreshResolvable` (below) and the +`tracebloc.controlPlanePullPolicy` decision both read it, and the CronJob +renders the verdict into the script's env as IMAGE_REGISTRY_RESOLVABLE — so the +pods' pull policy and the script's "can I reconcile here" guard cannot disagree. +Adding a registry here without a matching token arm in the script would make +the pods IfNotPresent while every tick WARNs "unknown registry"; the unit tests +pin the arms to this list. +*/}} +{{- define "tracebloc.imageRefreshResolvableRegistries" -}} +docker.io ghcr.io +{{- end -}} + +{{/* +tracebloc.tbRegistry — the registry the tracebloc-PUBLISHED control-plane images +(tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor, and +the requests-proxy, which runs the jobs-manager image) are pulled from. + +ONE precedence chain, so the four call sites, the image-refresh CronJob and +NOTES.txt cannot disagree about where those images live: + + 1. `global.imageRegistry` — a private mirror re-homes EVERY image the + chart pulls (#585), tracebloc/* included. + It always wins. + 2. `images.traceblocRegistry` — the tracebloc-only knob: moves just the + tracebloc-published images, leaving busybox, + squid, alpine/*, the device plugins and the + ingestor where they are. Also the per-edge + rollback: set it to the previous registry. + 3. "docker.io" — the chart default while the images are + published to Docker Hub. + +NOT routed through here, on purpose: `tracebloc/mysql-client` (frozen, +digest-pinned, published only to Docker Hub — see images.mysqlClient), the +third-party images (each has its own `registry` key), and — for now — the +training-image host JOB_IMAGE_HOST, which moves in its own step. + +Every read is nil-guarded and `| default`-chained: values.yaml ships +`global.imageRegistry: ""` (the key EXISTS, so `dig`'s own fallback never +applies — the trap image_refresh_test.yaml pins), and an edge upgrading with +`--reuse-values` from before `images.traceblocRegistry` existed has no such key +at all. Both must render the default, never "". + +Call with the ROOT context: {{ include "tracebloc.tbRegistry" . }} +*/}} +{{- define "tracebloc.tbRegistry" -}} +{{- $mirror := dig "imageRegistry" "" (.Values.global | default dict) -}} +{{- $own := dig "traceblocRegistry" "" (.Values.images | default dict) -}} +{{- $mirror | default ($own | default "docker.io") -}} +{{- end -}} + +{{/* +tracebloc.imageRefreshResolvable — "true" when the image-refresh script can +reconcile on this edge: the registry tracebloc.tbRegistry resolves to is one of +tracebloc.imageRefreshResolvableRegistries. Empty otherwise (a private mirror, +or a registry the script has no token arm for). Call with the ROOT context. +*/}} +{{- define "tracebloc.imageRefreshResolvable" -}} +{{- if has (include "tracebloc.tbRegistry" .) (splitList " " (include "tracebloc.imageRefreshResolvableRegistries" .)) -}} +true +{{- end -}} +{{- end -}} + {{/* tracebloc.ingestorDigest — the ONE effective digest for the spawned ingestor image. Renders the digest to pin to, or nothing at all to float on diff --git a/client/templates/egress-proxy-deployment.yaml b/client/templates/egress-proxy-deployment.yaml index 792d96ab..2080b26d 100644 --- a/client/templates/egress-proxy-deployment.yaml +++ b/client/templates/egress-proxy-deployment.yaml @@ -66,7 +66,7 @@ spec: resources: requests: cpu: {{ $epReq.cpu | default "50m" | quote }} - memory: {{ $epReq.memory | default "64Mi" | quote }} + memory: {{ $epReq.memory | default "160Mi" | quote }} limits: cpu: {{ $epLim.cpu | default "500m" | quote }} memory: {{ $epLim.memory | default "256Mi" | quote }} diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index d664eab1..d57bdaae 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -12,8 +12,10 @@ data: image-refresh.sh: | #!/bin/sh # Polls the registry for a new manifest digest and reconciles it onto the - # long-lived control-plane workloads, on docker.io under the floating - # CLIENT_ENV tag (#154, extended by #569). + # long-lived control-plane workloads, on IMAGE_REGISTRY (the registry the + # chart pulls the tracebloc-published images from -- see the + # tracebloc.tbRegistry helper) under the floating CLIENT_ENV tag (#154, + # extended by #569). # # Action on change: `kubectl set image =repo@digest`. # It used to be `kubectl rollout restart`. #569 changed the mechanism so @@ -180,16 +182,24 @@ data: SKIP_KEY="tracebloc.io/refresh-skip-streak" log "release=$RELEASE_NAME namespace=$RELEASE_NAMESPACE deployment=$DEPLOYMENT_NAME" - log " client images: tracebloc/{jobs-manager,pods-monitor,resource-monitor} on docker.io under tag=$IMAGE_TAG" + log " client images: tracebloc/{jobs-manager,pods-monitor,resource-monitor} on ${IMAGE_REGISTRY} under tag=$IMAGE_TAG" log " also reconciling: deployment/${REQUESTS_PROXY_DEPLOYMENT} (jobs-manager image), daemonset/${RESOURCE_MONITOR_DAEMONSET} in ${NODE_AGENTS_NAMESPACE}" # #569: a private-mirror install (global.imageRegistry) renders every pod - # image as /tracebloc/... , but the digests resolved below come from - # docker.io. Writing a docker.io digest onto a mirror reference is not just - # wrong, it is BREAKING: the mirror may not hold that digest at all, so - # `set image` would pin the control plane to an unpullable ref. Under the - # old `rollout restart` this mismatch was merely useless (it restarted pods - # to re-pull the mirror's own tag); with `set image` it must fail closed. + # image as /tracebloc/... , but this script can only resolve a tag + # to a digest on the public registries it has a token arm for (get_token + # below: docker.io, ghcr.io). Writing a digest resolved elsewhere onto a + # mirror reference is not just wrong, it is BREAKING: the mirror may not + # hold that digest at all, so `set image` would pin the control plane to an + # unpullable ref. Under the old `rollout restart` this mismatch was merely + # useless (it restarted pods to re-pull the mirror's own tag); with + # `set image` it must fail closed. + # + # The verdict is RENDERED by the chart (IMAGE_REGISTRY_RESOLVABLE, from the + # same tracebloc.imageRefreshResolvableRegistries declaration that decides + # the pods' imagePullPolicy) rather than re-derived here, so the guard and + # the pull policy cannot disagree about which edges the reconcile runs on. + # Absent or anything but "true" is refused: not knowing is not permission. # # Exit 0, not 1: a mirror install has deliberately taken over image # distribution, so "nothing to reconcile" is the correct steady state, not @@ -198,9 +208,9 @@ data: # The CronJob is still RENDERED under a mirror (the alpine/k8s runner image # is re-homed onto it, per global_image_registry_test.yaml); only the # reconcile is inert. - if [ "$IMAGE_REGISTRY" != "docker.io" ]; then - log "NOTE: images are served from private mirror '${IMAGE_REGISTRY}' (global.imageRegistry), but this script resolves digests from docker.io." - log " Pinning a docker.io digest onto a mirrored reference could pin an image the mirror does not hold, so auto-refresh is INERT on this edge." + if [ "${IMAGE_REGISTRY_RESOLVABLE:-}" != "true" ]; then + log "NOTE: images are served from '${IMAGE_REGISTRY}', a registry this script cannot resolve digests on anonymously (it knows docker.io and ghcr.io)." + log " Pinning a digest resolved elsewhere onto that reference could pin an image the registry does not hold, so auto-refresh is INERT on this edge." log " Update the control-plane images by syncing your mirror and restarting the workloads, or pin images..digest explicitly." log " (That restart DOES re-pull: because this reconcile is inert here, the chart renders imagePullPolicy=Always on the unpinned control-plane images rather than IfNotPresent -- see the tracebloc.controlPlanePullPolicy helper. Offline-restart safety is traded away on this edge precisely so it is not left with no update path at all.)" log "tick complete" @@ -547,7 +557,7 @@ data: # it changes no behaviour: `continue` still ends this image's tick either # way. Only the reporting is new. stale_key="tracebloc.io/stale-pin-${repo#*/}" - pin_latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "docker.io" || true)" + pin_latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)" if [ -z "$pin_latest" ]; then log " pinned by digest in values; skipping (could not resolve ${IMAGE_TAG} to compare -- not knowing is a finding, not agreement)" log " leaving any existing ${stale_key} as the last KNOWN state rather than asserting agreement" @@ -587,7 +597,7 @@ data: annotate_args="$annotate_args ${unpinned_stale_key}-" fi - latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "docker.io" || true)" + latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)" if [ -z "$latest" ]; then log " WARN: could not resolve latest digest (rate-limited or transient); skipping this tick" continue @@ -726,9 +736,10 @@ data: annotate_args="$annotate_args ${key}=${latest}" restart_needed=1 - # The reference we pin. IMAGE_REGISTRY is docker.io here — a mirror - # exits the tick far above — but render it from the variable so this - # stays honest if that guard is ever relaxed. + # The reference we pin: the SAME registry the digest above was resolved + # on, so a copy of this image on another registry can never be pinned by + # a digest it does not hold. A registry the script cannot resolve on + # exited the tick far above (IMAGE_REGISTRY_RESOLVABLE). ref="${IMAGE_REGISTRY}/${repo}@${latest}" # Map the image onto the workload+container(s) that run it. Container @@ -990,21 +1001,25 @@ spec: value: {{ include "tracebloc.resourceMonitorName" . | quote }} - name: NODE_AGENTS_NAMESPACE value: {{ .Values.nodeAgents.namespace.name | quote }} - # The registry the chart renders into every pod image. The - # script resolves digests from docker.io only, so it goes inert - # when this is a private mirror rather than pinning a digest the - # mirror may not hold (see the guard at the top of the script). - # - # `| default "docker.io"` is NOT redundant with dig's default: - # values.yaml ships `global.imageRegistry: ""`, so the key EXISTS - # and dig returns the empty string rather than its fallback. - # Without the trailing default this renders "", the mirror guard - # below sees "" != "docker.io" and goes inert on every DEFAULT - # install — silently disabling refresh fleet-wide. The - # `tracebloc.image` helper handles the same trap with its own - # `| default "docker.io"`. + # The registry the chart renders into the control-plane pod + # images -- ONE helper (tracebloc.tbRegistry: global mirror, else + # images.traceblocRegistry, else the chart default), so the + # script resolves digests on exactly the registry the pods pull + # from. The helper owns the `| default` chain: values.yaml ships + # `global.imageRegistry: ""`, so the key EXISTS and a bare dig + # returns "" rather than its fallback; rendering "" here would + # make the reconcile go inert on every DEFAULT install (the + # "IMAGE_REGISTRY defaults to ..., not the empty string" test). - name: IMAGE_REGISTRY - value: {{ dig "imageRegistry" "docker.io" (.Values.global | default dict) | default "docker.io" | quote }} + value: {{ include "tracebloc.tbRegistry" . | quote }} + # Whether the script can resolve digests on IMAGE_REGISTRY + # anonymously -- derived from the same declaration + # (tracebloc.imageRefreshResolvableRegistries) that switches the + # pods to IfNotPresent, so the guard at the top of the script and + # the pull policy agree by construction. "false" makes the tick + # log and exit 0 (a mirror install has taken over distribution). + - name: IMAGE_REGISTRY_RESOLVABLE + value: {{ include "tracebloc.imageRefreshResolvable" . | default "false" | quote }} - name: IMAGE_TAG # RESOLVED, not raw. This is the tag the refresh pulls, so a # documented alias here freezes the edge on a tag the publish diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 81007109..ac24ffcc 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -100,7 +100,7 @@ spec: # # It also removes an image: busybox was a second pull on every edge, for a # TCP connect the app image can already do. - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} securityContext: # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's # `restricted` SCC assigns a UID from the project's range, and the pod @@ -265,7 +265,7 @@ spec: {{- end }} containers: - name: api - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile. `Always` forced a # registry round-trip on every (re)start, so an offline Docker Desktop / @@ -333,13 +333,16 @@ spec: # unresourced-init set, and neither is reachability. # `scripts/tests/qos-reachability.sh` asserts it now. # - # requests.cpu is - # also raised (100m -> 250m) so a mass restart doesn't throttle its - # startup to a crawl (the ~9m recovery seen on lukas-test). + # requests.cpu was raised (100m -> 250m) so a mass restart doesn't throttle + # its startup to a crawl (the ~9m recovery seen on lukas-test); the interim + # trim (backend#2461) brings it to 150m -- the whole plane idles at 36m -- + # and lowers requests.memory to 640Mi while the LIMIT stays 1Gi, so the + # #1144 equality is deliberately broken here: see the derivation on + # values.yaml#resources for why the request moved and the limit did not. resources: requests: - cpu: {{ .Values.resources.jobsManager.requests.cpu | default "250m" | quote }} - memory: {{ .Values.resources.jobsManager.requests.memory | default "1Gi" | quote }} + cpu: {{ .Values.resources.jobsManager.requests.cpu | default "150m" | quote }} + memory: {{ .Values.resources.jobsManager.requests.memory | default "640Mi" | quote }} limits: cpu: {{ .Values.resources.jobsManager.limits.cpu | default "1000m" | quote }} memory: {{ .Values.resources.jobsManager.limits.memory | default "1Gi" | quote }} @@ -906,7 +909,7 @@ spec: {{- end }} {{- end }} - name: pods-monitor-container - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: same policy as the api container above -- see # tracebloc.controlPlanePullPolicy. imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.podsMonitor.digest "root" $) }} @@ -915,17 +918,20 @@ spec: capabilities: drop: ["ALL"] # #1144: same cluster-recovery treatment as the api container above — - # pods-monitor was also exit-137 killed in the mass restart. Pin + # pods-monitor was also exit-137 killed in the mass restart. #1144 pinned # requests.memory == limits.memory (512Mi) to lower its oom_score_adj # (a less-preferred node OOM victim; the pod stays Burstable — and cpu is # not the only reason: see the init-container note on the api container - # above, backend#2872), and raise + # above, backend#2872), and raised # requests.cpu (50m -> 100m) so its startup is not starved under - # contention. + # contention. The interim trim (backend#2461) lowers the REQUEST to 384Mi + # and keeps the 512Mi LIMIT: nothing has measured this container yet, so + # the ceiling that #1144's restart peak justified stays where it was + # (derivation on values.yaml#resources). resources: requests: cpu: {{ .Values.resources.podsMonitor.requests.cpu | default "100m" | quote }} - memory: {{ .Values.resources.podsMonitor.requests.memory | default "512Mi" | quote }} + memory: {{ .Values.resources.podsMonitor.requests.memory | default "384Mi" | quote }} limits: cpu: {{ .Values.resources.podsMonitor.limits.cpu | default "500m" | quote }} memory: {{ .Values.resources.podsMonitor.limits.memory | default "512Mi" | quote }} diff --git a/client/templates/mysql-deployment.yaml b/client/templates/mysql-deployment.yaml index b3f9dbf7..4a2cbdc4 100644 --- a/client/templates/mysql-deployment.yaml +++ b/client/templates/mysql-deployment.yaml @@ -158,10 +158,10 @@ spec: # PodDisruptionBudget cover the eviction angle instead. resources: requests: - memory: {{ .Values.resources.mysql.requests.memory | default "1Gi" | quote }} - cpu: {{ .Values.resources.mysql.requests.cpu | default "250m" | quote }} + memory: {{ .Values.resources.mysql.requests.memory | default "704Mi" | quote }} + cpu: {{ .Values.resources.mysql.requests.cpu | default "150m" | quote }} limits: - memory: {{ .Values.resources.mysql.limits.memory | default "1Gi" | quote }} + memory: {{ .Values.resources.mysql.limits.memory | default "704Mi" | quote }} {{- /* CPU limit is intentionally unset by default (see comment block above). Schema permits it, so honour an explicit override here diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index 0ac6505d..1ef868ce 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -41,7 +41,7 @@ spec: these keys. */}} {{- $rpDigest := (dig "requestsProxy" "digest" "" $rpImages) | default (dig "jobsManager" "digest" "" $rpImages) }} - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile (see # tracebloc.controlPlanePullPolicy; it stays Always on an edge the @@ -134,8 +134,8 @@ spec: {{- $rpLim := default dict $rp.limits }} resources: requests: - cpu: {{ $rpReq.cpu | default "100m" | quote }} - memory: {{ $rpReq.memory | default "256Mi" | quote }} + cpu: {{ $rpReq.cpu | default "50m" | quote }} + memory: {{ $rpReq.memory | default "128Mi" | quote }} limits: cpu: {{ $rpLim.cpu | default "1000m" | quote }} memory: {{ $rpLim.memory | default "512Mi" | quote }} diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index fca159ff..81bf11c4 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -136,7 +136,7 @@ spec: entry). `dig` is not usable here — it rejects chartutil.Values. */}} {{- $rmDigest := (default (dict) (default (dict) .Values.images).resourceMonitor).digest | default "" }} - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (include "tracebloc.tbRegistry" .)) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # see tracebloc.controlPlanePullPolicy. `Always` made every node's pod # re-pull on restart, so an offline restart wedged the DaemonSet in diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index 3787fbc7..e1f8ddda 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -770,11 +770,13 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: 'kubectl set image -n "\$NODE_AGENTS_NAMESPACE"' - # Mirror guard: a docker.io digest must never be pinned onto a mirrored - # reference the mirror may not hold. + # Mirror guard: a digest resolved on a public registry must never be pinned + # onto a mirrored reference the mirror may not hold. The verdict is the + # chart-rendered IMAGE_REGISTRY_RESOLVABLE (tracebloc_registry_test.yaml + # covers its values); anything but "true" -- including absent -- refuses. - matchRegex: path: data["image-refresh.sh"] - pattern: 'if \[ "\$IMAGE_REGISTRY" != "docker.io" \]; then' + pattern: 'if \[ "\$\{IMAGE_REGISTRY_RESOLVABLE:-\}" != "true" \]; then' # The pods-monitor container name is contractual with the deployment. - matchRegex: path: data["image-refresh.sh"] diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 964de569..ef289d79 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -70,17 +70,21 @@ tests: path: spec.template.spec.containers[0].resources.requests - isNotEmpty: path: spec.template.spec.containers[0].resources.limits - # #1144: requests.memory pinned == limits.memory (1Gi api / 512Mi - # pods-monitor) to lower oom_score_adj during a mass restart. + # #1144 pinned requests.memory == limits.memory (1Gi api / 512Mi + # pods-monitor) to lower oom_score_adj during a mass restart. The interim + # trim (backend#2461, client#1032) lowers the REQUESTS (640Mi / 384Mi) and + # keeps the LIMITS (1Gi / 512Mi) -- the limit is the hard ceiling the + # restart peak justified and nothing has re-measured it; the request is + # what the scheduler and admission subtract. Derivation: values.yaml#resources. - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 640Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory value: 1Gi - equal: path: spec.template.spec.containers[1].resources.requests.memory - value: 512Mi + value: 384Mi - equal: path: spec.template.spec.containers[1].resources.limits.memory value: 512Mi diff --git a/client/tests/mysql_test.yaml b/client/tests/mysql_test.yaml index e0d2ad7b..90808612 100644 --- a/client/tests/mysql_test.yaml +++ b/client/tests/mysql_test.yaml @@ -40,13 +40,13 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.limits.memory - value: 1Gi + value: 704Mi - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m + value: 150m # CPU limit deliberately unset — InnoDB lock-wait timeouts under throttling. - notExists: path: spec.template.spec.containers[0].resources.limits.cpu diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml index 19f66ea8..5161e1d0 100644 --- a/client/tests/requests_proxy_test.yaml +++ b/client/tests/requests_proxy_test.yaml @@ -127,10 +127,10 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].resources.requests.cpu - value: 100m + value: 50m - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 256Mi + value: 128Mi - equal: path: spec.template.spec.containers[0].resources.limits.cpu value: 1000m diff --git a/client/tests/tracebloc_registry_test.yaml b/client/tests/tracebloc_registry_test.yaml new file mode 100644 index 00000000..a3386a61 --- /dev/null +++ b/client/tests/tracebloc_registry_test.yaml @@ -0,0 +1,439 @@ +suite: images.traceblocRegistry — the tracebloc-only registry knob (tracebloc.tbRegistry) +# The tracebloc-PUBLISHED control-plane images (jobs-manager, pods-monitor, +# resource-monitor, and the requests-proxy that runs the jobs-manager image) are +# published to both docker.io and ghcr.io at the same digests. ONE helper, +# tracebloc.tbRegistry, decides which registry the chart pulls them from, as one +# precedence chain: global.imageRegistry (a mirror re-homes everything) wins, +# then images.traceblocRegistry, then the chart default. +# +# Pinned here, each of them a machine check for a sentence in values.yaml: +# 1. The knob moves EXACTLY the tracebloc control-plane sites: busybox, the +# ingestor repo and JOB_IMAGE_HOST do not follow it, and mysql-client stays +# on docker.io (frozen, digest-pinned, published nowhere else). +# 2. The mirror wins over the knob; an empty knob renders the default, not "". +# 3. The image-refresh CronJob resolves digests on the same registry the pods +# pull from, and its reconcile verdict (IMAGE_REGISTRY_RESOLVABLE) and the +# pods' imagePullPolicy are derived from ONE declaration +# (tracebloc.imageRefreshResolvableRegistries) -- so a registry the script +# has no token arm for makes the pods Always AND the tick inert, together. +# 4. NOTES.txt reports the registry the pods actually pull from. +# helm-unittest 0.5.2 applies asserts to EVERY template a test names, so each +# test below names exactly one. +templates: + - templates/jobs-manager-deployment.yaml + - templates/requests-proxy-deployment.yaml + - templates/resource-monitor-daemonset.yaml + - templates/mysql-deployment.yaml + - templates/image-refresh-cronjob.yaml + - templates/NOTES.txt +release: + name: t + namespace: tracebloc +set: + clientId: "test-id" + clientPassword: "test" +tests: + - it: the chart default pulls the jobs-manager Deployment's images from docker.io + template: templates/jobs-manager-deployment.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[1].image + pattern: "^docker\\.io/tracebloc/pods-monitor:" + + - it: the chart default pulls the requests-proxy image from docker.io + template: templates/requests-proxy-deployment.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + + - it: the chart default pulls the resource-monitor image from docker.io + template: templates/resource-monitor-daemonset.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/resource-monitor:" + + - it: an EMPTY knob renders the default registry on the pods, not an empty one (the dig-empty trap) + # values.yaml ships every registry key as a string, so `dig` finds the key + # and returns "" instead of its fallback; an edge that clears the knob, or a + # --reuse-values replay from before it existed, must still get a registry. + # The pods are guarded twice (tracebloc.image applies its own default to an + # empty registry), so this test alone would stay green if the helper lost + # its `| default` chain -- the IMAGE_REGISTRY test right below is the one + # with no second net, and it is what reddens under that mutation. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager:" + + - it: an EMPTY knob renders the default registry into IMAGE_REGISTRY, not an empty one + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: "" + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "docker.io" + + - it: images.traceblocRegistry re-homes the jobs-manager Deployment (init, api, pods-monitor) + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + - matchRegex: + path: spec.template.spec.containers[1].image + pattern: "^ghcr\\.io/tracebloc/pods-monitor:" + + - it: images.traceblocRegistry re-homes the requests-proxy + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + + - it: images.traceblocRegistry re-homes the resource-monitor + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^ghcr\\.io/tracebloc/resource-monitor:" + + - it: a digest pin keeps the knob's registry (repo@digest, not repo:tag) + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "ghcr.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + - it: the knob does NOT move mysql-client (frozen, digest-pinned, docker.io only) + # The open question on mysql-client, recorded on the epic: it stays on + # Docker Hub. This is that decision as a machine check -- routing the mysql + # site through tracebloc.tbRegistry would make this test red. + template: templates/mysql-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/mysql-client@sha256:" + + - it: the knob does NOT move busybox + # busybox keeps its own precedence (global mirror, else docker.io). + template: templates/mysql-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: "^docker\\.io/library/busybox:" + + - it: the knob does NOT move the ingestor repository or JOB_IMAGE_HOST + # The ingestor is already on ghcr.io by repository; the training-image host + # moves in its own step, so it must still read docker.io/ here. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_REPOSITORY + value: "ghcr.io/tracebloc/ingestor" + - contains: + path: spec.template.spec.containers[0].env + content: + name: JOB_IMAGE_HOST + value: "docker.io/" + + - it: global.imageRegistry wins over images.traceblocRegistry on the jobs-manager + template: templates/jobs-manager-deployment.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^mirror\\.corp\\.example/tracebloc/jobs-manager:" + + - it: global.imageRegistry wins over images.traceblocRegistry on the resource-monitor + template: templates/resource-monitor-daemonset.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^mirror\\.corp\\.example/tracebloc/resource-monitor:" + + - it: global.imageRegistry wins over images.traceblocRegistry in IMAGE_REGISTRY + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "mirror.corp.example" + + - it: image-refresh resolves digests on the registry the pods pull from, never a docker.io literal + # The pinned reference is built from the same variable the digest was + # resolved on, so a copy on another registry can never be pinned by a digest + # it does not hold. + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'pin_latest="\$\(get_latest_digest "\$repo" "\$IMAGE_TAG" "\$IMAGE_REGISTRY" \|\| true\)"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'latest="\$\(get_latest_digest "\$repo" "\$IMAGE_TAG" "\$IMAGE_REGISTRY" \|\| true\)"' + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'get_latest_digest "\$repo" "\$IMAGE_TAG" "docker\.io"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'ref="\$\{IMAGE_REGISTRY\}/\$\{repo\}@\$\{latest\}"' + + - it: the script carries a token arm for every registry the chart declares resolvable + # tracebloc.imageRefreshResolvableRegistries is "docker.io ghcr.io". A + # registry declared there without a get_token / get_latest_digest arm would + # make the pods IfNotPresent while every tick WARNs "unknown registry". + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["image-refresh.sh"] + pattern: '_url="https://auth\.docker\.io/token\?service=registry\.docker\.io&scope=repository:\$\{_repo\}:pull"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: '_url="https://ghcr\.io/token\?service=ghcr\.io&scope=repository:\$\{_repo\}:pull"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'docker\.io\) _host="registry-1\.docker\.io" ;;' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'ghcr\.io\) _host="ghcr\.io" ;;' + + - it: on docker.io the reconcile is resolvable + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "true" + + - it: on docker.io the pods are IfNotPresent + template: templates/jobs-manager-deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the reconcile is resolvable and IMAGE_REGISTRY follows + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "ghcr.io" + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "true" + + - it: on ghcr.io the jobs-manager pods are IfNotPresent + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + - equal: + path: spec.template.spec.containers[1].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the requests-proxy is IfNotPresent + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: on ghcr.io the resource-monitor is IfNotPresent + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: a registry the script has no token arm for makes the tick inert + # The two consumers of tracebloc.imageRefreshResolvableRegistries must move + # as one: IfNotPresent with an inert reconcile is a frozen control plane + # with a green CronJob (#569, Bugbot High). + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + images: + traceblocRegistry: quay.example + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "quay.example" + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "false" + + - it: a registry the script has no token arm for makes the jobs-manager pods Always + # The two consumers of tracebloc.imageRefreshResolvableRegistries must move + # as one: IfNotPresent with an inert reconcile is a frozen control plane + # with a green CronJob (#569, Bugbot High). + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + - equal: + path: spec.template.spec.containers[1].imagePullPolicy + value: Always + + - it: a registry the script has no token arm for makes the requests-proxy Always + template: templates/requests-proxy-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: a registry the script has no token arm for makes the resource-monitor Always + template: templates/resource-monitor-daemonset.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: a global mirror is not resolvable even when the knob names a public registry + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY_RESOLVABLE + value: "false" + + - it: a global mirror keeps the pods Always even when the knob names a public registry + template: templates/jobs-manager-deployment.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: NOTES prints the default registry + template: templates/NOTES.txt + asserts: + - matchRegexRaw: + pattern: "Image registry:.*docker\\.io.*\\(tracebloc control-plane images\\)" + + - it: NOTES follows images.traceblocRegistry + template: templates/NOTES.txt + set: + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegexRaw: + pattern: "Image registry:.*ghcr\\.io" + - notMatchRegexRaw: + pattern: "Image registry:.*docker\\.io" + + - it: NOTES follows the global mirror over the knob + template: templates/NOTES.txt + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: ghcr.io + asserts: + - matchRegexRaw: + pattern: "Image registry:.*mirror\\.corp\\.example" + diff --git a/client/values.schema.json b/client/values.schema.json index 1a5abdaf..c50422fc 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -791,6 +791,11 @@ "type": "object", "description": "Container image pinning. Prefer digest over tag for immutability.", "properties": { + "traceblocRegistry": { + "type": "string", + "pattern": "^([A-Za-z0-9.-]+(:[0-9]+)?)?$", + "description": "Registry the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) are pulled from. A bare host, no scheme; empty renders the chart default. global.imageRegistry always wins when set. Moves ONLY the tracebloc images (mysql-client stays on docker.io; third-party images keep their own registry keys; JOB_IMAGE_HOST is not routed through it yet). Image-refresh resolves digests on this registry and can only do so anonymously on docker.io and ghcr.io -- any other value makes the reconcile inert and the pods imagePullPolicy=Always, like a mirror." + }, "jobsManager": { "type": "object", "properties": { diff --git a/client/values.yaml b/client/values.yaml index 564c0962..4e62469c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -12,10 +12,13 @@ global: # When set, EVERY image the chart pulls is re-homed onto this host — the # tracebloc services, the spawned ingestor + training-job images, and the # alpine/*, ubuntu/squid, busybox, curl helper images — which must already host - # the same repositories + tags/digests. Unset (the default) pulls tracebloc/* - # and the helper images from docker.io and the ingestor from ghcr.io. If the - # mirror needs authentication, also set dockerRegistry below. The standalone - # installer writes this from TRACEBLOC_IMAGE_REGISTRY. See docs/INSTALL.md. + # the same repositories + tags/digests. Unset (the default) pulls the + # tracebloc-published control-plane images from images.traceblocRegistry + # (below), the helper images from docker.io and the ingestor from ghcr.io. If + # the mirror needs authentication, also set dockerRegistry below. The + # standalone installer writes this from TRACEBLOC_IMAGE_REGISTRY. See + # docs/INSTALL.md. To move ONLY the tracebloc images, leave this unset and set + # images.traceblocRegistry instead. imageRegistry: "" # -- Environment variables passed to containers @@ -609,7 +612,7 @@ egressProxy: resources: requests: cpu: "50m" - memory: "64Mi" + memory: "160Mi" limits: cpu: "500m" memory: "256Mi" @@ -906,15 +909,39 @@ telemetryCollector: # * neither -> Always (floating tag + restart is the # only update path there is) # -# The third case is real, not theoretical: a `global.imageRegistry` mirror makes -# the reconcile inert (it resolves digests from docker.io, which the mirror may -# not hold), and `imageRefresh.enabled: false` disables it outright. Those edges +# The third case is real, not theoretical: a registry the reconcile cannot +# resolve digests on anonymously -- a `global.imageRegistry` mirror, or an +# `images.traceblocRegistry` other than docker.io / ghcr.io -- makes it inert +# (a digest resolved elsewhere may not exist there), and +# `imageRefresh.enabled: false` disables it outright. Those edges # keep the pre-#569 behaviour deliberately — a frozen control plane with a green # CronJob and no signal is worse than a restart that needs the network. # # Setting `digest` here is still meaningful: it is an explicit operator pin # that also opts the image OUT of auto-refresh. images: + # Registry the tracebloc-PUBLISHED control-plane images are pulled from: + # tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor + # and the requests-proxy (which runs the jobs-manager image). A bare host, no + # scheme. Resolved by the tracebloc.tbRegistry helper as ONE precedence chain + # -- `global.imageRegistry` (a mirror re-homes everything) always wins, then + # this, then the chart default -- so the pods, the image-refresh CronJob and + # NOTES.txt cannot disagree about where those images live. + # + # This knob moves ONLY the tracebloc images: busybox, squid, alpine/*, the + # device plugins and the ingestor keep their own registries, and + # tracebloc/mysql-client stays on docker.io (frozen, digest-pinned, published + # nowhere else -- see images.mysqlClient). The training-image host + # (JOB_IMAGE_HOST) is not routed through it yet; that moves in its own step. + # + # The tracebloc images are published to BOTH docker.io and ghcr.io, at the + # same digests. Setting this to "ghcr.io" on one edge is how to try the + # ghcr.io copies before the chart default moves; setting it back is the + # per-edge rollback once it has. Image-refresh follows it: the CronJob + # resolves digests on this registry (it can do so anonymously on docker.io + # and ghcr.io; any other value makes the reconcile inert and the pods + # imagePullPolicy=Always, exactly as a global.imageRegistry mirror does). + traceblocRegistry: "docker.io" jobsManager: digest: "" # -- TRAINING IMAGES: the digest-pinned spawn path (RFC-1246 P2, landed for @@ -1328,35 +1355,80 @@ images: # a mass restart does not throttle their startup. Same requests.memory==limits # philosophy mysql already uses (mysql likewise omits its cpu limit). resources: + # INTERIM TRIM (backend#2461, 2026-09-10) -- read before "fixing" any number back up. + # + # Requests are what the scheduler and the jobs-manager's admission SUBTRACT from + # the node; they are not what the process uses. Measured on an installer edge + # (backend#2461): the control plane requested 1050m / 3276Mi while using + # 36m / ~1022Mi. Since chart 1.9.109 the kubelet reservation is honest + # (#1017) and since client-runtime#544 the admission subtracts the chart's + # real requests, so on the smallest supported node -- 2 cores / 8 GiB -- the + # arithmetic became visible: allocatable 1900m / 5538Mi, minus these requests, + # minus k3s's own coredns + metrics-server (200m / 140Mi), left 900m / 2390Mi, + # and the smallest training envelope needs 1000m / 3072Mi. Every task on + # staging was refused NEVER_STEADY_STATE (e2e-test-agent fleet 34473120598). + # + # THE FOOTPRINT COUNTS THE COLLECTOR. `helm template` cannot look up the token + # Secret, so the render -- and therefore control-plane-footprint.sh, the + # installer embed and this derivation -- carries the telemetry collector's + # 100m / 128Mi. That is the right worst case: on a real edge the collector IS + # rendered once jobs-manager has written the token Secret (backend#2274), so an + # edge that trains on install must still train after that. With it counted the + # render is 650m / 2272Mi, and the node above leaves 1050m / 3126Mi (was + # 900m / 2390Mi): 50m and 54Mi over the envelope. Thin, and honest. + # + # mysql 250m/1Gi -> 150m/704Mi peak measured 0.40-0.55x of 1Gi (max ~563Mi); 704 = +25% headroom; requests==limits kept + # jobs-manager 250m/1Gi -> 150m/640Mi 168Mi idle (0.16x); LIMIT STAYS 1Gi (see below); cpu: the whole plane idles at 36m + # pods-monitor 100m/512Mi -> 100m/384Mi no measurement yet; LIMIT STAYS 512Mi (see below) + # requests-proxy 100m/256Mi -> 50m/128Mi a few requests a minute; limits unchanged (1000m/512Mi) + # egress-proxy 50m/64Mi -> 50m/160Mi UP: measured 2.45x OVER its request (backend#2461) + # collector 100m/128Mi unchanged unmeasured (distroless, no readable cgroup) -- not trimmed on a guess + # + # WHY jobs-manager AND pods-monitor NOW HAVE requests < limits. #1144 pinned + # requests.memory == limits.memory on both so a cluster-recovery mass restart + # would not pick them as OOM victims; the PODS were Burstable then and still + # are (the unresourced init containers, see above). The trim lowers only the + # REQUEST -- what the scheduler and admission subtract -- and leaves the LIMIT + # where it was, because the limit is the hard cgroup ceiling and the restart + # peak that justifies it is exactly the leg backend#2461's load run has not + # produced yet (review on client#1032). The cost is a higher oom_score_adj + # under node pressure; the alternative was a hard ceiling nothing has measured. + # When the restart leg lands, set BOTH from its p95 and restore the equality. + # + # What this is NOT: the final trim. Re-derive from the load run; do not + # hand-tune here. Guards that read these values: control-plane-footprint.sh + # (ceiling), the generated embed in scripts/lib/install-client-helm.sh + # (scripts/gen-footprint-embed.sh --check), envelope-schedulability.sh. mysql: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "704Mi" limits: - memory: "1Gi" + memory: "704Mi" # No cpu limit — see comment above. jobsManager: requests: - cpu: "250m" - memory: "1Gi" + cpu: "150m" + memory: "640Mi" limits: cpu: "1000m" memory: "1Gi" podsMonitor: requests: cpu: "100m" - memory: "512Mi" + memory: "384Mi" limits: cpu: "500m" memory: "512Mi" # requests-proxy serves the Service Bus / backend communication path and - # is mostly idle (a few req/min). 100m/256Mi requests with headroom for - # the occasional burst; revisit if heap profiling shows growth. Schema - # at values.schema.json#resources.requestsProxy validates overrides. + # is mostly idle (a few req/min). 50m/128Mi requests (trimmed from 100m/256Mi, + # backend#2461) with the limits left at 1000m/512Mi for the occasional burst; + # revisit if heap profiling shows growth. Schema at + # values.schema.json#resources.requestsProxy validates overrides. requestsProxy: requests: - cpu: "100m" - memory: "256Mi" + cpu: "50m" + memory: "128Mi" limits: cpu: "1000m" memory: "512Mi" @@ -1730,10 +1802,13 @@ autoUpgrade: # prefix rewrite is not mistaken for a revert). Steady state is one HEAD # per image per tick, well under Docker Hub's 100/6h anonymous # pull-rate limit. -# - Private mirrors (global.imageRegistry): the script resolves digests -# from docker.io, so pinning one onto a mirrored reference could pin -# an image the mirror does not hold. It logs and goes INERT instead. -# Under the old `rollout restart` the mismatch was merely useless; with +# - The script resolves digests on the SAME registry the pods pull the +# tracebloc images from (images.traceblocRegistry, or the +# global.imageRegistry mirror when set), and it can only do so +# anonymously on docker.io and ghcr.io. Anywhere else -- a private +# mirror -- pinning a digest resolved elsewhere could pin an image the +# registry does not hold, so it logs and goes INERT instead. Under the +# old `rollout restart` the mismatch was merely useless; with # `set image` it has to fail closed. # # Pinned-digest interaction: if images.jobsManager.digest, @@ -1756,8 +1831,9 @@ autoUpgrade: # requires) are namespace-wide and read-only. Still far narrower than # autoUpgrade, which holds cluster-admin. # -# The script reaches out only to docker.io's public registry endpoints; if -# your cluster requires egress through a proxy, set HTTPS_PROXY via env +# The script reaches out only to the public token + manifest endpoints of +# the registry the tracebloc images are pulled from (docker.io or ghcr.io); +# if your cluster requires egress through a proxy, set HTTPS_PROXY via env # or disable this CronJob. # # Disabled-on-this-cluster check after install: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 498f760c..0480884b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -34,7 +34,7 @@ The standalone installer runs a **preflight** check that verifies this connectiv | Host | Why | |---|---| | `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox + the tracebloc client images | -| `ghcr.io` | k3d node images + the ingestor image | +| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | k3d node images + the ingestor image; the tracebloc control-plane images when `images.traceblocRegistry` points there | | `api.tracebloc.io` (`dev-api`/`stg-api` for non-prod) | client credential check + the running client's platform connection | | `tracebloc.github.io` | the tracebloc Helm chart repository | @@ -55,6 +55,8 @@ Some sites hard-block Docker Hub / GHCR outright — the images aren't reachable The chart follows the **`global.imageRegistry`** convention: set it once and **every** image the chart pulls — the tracebloc services, the spawned ingestor, the training-job images, and the `alpine/*`, `ubuntu/squid`, `busybox`, `curl` helper images — is re-homed onto your registry. No per-image overrides. +**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. To point just the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) at a different registry — they are published to both `docker.io` and `ghcr.io` at the same digests — set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset. `global.imageRegistry` always wins when both are set. The image-refresh CronJob follows the same value and can resolve digests anonymously on `docker.io` and `ghcr.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the training-image host yet. + **1. A private/mirror registry your site *can* reach.** First mirror the images into it. List exactly what to copy (tags/digests stay authoritative across chart versions) with: diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index de49f2e7..b37b578c 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -5263,8 +5263,8 @@ $script:TbEnvelopeNodeMinMemBytes = 5368709120 # this bootstrap is signed and guarantees neither helm nor python3. What keeps it # honest is `scripts/gen-footprint-embed.sh --check` in `make drift`: both # installers' values must equal a fresh render of the chart in the same tree. -$script:TbCpFootprintMemBytes = 3288334336 -$script:TbCpFootprintCpuMilli = 900 +$script:TbCpFootprintMemBytes = 2382364672 +$script:TbCpFootprintCpuMilli = 650 # ── end generated footprint ───────────────────────────────────────────────── # Set by Get-TrainingResources when the machine is readable but below the diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 65c45a07..5e55501b 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -263,8 +263,8 @@ _TB_ENVELOPE_NODE_MIN_MEM_BYTES=5368709120 # signed. What keeps it honest is `scripts/gen-footprint-embed.sh --check` in # `make drift` (the required Source-of-truth drift job): the value below must # equal a fresh render of the chart in the same tree, or CI reddens. -_TB_CP_FOOTPRINT_MEM_BYTES=3288334336 -_TB_CP_FOOTPRINT_CPU_MILLI=900 +_TB_CP_FOOTPRINT_MEM_BYTES=2382364672 +_TB_CP_FOOTPRINT_CPU_MILLI=650 # ── end generated ─────────────────────────────────────────────────────────── # ── the fallback training envelope (precedence step 4) ────────────────────── @@ -724,12 +724,17 @@ _training_provenance() { # # Everything above sizes the envelope against ALLOCATABLE and stops. Nothing then # asked whether the number written can be scheduled beside what else runs on the -# node: the chart's own control plane (3136 MiB / 900 m, from the render) plus -# the distribution's system pods (~140 MiB / 200 m of coredns and metrics-server -# on k3s). Measured on a real single-node install, the platform out-requests the -# 3 GiB reserve the envelope subtracts, so `allocatable - 3 GiB` over-asked at -# EVERY machine size and the training pod sat `Pending / Insufficient memory` +# node: the chart's own control plane (the embedded _TB_CP_FOOTPRINT_* constants, +# generated from the render by scripts/gen-footprint-embed.sh) plus the +# distribution's system pods (~140 MiB / 200 m of coredns and metrics-server on +# k3s). When this was written the chart requested 3136 MiB / 900 m -- MORE than +# the 3 GiB reserve the envelope subtracts -- so `allocatable - 3 GiB` over-asked +# at EVERY machine size and the training pod sat `Pending / Insufficient memory` # while the installer printed `Training size: ...` and cli doctor said OK. +# backend#2461's interim trim (chart 1.9.112) brought the control plane under the +# reserve, so today the fit below is a no-op on every golden vector; it stays +# because the constants are generated, not promised, and the next chart that +# out-requests the reserve should be reduced here rather than sit Pending again. # # The two functions below make the installer own that question at the moment it # writes the envelope. The 3 GiB constant itself is not touched here: fixing the diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 9f3572bc..22c38328 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -9,12 +9,12 @@ b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gp d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea scripts/lib/setup-linux.sh 669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c scripts/lib/cluster.sh 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62 scripts/lib/gpu-plugins.sh -320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a scripts/lib/install-client-helm.sh +9a473b2d11545183be0bb24a1446eb8a51bd6a525d5d08dcd09a574f3bd10e06 scripts/lib/install-client-helm.sh 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh 424d6b9fa6990336211540be7288e829308dd65ae4e0a4046cd6fcba2df9c943 scripts/lib/assess.sh e47bf0351d34af486b2abda48b0263fa3969585f0d1669441d202fd614d1be8b scripts/lib/probe.sh 354da97b583c6bc34156bb66d742e5ce0d84bc63a54daea4edcbe026c298ddbd scripts/lib/summary.sh 80fe2c2a299f83778115e90fc956ecfeace546ba05b81edbcedc23b01299df7e scripts/lib/diagnose.sh -b2a223a1e819e7056f27707e9e5bd25c6c4bdd830212806240a2559e71f1761d scripts/install-k8s.ps1 +07722bab739746746f85504867ba3d32d85f9bc39f318b0789880bb7d6c13548 scripts/install-k8s.ps1 fc6eeeb4e4114ce74566992519a90320c52434d604218a30ba8ef2c3fa266ca5 scripts/lib/telemetry.ps1 diff --git a/scripts/tests/control-plane-footprint.bats b/scripts/tests/control-plane-footprint.bats index 50c61308..f65020ee 100644 --- a/scripts/tests/control-plane-footprint.bats +++ b/scripts/tests/control-plane-footprint.bats @@ -75,7 +75,7 @@ YAML # guard hands to backend#2460/#2461. # # THIS FIXTURE EXISTS BECAUSE THE REAL CHART CANNOT TELL THE TWO APART. On the - # live render the number is 3136 MiB either way -- the chart's init containers + # live render the number is the same either way -- the chart's init containers # carry no requests -- so the fix is INERT there and a green run proves nothing # about the formula. Here the two answers differ, deliberately: # @@ -162,7 +162,13 @@ YAML } @test "guard: RATCHET reddens when the footprint would exceed the ceiling" { - TB_CP_FOOTPRINT_MEM_CEIL=3000 run bash "$GUARD" + # The ceiling is DERIVED from the live render (one MiB under it), so this test + # keeps reddening whatever the chart requests -- a written-down 3000 stopped + # breaching once backend#2461 trimmed the render from 3136 to 2336 MiB. + local mib + mib="$(bash "$GUARD" --print-footprint 2>/dev/null | cut -d' ' -f1)" + [[ "$mib" =~ ^[0-9]+$ ]] || { echo "could not derive the render footprint: '$mib'"; return 1; } + TB_CP_FOOTPRINT_MEM_CEIL=$(( mib - 1 )) run bash "$GUARD" [ "$status" -eq 1 ] || { echo "expected exit 1, got $status: $output"; return 1; } printf '%s\n' "$output" | grep -q 'exceed the recorded ceiling' || { echo "$output"; return 1; } } diff --git a/scripts/tests/control-plane-footprint.sh b/scripts/tests/control-plane-footprint.sh index a82ecfa3..037b9f4e 100755 --- a/scripts/tests/control-plane-footprint.sh +++ b/scripts/tests/control-plane-footprint.sh @@ -14,11 +14,14 @@ # constant still covers it. Grep the three consumers for `3008`, `platform # footprint`, `control plane requests`: zero hits. The number was invisible. # -# Measured here by rendering the chart: the steady-state control plane requests -# ~3136 MiB, already ABOVE the 3 GiB (3072 MiB) the envelope reserves for it. -# That 64 MiB overshoot is the memory half of the reason a training pod on a -# freshly-installed single-node edge can sit `Pending / Insufficient memory` -# (backend#2870). CPU fits: 900m requested against a 1000m reserve. +# Measured here by rendering the chart: the steady-state control plane requested +# ~3136 MiB / 900 m before backend#2461's interim trim (2026-09-10) -- ABOVE the +# 3 GiB (3072 MiB) the envelope reserved for it; 2272 MiB / 650 m after, under it. +# That 64 MiB overshoot was the memory half of the reason a training pod on a +# freshly-installed single-node edge could sit `Pending / Insufficient memory` +# (backend#2870). The render COUNTS THE TELEMETRY COLLECTOR (helm template cannot +# look up the token Secret, so the collector renders): that is the worst case a +# real edge reaches once the Secret exists, and the one the ceiling must hold. # # WHAT THIS GUARD DOES, AND DELIBERATELY DOES NOT # ----------------------------------------------- @@ -70,8 +73,8 @@ installer="$root/scripts/lib/install-client-helm.sh" # same PR -- which is the moment to weigh whether the training envelope can still # afford it. Overridable so the guard's own test can drive a lower ceiling and # watch a real render breach it. -MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-3136}" -CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-900}" +MEM_CEIL_MIB="${TB_CP_FOOTPRINT_MEM_CEIL:-2272}" +CPU_CEIL_MILLI="${TB_CP_FOOTPRINT_CPU_CEIL:-650}" command -v helm >/dev/null 2>&1 || { echo "[ERROR] helm is required to render the chart footprint" >&2; exit 3; } command -v python3 >/dev/null 2>&1 || { echo "[ERROR] python3 is required to sum the rendered requests" >&2; exit 3; } @@ -169,9 +172,9 @@ try: # and cpu, so a pod can take its memory from the init side and its cpu # from the app side. # - # THE NUMBER DOES NOT MOVE ON THIS CHART -- 3136 MiB / 900 m either way, - # because these init containers carry no requests. So the "64 MiB OVER" - # finding stands; the method was wrong, the conclusion was not. It also + # THE NUMBER DOES NOT MOVE ON THIS CHART -- the render is the same + # either way, because these init containers carry no requests. So the + # original "64 MiB OVER" finding stood; the method was wrong, the conclusion was not. It also # means the fix is INERT on the real render, which is why the bats # fixture is built so the two formulas disagree. def _req(c, key): diff --git a/scripts/tests/envelope-schedulability-mutations.sh b/scripts/tests/envelope-schedulability-mutations.sh index 89abee65..09600db6 100644 --- a/scripts/tests/envelope-schedulability-mutations.sh +++ b/scripts/tests/envelope-schedulability-mutations.sh @@ -110,7 +110,7 @@ run_case "unreadable cluster writes a carried non-floor size" "$LIB" \ printf '\nthe embed:\n' run_case "embedded footprint drifts from the render" "$LIB" \ - '_TB_CP_FOOTPRINT_MEM_BYTES=3288334336' \ + '_TB_CP_FOOTPRINT_MEM_BYTES=2382364672' \ '_TB_CP_FOOTPRINT_MEM_BYTES=3221225472' 1 printf '\n%s passed, %s failed\n' "$pass" "$fail" diff --git a/scripts/tests/envelope-schedulability.bats b/scripts/tests/envelope-schedulability.bats index 25e1e706..e65449b9 100644 --- a/scripts/tests/envelope-schedulability.bats +++ b/scripts/tests/envelope-schedulability.bats @@ -171,9 +171,12 @@ YAML # It walked the golden vectors -- a guard that passes having checked nothing is # the class this repo catalogues. The table has 13 rows today; assert a floor, # not the exact count, so a vector added upstream does not redden this. - [[ "$output" =~ ok\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } - [ "${BASH_REMATCH[2]}" -ge 10 ] || { echo "only ${BASH_REMATCH[2]} measurable vectors"; return 1; } - [ "${BASH_REMATCH[1]}" -ge 1 ] || return 1 + # Two legitimate shapes: `ok positive control: N/M` while the platform out-requests + # the reserve, or `note positive control: N/M ... does not apply` once it fits + # inside it (backend#2461). Either way the guard must have walked the table. + [[ "$output" =~ (ok|note)\ +positive\ control:\ ([0-9]+)/([0-9]+) ]] || { echo "$output"; return 1; } + [ "${BASH_REMATCH[3]}" -ge 10 ] || { echo "only ${BASH_REMATCH[3]} measurable vectors"; return 1; } + if [[ "${BASH_REMATCH[1]}" == "ok" ]]; then [ "${BASH_REMATCH[2]}" -ge 1 ] || return 1; fi } @test "envelope-schedulability.sh: CANNOT TELL (exit 2) when the tree has no chart to derive from" { @@ -360,10 +363,11 @@ for key, kind in (("cpu_to_milli", "cpu"), ("mem_to_bytes", "mem")): } helm() { return 1; } unset TRACEBLOC_TRAINING_RESOURCES - # ONE system pod here (70Mi / 100m), so cpu fits (3000 + 900 + 100 = 4000) and - # only memory reduces (5120 + 3136 + 70 > 8192 -> 4 GiB): cpu=3,memory=4Gi. - # Anything else in the capture -- a stray echo -- would corrupt the value. + # ONE system pod here (70Mi / 100m). Since backend#2461's trim the resolver's + # cpu=3,memory=5Gi fits (5120 + 2336 + 70 <= 8192; 3000 + 750 + 100 <= 4000), so + # the fit writes it unchanged. Anything else in the capture -- a stray echo -- + # would corrupt the value, which is what this test is for. local captured captured="$(_resolve_training_size; _fit_training_envelope; printf '%s' "$_TB_TRAINING_SIZE")" - [ "$captured" = "cpu=3,memory=4Gi" ] || { echo "captured '$captured'"; return 1; } + [ "$captured" = "cpu=3,memory=5Gi" ] || { echo "captured '$captured'"; return 1; } } diff --git a/scripts/tests/envelope-schedulability.sh b/scripts/tests/envelope-schedulability.sh index 1e552818..d22e04c1 100644 --- a/scripts/tests/envelope-schedulability.sh +++ b/scripts/tests/envelope-schedulability.sh @@ -7,11 +7,15 @@ # --------------- # backend#2870: the installer sized the training envelope as `allocatable - 3 GiB` # and nothing then asked whether that number could be scheduled beside what the -# chart installs. It could not: the control plane requests 3136 MiB (from the -# render) against the 3072 MiB reserve, so every install over-asked and the +# chart installs. It could not: the control plane then requested 3136 MiB (from +# the render) against the 3072 MiB reserve, so every install over-asked and the # training pod sat Pending. No test in this repo could see it, because every # test compared the installers to the CONTRACT and none compared the arithmetic -# to a CLUSTER. +# to a CLUSTER. backend#2461's interim trim (2026-09-10) took the render to +# 2336 MiB / 750 m, UNDER the reserve -- so the over-ask no longer occurs on a +# real node and the positive control below reports that instead of failing; +# the reduce/refuse paths are exercised on nodes DERIVED from the footprint +# (SMALL_MI / TINY_MI), never a written-down machine size. # # WHAT IT ASSERTS, and how it derives rather than restates (backend#1729 rule 1) # ---------------------------------------------------------------------------- @@ -23,7 +27,9 @@ # 1. the installer's embedded footprint constants equal that render; # 2. for EVERY single-node golden vector in the contract table, the envelope # written + chart footprint + system pods <= allocatable, on memory AND cpu; -# 3. the ticket's 8 GiB reproduction is REDUCED, with the arithmetic; +# 3. the ticket's 8 GiB reproduction now FITS unreduced (backend#2461 brought +# the footprint under the reserve), and a node sized from the live +# footprint to over-ask is REDUCED, with the arithmetic (3b); # 4. a machine where not even a 1-core / 1-GiB run fits is REFUSED; # 5. a cpu-only overshoot reduces cpu alone (DoD part 5: cover cpu too); # 6. a human's pin is never altered, only warned; @@ -190,32 +196,56 @@ if (( NEED_B > _TB_ENVELOPE_OVERHEAD_MEM_BYTES || NEED_M > _TB_ENVELOPE_OVERHEAD bad "positive control: the platform out-requests the reserve, yet no vector over-asked before the fit -- this guard cannot see the defect it exists for" fi else - echo " note the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve; the fit is a no-op by construction and the positive control does not apply" + echo " note positive control: ${over_before}/${checked} measurable vectors over-asked -- the platform (${NEED_B} B / ${NEED_M} m) now fits inside the reserve, so the fit is a no-op by construction and the control does not apply (backend#2461)" fi -# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. +# 3. THE TICKET'S REPRODUCTION: an 8 GiB node. Before backend#2461 it over-asked +# (5120 + 3136 + 140 > 8192) and was REDUCED; after the trim it fits unreduced. +# Pinning the fit is the regression guard: a chart change that pushes the +# footprint back over the reserve reddens here first. resolve_and_fit "4 8Gi" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]]; then - ok "8 GiB reproduction: ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" +if [[ "$_TB_FIT_VERDICT" == "fits" && "$_TB_TRAINING_SIZE" == "$BEFORE" ]]; then + ok "8 GiB reproduction: ${BEFORE} fits unreduced beside $(( NEED_B / MIB )) MiB / ${NEED_M} m (backend#2461 closed the over-ask)" +else + bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want fits, unreduced)" +fi + +# 3b. THE REDUCE PATH still has to be exercised, so the node is DERIVED from the +# footprint: below 4 GiB the resolver falls to the 1-core / 2-GiB floor, and +# SMALL_MI leaves room for 1 GiB but not for that floor, so the fit must +# reduce to 1 GiB with the arithmetic on screen. Moves with the render. +SMALL_MI=$(( NEED_B / MIB + 1024 + 64 )) +resolve_and_fit "4 ${SMALL_MI}Mi" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && [[ "$_TB_FIT_LINES" == *"OVER"* ]] && [[ "$_TB_FIT_LINES" == *"reduced ${BEFORE} -> ${_TB_TRAINING_SIZE}"* ]] && [[ "$_TB_TRAINING_SIZE" == "cpu=1,memory=1Gi" ]]; then + ok "${SMALL_MI} MiB node (derived): ${BEFORE} -> ${_TB_TRAINING_SIZE}, arithmetic printed" printf '%s\n' "$_TB_FIT_LINES" | sed 's/^/ /' else - bad "8 GiB reproduction: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" + bad "${SMALL_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE} (want reduced to cpu=1,memory=1Gi)" fi -# 4. REFUSAL when not even 1 core / 1 GiB fits. -resolve_and_fit "4 4Gi" +# 4. REFUSAL when not even 1 core / 1 GiB fits -- TINY_MI is derived so that +# allocatable minus the platform is 64 MiB short of a 1 GiB run. +TINY_MI=$(( NEED_B / MIB + 1024 - 64 )) +resolve_and_fit "4 ${TINY_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "refused" && "$_TB_FIT_LINES" == *"not even a 1-core / 1-GiB run"* ]]; then - ok "4 GiB node: refused, with the arithmetic" + ok "${TINY_MI} MiB node (derived): refused, with the arithmetic" else - bad "4 GiB node: verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" + bad "${TINY_MI} MiB node (derived): verdict '${_TB_FIT_VERDICT}' size '${_TB_TRAINING_SIZE}' (want refused)" fi -# 5. CPU-ONLY overshoot: memory already fits (the GiB floor absorbed it), cpu does not. +# 5. CPU-ONLY overshoot: memory already fits, cpu does not. The resolver leaves +# one core for the platform and the trimmed platform needs 950 m, so no real +# node can produce this any more -- the pre-trim footprint could. The case +# keeps DoD part 5 covered by REPLAYING that footprint (900 m, the value the +# embed carried before backend#2461) through the real fit; the memory side is +# untouched so only cpu can move. +saved_cpu="$_TB_CP_FOOTPRINT_CPU_MILLI"; _TB_CP_FOOTPRINT_CPU_MILLI=900 resolve_and_fit "8 63928Mi" +_TB_CP_FOOTPRINT_CPU_MILLI="$saved_cpu" a_mem="$(env_mem_b "$_TB_TRAINING_SIZE")"; b_mem="$(env_mem_b "$BEFORE")" a_cpu="$(env_cpu_m "$_TB_TRAINING_SIZE")"; b_cpu="$(env_cpu_m "$BEFORE")" -if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + NEED_M <= 8000 )); then - ok "cpu-only overshoot: ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" +if [[ "$_TB_FIT_VERDICT" == "reduced" ]] && (( a_mem == b_mem && a_cpu < b_cpu && a_cpu + 900 + SYS_M <= 8000 )); then + ok "cpu-only overshoot (pre-trim 900 m replayed): ${BEFORE} -> ${_TB_TRAINING_SIZE} (memory kept, cpu reduced)" else bad "cpu-only overshoot: verdict '${_TB_FIT_VERDICT}', ${BEFORE} -> ${_TB_TRAINING_SIZE}" fi @@ -237,9 +267,10 @@ else bad "measured system pods: $(( ${_TB_SYS_MEM_BYTES:-0} / MIB )) MiB / ${_TB_SYS_CPU_MILLI:-0} m (want ${SYS_MIB} / ${SYS_M}); note: ${_TB_SYS_NOTE:-}" fi -# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces. +# 7b. Pods unreadable: chart-only, and the verdict SAYS so; still reduces on the +# derived SMALL_MI node (the 8 GiB machine no longer needs reducing). PODS_READABLE=0 -resolve_and_fit "4 8Gi" +resolve_and_fit "4 ${SMALL_MI}Mi" if [[ "$_TB_FIT_VERDICT" == "reduced" && "$_TB_FIT_LINES" == *"NOT measured"*"chart derivation only"* ]]; then ok "pod list unreadable: verified against the chart derivation only, and said so (${BEFORE} -> ${_TB_TRAINING_SIZE})" else diff --git a/scripts/tests/image-refresh-skip-streak.bats b/scripts/tests/image-refresh-skip-streak.bats index 9fb7ad3e..f14fc9b9 100644 --- a/scripts/tests/image-refresh-skip-streak.bats +++ b/scripts/tests/image-refresh-skip-streak.bats @@ -107,15 +107,16 @@ exit 6 CURLSTUB chmod +x "$STUB_DIR/curl" - # The env the pod supplies. IMAGE_REGISTRY must be docker.io or the mirror - # guard exits 0 long before the settled guard is reached. + # The env the pod supplies. IMAGE_REGISTRY_RESOLVABLE is the chart-rendered + # verdict the mirror guard reads (tracebloc.imageRefreshResolvable); anything + # but "true" exits 0 long before the settled guard is reached. export PATH="$STUB_DIR:$PATH" export RELEASE_NAME=stg RELEASE_NAMESPACE=tracebloc export DEPLOYMENT_NAME=stg-jobs-manager export REQUESTS_PROXY_DEPLOYMENT=stg-requests-proxy export RESOURCE_MONITOR_DAEMONSET=stg-resource-monitor export NODE_AGENTS_NAMESPACE=tracebloc-node-agents - export IMAGE_REGISTRY=docker.io IMAGE_TAG=stg + export IMAGE_REGISTRY=docker.io IMAGE_REGISTRY_RESOLVABLE=true IMAGE_TAG=stg export ROLLOUT_TIMEOUT=300s # The script builds an explicit in-cluster kubeconfig from these (#634) and # runs under `set -u`, so they are not optional. diff --git a/scripts/tests/image-refresh-stale-pin.bats b/scripts/tests/image-refresh-stale-pin.bats index 8b84e958..32bbb310 100644 --- a/scripts/tests/image-refresh-stale-pin.bats +++ b/scripts/tests/image-refresh-stale-pin.bats @@ -75,6 +75,9 @@ run_branch() { cat > "$TMP/harness.sh" < 8192, and 3000 + 900 + 200 = 4100 m > 4000. + # 5120 + platform <= 8192 MiB and 3000 + 950 <= 4000 m: the resolver's + # cpu=3,memory=5Gi is written as-is, and nothing is reported OVER. _sched_cluster '4 8Gi' run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] || { echo "$output"; return 1; } - grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } - grep -q 'RESOURCE_LIMITS: "memory=4Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 + grep -q 'RESOURCE_REQUESTS: "cpu=3,memory=5Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + [[ "$output" != *"OVER"* ]] || { echo "$output"; return 1; } +} + +@test "install_client_helm: an envelope that over-asks is REDUCED in the written values, with the arithmetic (backend#2870)" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + has() { return 0; } + verify_credentials() { printf valid; } + unset TRACEBLOC_TRAINING_RESOURCES + # A node DERIVED from the footprint: below 4 GiB the resolver falls to the + # 1-core / 2-GiB floor; this node leaves room for 1 GiB but not 2, so the fit + # must reduce to memory=1Gi and print the OVER arithmetic. + local need small over + need="$(_sched_need_mib)"; small=$(( need + 1024 + 64 )); over=$(( 2048 + need - small )) + _sched_cluster "4 ${small}Mi" + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + grep -q 'RESOURCE_REQUESTS: "cpu=1,memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || { cat "$HOST_DATA_DIR/values.yaml"; return 1; } + grep -q 'RESOURCE_LIMITS: "memory=1Gi"' "$HOST_DATA_DIR/values.yaml" || return 1 grep -q 'RESOURCE_PROVENANCE: "installer"' "$HOST_DATA_DIR/values.yaml" || return 1 # The arithmetic is on screen, not just the verdict. - [[ "$output" == *"204 MiB OVER"* ]] || { echo "$output"; return 1; } - [[ "$output" == *"reduced cpu=3,memory=5Gi -> cpu=2,memory=4Gi"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"${over} MiB OVER"* ]] || { echo "$output"; return 1; } + [[ "$output" == *"reduced cpu=1,memory=2Gi -> cpu=1,memory=1Gi"* ]] || { echo "$output"; return 1; } } @test "install_client_helm: REFUSES to write an envelope when not even a 1-core/1-GiB run fits (backend#2870)" { @@ -1427,8 +1454,9 @@ _sched_cluster() { # $1 = node line(s) has() { return 0; } verify_credentials() { printf valid; } unset TRACEBLOC_TRAINING_RESOURCES - # 4 GiB allocatable: 4096 - 3136 - 140 = 820 MiB is all that is left. - _sched_cluster '4 4Gi' + # Derived: allocatable minus the platform is 64 MiB short of a 1 GiB run. + local tiny; tiny=$(( $(_sched_need_mib) + 1024 - 64 )) + _sched_cluster "4 ${tiny}Mi" run install_client_helm <<< $'myid\nmypw' [ "$status" -ne 0 ] || { echo "$output"; return 1; } [[ "$output" == *"Refusing to write a training envelope"* ]] || { echo "$output"; return 1; } diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 50804852..bfbd2aee 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -9780,41 +9780,62 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } $failures -join "`n" | Should -BeNullOrEmpty $checked | Should -BeGreaterOrEqual 10 -Because "a guard that checked almost nothing proves almost nothing" - # POSITIVE CONTROL: the platform out-requests the reserve today, so at least - # one vector must over-ask BEFORE the fit -- else this suite could never have - # seen the defect it exists for. + # POSITIVE CONTROL: while the platform out-requested the reserve (before + # backend#2461's trim) at least one vector had to over-ask BEFORE the fit -- + # else this suite could never have seen the defect it exists for. The guard is + # conditional on the embedded footprint, so it re-arms by itself the day a + # chart out-requests the reserve again. if ($needB -gt $script:TbEnvelopeOverheadMemBytes -or $needM -gt $script:TbEnvelopeOverheadCpuMilli) { $overBefore | Should -BeGreaterThan 0 -Because "the platform out-requests the reserve, yet no vector over-asked before the fit" } } - It "3. the ticket's reproduction: an 8 GiB node is REDUCED, arithmetic printed" { + It "3. the ticket's reproduction: an 8 GiB node now FITS unreduced (backend#2461 closed the over-ask)" { + # Before the interim trim the platform asked 3136 MiB / 900 m and this node was + # REDUCED to cpu=2,memory=4Gi; at 2336 MiB / 750 m the resolver's cpu=3,memory=5Gi + # fits: 5120 + 2476 <= 8192 and 3000 + 950 <= 4000. Pinning the fit is the + # regression guard for a chart change that pushes the footprint back up. $r = Invoke-FitOn -Nodes @('4 8Gi') + $r.Fit.Verdict | Should -Be 'fits' + $r.Fit.Size | Should -Be $r.Before + } + It "3b. a node DERIVED from the footprint (moves with the render) is REDUCED, arithmetic printed" { + # Below 4 GiB the resolver falls to the 1-core / 2-GiB floor; this node leaves + # room for 1 GiB but not for that floor, so the fit must reduce to 1 GiB with + # the OVER arithmetic on screen. + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*OVER*' ($r.Fit.Lines -join "`n") | Should -BeLike "*reduced $($r.Before) -> $($r.Fit.Size)*" - # The numbers on this node against the canned cluster: cpu 3000+900+200 = 4100 - # > 4000 -> 2 cores; memory 5120+3136+140 = 8396 > 8192 -> 8192-3276 = 4916 MiB - # -> 4 GiB. Both dimensions over-asked, both reduced. - $r.Fit.Size | Should -Be 'cpu=2,memory=4Gi' + $r.Fit.Size | Should -Be 'cpu=1,memory=1Gi' } - - It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic" { - $r = Invoke-FitOn -Nodes @('4 4Gi') + It "4. REFUSED when not even 1 core / 1 GiB fits, with the arithmetic (node derived)" { + # allocatable minus the platform is 64 MiB short of a 1 GiB run + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $tiny = $needMib + 1024 - 64 + $r = Invoke-FitOn -Nodes @("4 ${tiny}Mi") $r.Fit.Verdict | Should -Be 'refused' ($r.Fit.Lines -join "`n") | Should -BeLike '*not even a 1-core / 1-GiB run*' } - - It "5. cpu-only overshoot reduces cpu alone (memory kept)" { - $r = Invoke-FitOn -Nodes @('8 63928Mi') - $r.Fit.Verdict | Should -Be 'reduced' - (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) - $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) - $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) - $aCpu | Should -BeLessThan $bCpu - ($aCpu + $script:TbCpFootprintCpuMilli + $script:SysM) | Should -BeLessOrEqual 8000 + It "5. cpu-only overshoot reduces cpu alone (memory kept) -- the pre-trim 900 m replayed" { + # The resolver leaves one core for the platform and the trimmed platform needs + # 950 m, so no real node produces a cpu-only overshoot any more; the pre-trim + # footprint (900 m) did. Replaying it through the real fit keeps DoD part 5 + # covered; memory is untouched so only cpu can move. + $saved = $script:TbCpFootprintCpuMilli + try { + $script:TbCpFootprintCpuMilli = 900 + $r = Invoke-FitOn -Nodes @('8 63928Mi') + $r.Fit.Verdict | Should -Be 'reduced' + (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key memory) | Should -Be (Get-TbEnvelopeDimension -Size $r.Before -Key memory) + $aCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Fit.Size -Key cpu) + $bCpu = ConvertTo-TbCpuMilli (Get-TbEnvelopeDimension -Size $r.Before -Key cpu) + $aCpu | Should -BeLessThan $bCpu + ($aCpu + 900 + $script:SysM) | Should -BeLessOrEqual 8000 + } finally { $script:TbCpFootprintCpuMilli = $saved } } - It "6. a human's pin is warned, never altered (pinned-over)" { $r = Invoke-FitOn -Nodes @('4 8Gi') -Override 'cpu=4,memory=16Gi' $r.Prov | Should -Be 'user' @@ -9836,7 +9857,10 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "7b. pods unreadable: verified against the chart derivation only, and it SAYS so; still reduces" { - $r = Invoke-FitOn -Nodes @('4 8Gi') -PodsReadable $false + # on the derived small node -- the 8 GiB machine no longer needs reducing + $needMib = [long]($script:TbCpFootprintMemBytes / 1MB) + [long]$script:SysMib + $small = $needMib + 1024 + 64 + $r = Invoke-FitOn -Nodes @("4 ${small}Mi") -PodsReadable $false $r.Fit.Verdict | Should -Be 'reduced' ($r.Fit.Lines -join "`n") | Should -BeLike '*NOT measured*chart derivation only*' } @@ -9867,19 +9891,22 @@ Describe "Resolve-TbTrainingFit -- envelope schedulability (backend#2870, client } It "the reduction below the contract floor flags Undersized, and a fit does not" { - # 4c/5Gi: the resolver leaves cpu=3,memory=2Gi (5120-3072 = 2048 MiB, the - # floor exactly). Memory: 2048+3136+140 > 5120 -> 5120-3276 = 1844 MiB -> 1 GiB, - # below the 2 GiB floor -> Undersized. Cpu: 4000-1100 = 2900 m -> 2 cores, so - # the run is still a requestable shape and is REDUCED, not refused. (On a - # 2-core node the same memory would reduce but cpu would leave 900 m -- under - # one core -- and the verdict is refused; that is case 4's territory.) - $r = Invoke-FitOn -Nodes @('4 5Gi') - $r.Fit.Verdict | Should -Be 'reduced' - $r.Fit.Size | Should -Be 'cpu=2,memory=1Gi' - $r.Fit.Undersized | Should -BeTrue - # A genuine fit: an installer-sized envelope ALWAYS over-asks today (the - # resolver leaves 3 GiB and the platform asks 3136 MiB plus system pods -- the - # positive control above), so use a carried installer size that fits. + # A reduction that lands BELOW the 2 GiB floor needs the resolver to start at + # or above the floor (node >= 5 GiB) AND the platform to push it under -- which + # the trimmed 2336 MiB no longer can (5120 - 2476 = 2644 MiB >= 2 GiB). Replay + # the pre-trim footprint (3136 MiB) through the real fit, as case 5 does for + # cpu: 4c/5Gi -> cpu=3,memory=2Gi; memory 2048+3136+140 > 5120 -> 5120-3276 = + # 1844 MiB -> 1 GiB, below the floor -> Undersized. Cpu: 4000-950 = 3050 m -> + # 3 cores, still requestable, so REDUCED, not refused. + $saved = $script:TbCpFootprintMemBytes + try { + $script:TbCpFootprintMemBytes = 3136L * 1MB # Int64 literal: no Int32 product to argue about (Bugbot, twice) + $r = Invoke-FitOn -Nodes @('4 5Gi') + $r.Fit.Verdict | Should -Be 'reduced' + $r.Fit.Size | Should -Be 'cpu=3,memory=1Gi' + $r.Fit.Undersized | Should -BeTrue + } finally { $script:TbCpFootprintMemBytes = $saved } + # A genuine fit, on the real footprint: a carried installer size that fits. $r2 = Invoke-FitOn -Nodes @('16 64Gi') -Carried @{ Size = 'cpu=4,memory=12Gi'; Provenance = 'installer' } $r2.Before | Should -Be 'cpu=4,memory=12Gi' $r2.Fit.Verdict | Should -Be 'fits'