diff --git a/client/Chart.yaml b/client/Chart.yaml index 14a392bd..68ec1618 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.112 -appVersion: "1.9.112" +version: 1.9.113 +appVersion: "1.9.113" keywords: - tracebloc - kubernetes diff --git a/client/MIGRATION.md b/client/MIGRATION.md index 6a457ca7..07e6d0fc 100644 --- a/client/MIGRATION.md +++ b/client/MIGRATION.md @@ -2,6 +2,54 @@ This guide explains how to migrate from the legacy per-platform charts (`aks/`, `bm/`, `eks/`, `oc/`) to the unified `client/` chart. +## Upgrading to 1.9.113 — the control-plane images pull from `ghcr.io` by default + +`images.traceblocRegistry` now defaults to **`ghcr.io`**: the four +tracebloc-published control-plane images (jobs-manager, pods-monitor, +resource-monitor, and the requests-proxy, which runs the jobs-manager image) +are pulled from GitHub Container Registry instead of Docker Hub. The +image-refresh CronJob resolves digests there too (`IMAGE_REGISTRY`), and +`NOTES.txt` reports it. Nothing else moves: `tracebloc/mysql-client`, busybox +and the other third-party images keep their registries, the training-image host +(`JOB_IMAGE_HOST`) is unchanged, and a `global.imageRegistry` mirror still wins +over everything. + +**Why:** the control-plane packages are public on GHCR and published there by +digest alongside the Docker Hub copies (the GHCR migration). The same digests +exist on both registries, so this changes where the bytes come from, not which +bytes run. + +**What you need to do: nothing for most edges.** + +- **Egress.** No new host: `ghcr.io` (and `pkg-containers.githubusercontent.com`, + where GHCR redirects layer downloads) is already required for the ingestor + image and probed by the installer preflight. If your allowlist was built by + hand from an older egress table, add both before upgrading. +- **One rollout.** The image reference in each control-plane pod template + changes (`docker.io/…` → `ghcr.io/…`), so the upgrade rolls those pods once + and the kubelet pulls the new reference. `imagePullPolicy: IfNotPresent` + behaves as before from then on. +- **Digest pins.** `images.*.digest` values keep working unchanged — the pin + renders as `ghcr.io/tracebloc/@`, and the digest is the same + on both registries. +- **Mirrors.** Edges with `global.imageRegistry` set are unaffected: the mirror + re-homes every image and always wins. + +**Rollback (per edge):** point the knob back at Docker Hub. It is user-supplied, +so it persists across the fleet auto-upgrade until you clear it: + +```bash +helm upgrade tracebloc/client -n \ + --reset-then-reuse-values --set images.traceblocRegistry=docker.io +``` + +Confirm which registry an edge pulls from: + +```bash +kubectl get deploy -n -jobs-manager \ + -o jsonpath='{.spec.template.spec.containers[0].image}{"\n"}' +``` + ## Upgrading to 1.9.105 — `env.GPU_LIMITS` alone now renders an equal `GPU_REQUESTS` (client#995) Nothing to do if your values set both keys, or neither — that is every edge the diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 4b8e2e69..aaf9346c 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -618,8 +618,10 @@ NOTES.txt cannot disagree about where those images live: squid, alpine/*, the device plugins and the ingestor where they are. Also the per-edge rollback: set it to the previous registry. - 3. "docker.io" — the chart default while the images are - published to Docker Hub. + 3. "ghcr.io" — the chart default since the GHCR migration. + The images are still dual-published to + Docker Hub at the same digests, so + "docker.io" is the documented rollback. NOT routed through here, on purpose: `tracebloc/mysql-client` (frozen, digest-pinned, published only to Docker Hub — see images.mysqlClient), the @@ -630,14 +632,18 @@ Every read is nil-guarded and `| default`-chained: values.yaml ships `global.imageRegistry: ""` (the key EXISTS, so `dig`'s own fallback never applies — the trap image_refresh_test.yaml pins), and an edge upgrading with `--reuse-values` from before `images.traceblocRegistry` existed has no such key -at all. Both must render the default, never "". +at all. Both must render the default, never "". The literal below RESTATES +values.yaml's `images.traceblocRegistry` default -- unavoidably, since a +template cannot read the chart's defaults apart from the merged values -- so +tests/tracebloc_registry_test.yaml pins both to one value: the chart-default +tests read values.yaml, the EMPTY-knob tests read this literal. Call with the ROOT context: {{ include "tracebloc.tbRegistry" . }} */}} {{- define "tracebloc.tbRegistry" -}} {{- $mirror := dig "imageRegistry" "" (.Values.global | default dict) -}} {{- $own := dig "traceblocRegistry" "" (.Values.images | default dict) -}} -{{- $mirror | default ($own | default "docker.io") -}} +{{- $mirror | default ($own | default "ghcr.io") -}} {{- end -}} {{/* diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index d57bdaae..352c0921 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -100,13 +100,35 @@ data: # would rewrite `repo:tag` to `repo@digest` on every fresh install — a spec # change, therefore a rollout, for byte-identical content, and for the # resource-monitor DaemonSet that is a rollout across every node. So the - # first tick only RECORDS. The NEXT tick, seeing `recorded == latest` but the - # workload still on `:tag`, pins the digest (client-runtime#199) — so a fresh edge - # becomes reproducible ~one interval post-install, NOT "at the next upstream - # release". Between the two it is still restart-safe offline (IfNotPresent). + # first tick only RECORDS. It stays restart-safe offline throughout + # (IfNotPresent). # - # COST, stated honestly (@shujaatTracebloc / @LukasWodka on #1008): that - # re-pin is not "one cheap rollout". It enters the shared #563 flap path — + # A fresh install then STAYS on `:tag` (#1008). The re-pin on + # `recorded == latest` reads the live workload and, when it is off the digest, + # skips the roll ONLY on positive fresh-install evidence — two markers: + # `tracebloc.io/first-observed-` (stamped by the first-observation arm + # above) and `tracebloc.io/digest-applied-` (stamped only after a + # re-image rollout succeeds). first_observed present AND applied absent is a + # workload we watched born on `:tag` here, so it is left on `:tag`: the + # install already pulled the current tag, there is no stale-`:tag`-layer + # exposure to repair, and the first genuine UPSTREAM digest change pins it + # (and stamps applied). applied present is an established edge and rolls — + # the client-runtime#199 repair of a helm re-render that reverted the pin. + # + # NEITHER marker is a PRE-MARKER / legacy edge (pinned by a version that + # predates these markers): it is NOT skipped. The upgrade shipping this chart + # re-renders the Deployment back to `:tag` (the same #199 premise), so on the + # next tick every established-but-unmarked edge is byte-for-byte the fresh + # shape; skipping on absence alone would strand the WHOLE existing fleet on a + # possibly-stale `:tag` until the next upstream digest change (LukasWodka on + # #1008). So a legacy edge takes the repair roll once, which stamps applied. + # This trades "a fresh edge becomes digest-reproducible ~one interval + # post-install" for skipping the fresh-install roll outright — the cost below + # is why (@shujaatTracebloc / @LukasWodka on #1008). + # + # COST of that roll — which the marker gate now avoids on a fresh install — + # stated honestly (@shujaatTracebloc / @LukasWodka on #1008): the re-pin is + # not "one cheap rollout". It enters the shared #563 flap path — # `rollout status` on the resource-monitor DaemonSet, whose # `desiredNumberScheduled` counts every node (tolerations: Exists), so it can # never settle on a fleet with one NotReady/cordoned node; three such ticks @@ -114,11 +136,10 @@ data: # lockout and stop refresh for ALL control-plane images until a human clears # ATTEMPT_KEY, while the CronJob stays green. And jobs-manager is # `strategy: Recreate`, so its extra rollout is full downtime + wait-for-mysql - # for byte-identical content, on every fresh install and again after each - # chart-version bump. The widening is kept deliberately — it also repairs a - # reinstall onto a node whose `:tag` layer is already stale — but that is the - # price, and a follow-up may gate the re-pin on "have we ever applied a digest - # here?" so a genuine fresh install skips the flap path entirely. + # for byte-identical content. That price is now paid ONLY when it buys + # something — repairing an established or legacy edge whose pin a helm + # re-render reverted, including a chart-version bump on such an edge. A fresh + # install, watched born on `:tag` here, no longer pays it at all. # # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys # or container/env-array filtering motivates it; the rest stays in pure @@ -472,7 +493,8 @@ data: # ================================================================= # Pass 1 — the control-plane images: jobs-manager, pods-monitor, - # resource-monitor. docker.io, floating CLIENT_ENV tag, + # resource-monitor. IMAGE_REGISTRY (tracebloc.tbRegistry), floating + # CLIENT_ENV tag, # annotation-based source of truth, action = kubectl set image. # ================================================================= @@ -517,6 +539,27 @@ data: pinned="${rest%%|*}" pin_digest="${rest#*|}" + # #1008: two durable markers that let the `recorded == latest` / off-digest + # arm below tell a genuine fresh install from a helm-revert of an applied + # pin -- WITHOUT mistaking the pre-marker fleet for either. The skip fires + # only on POSITIVE fresh-install evidence, never on mere absence: + # first_observed_key written by the first-observation arm (we saw this + # workload with no annotation and recorded the digest + # WITHOUT applying it -- proof it was born on :tag here). + # applied_key written alongside the digest record only after a + # re-image rollout succeeds -- proof a digest was + # actually rolled onto this workload. + # Fresh install = first_observed present AND applied absent -> skip the roll. + # Established edge = applied present -> roll (the client-runtime#199 repair). + # Pre-marker / legacy edge = NEITHER marker (nothing wrote them before this + # version): it is NOT positively a fresh install, so it keeps the repair + # roll -- the fleet upgrading INTO this version reverts to :tag on the + # upgrade hop, and skipping it on absence alone would strand every such edge + # on a possibly-stale :tag until the next upstream digest change (LukasWodka + # on #1008). Same `${repo#*/}` suffix convention as the stale-pin keys. + applied_key="tracebloc.io/digest-applied-${repo#*/}" + first_observed_key="tracebloc.io/first-observed-${repo#*/}" + log "checking ${repo}:${IMAGE_TAG} (annotation=${key})" if [ "$pinned" = "1" ]; then @@ -603,7 +646,18 @@ data: continue fi - recorded="$(get_annotation "$key" || true)" + # FAIL-CLOSED, not `|| true` (Bugbot on #1008). A read ERROR here must NOT + # collapse to "" and fall into the first-observation arm below: that arm now + # stamps first_observed_key, which SUPPRESSES the client-runtime#199 repair + # on later ticks. So a transient kubectl/jq error on a legacy edge (which + # genuinely HAS a recorded digest) would mis-stamp it as first-observed and + # strand it on :tag. get_annotation returns non-zero on a read error and + # zero+empty only on genuine absence, so skip this image this tick on error + # and retry -- the same stance the marker reads and the settled guard take. + if ! recorded="$(get_annotation "$key")"; then + log " WARN: could not read ${key} (kubectl/jq error); skipping this image this tick rather than mistaking a read error for a first observation. Will retry next tick." + continue + fi log " latest=$latest" log " recorded=${recorded:-}" @@ -612,9 +666,14 @@ data: # First-observation path: record without re-imaging. No evidence # the running pod is on an older digest, and rewriting repo:tag to # repo@digest for identical content would roll every pod on install - # (see the first-tick contract in the header). + # (see the first-tick contract in the header). Stamp first_observed_key + # too (#1008): this is the POSITIVE evidence that lets the off-digest arm + # skip the fresh-install roll -- a workload we watched from its first + # annotation-less tick. A pre-marker edge upgrading into this version + # never gets this stamp (its first observation predates the marker), so + # it is not mistaken for a fresh install and keeps the repair roll. log " first observation; recording without re-imaging" - annotate_args="$annotate_args ${key}=${latest}" + annotate_args="$annotate_args ${key}=${latest} ${first_observed_key}=1" continue fi @@ -630,11 +689,14 @@ data: # pin whenever the live workload is not already on the pinned digest. # # The re-pin is BOUNDED TO ONE TICK: it writes the digest back and the - # next tick sees the workload on it and no-ops. It also converges a FRESH - # install to the digest one tick after first observation -- the first-tick - # contract in the header records without re-imaging, and this completes - # it, because staying on `:tag` is exactly the steady-state stale-`:tag` - # exposure this fix closes. + # next tick sees the workload on it and no-ops. It fires to repair an + # established or legacy edge whose pin a helm re-render reverted onto + # `:tag`, the steady-state stale-`:tag` exposure this fix closes. A fresh + # install -- one we watched born on `:tag` (first_observed set, applied + # absent) -- is GATED OUT of this roll (see the two-marker check in the + # off-digest arm below) and left on `:tag` (#1008); the first genuine + # upstream digest change pins it. A pre-marker/legacy edge has neither + # marker and is NOT gated out, so the existing fleet is not stranded. # # Compare on the @sha256 DIGEST, not the whole image reference: a mutating # admission webhook that rewrites the registry PREFIX to an internal mirror @@ -724,8 +786,53 @@ data: log " deployment/${REQUESTS_PROXY_DEPLOYMENT} runs '${rp_have}', not digest ${latest}" log " -- re-pinning the requests-proxy digest" else - log " digest unchanged, but the workload runs '${have}', not digest ${latest}" - log " (fresh install, or a helm re-render reverted the pin onto :${IMAGE_TAG}) -- re-pinning the digest" + # The workload is fully on :tag, not the digest. THREE shapes reach + # here, and the skip must fire only on the fresh-install one (#1008): + # * ESTABLISHED edge, helm-reverted -- a prior tick pinned the digest + # (rollout succeeded, so ${applied_key} is set), then a helm + # re-render discarded it back to :tag. On a node whose :tag layer + # is stale this silently runs an OLD image, so re-pinning is the + # client-runtime#199 repair and MUST run. (applied present) + # * FRESH INSTALL -- the first-observation tick recorded the digest + # and applied nothing, and the workload has been on :tag since + # install. No prior digest to have reverted FROM: the install just + # pulled the current :tag, so re-pinning buys no staleness + # protection -- it only pays the full fresh-install roll (the #563 + # flap path on the resource-monitor DaemonSet under + # `tolerations: Exists`, and a `strategy: Recreate` downtime on + # jobs-manager) for byte-identical content. + # (first_observed present AND applied absent) + # * PRE-MARKER / LEGACY edge -- pinned by a version that predates + # these markers, so it has NEITHER. The upgrade that ships this + # chart re-renders the Deployment back to :tag (the same #199 + # premise), so on the next tick it is byte-for-byte the fresh + # shape. Skipping on absence alone would strand the WHOLE existing + # fleet on a possibly-stale :tag until the next upstream digest + # change (weeks on :prod) -- the exact #199 exposure, fleet-wide. + # So a legacy edge (no first_observed) is NOT skipped: it takes the + # repair roll, which then stamps applied_key. (neither marker) + # Skip ONLY on positive fresh-install evidence. A read ERROR on either + # marker (as opposed to genuine absence) skips the re-pin this tick and + # retries -- the same fail-closed stance the unreadable-`have` skip + # above takes -- rather than guessing the shape. + if ! first_observed="$(get_annotation "$first_observed_key")"; then + log " digest unchanged and workload off-digest, but ${first_observed_key} was unreadable (API/jq error) -- skipping re-pin this tick, will retry when readable" + continue + fi + if ! applied="$(get_annotation "$applied_key")"; then + log " digest unchanged and workload off-digest, but ${applied_key} was unreadable (API/jq error) -- skipping re-pin this tick, will retry when readable" + continue + fi + if [ -n "$first_observed" ] && [ -z "$applied" ]; then + log " digest unchanged and workload runs '${have}' on :${IMAGE_TAG}, but this is a fresh install (observed here, no digest ever applied) -- NOT rolling (#1008); it stays on :${IMAGE_TAG} (IfNotPresent-safe) until a genuine upstream digest change pins it." + continue + fi + if [ -z "$first_observed" ] && [ -z "$applied" ]; then + log " digest unchanged, but the workload runs '${have}', not digest ${latest}, and this edge predates the fresh-install markers -- re-pinning to repair a possibly-stale :${IMAGE_TAG} on the upgrade hop (client-runtime#199)" + else + log " digest unchanged, but the workload runs '${have}', not digest ${latest}" + log " (a helm re-render reverted the pin onto :${IMAGE_TAG}) -- re-pinning the digest" + fi fi # Fall through to the re-image path (ref + case block) with recorded # already == latest: the annotate below is an idempotent re-write, and @@ -733,7 +840,14 @@ data: else log " digest changed (${recorded} -> ${latest}); re-image needed" fi - annotate_args="$annotate_args ${key}=${latest}" + # Record the digest AND stamp applied_key -- proof a digest has now been + # rolled onto this workload (#1008). Both land in the SAME annotate below, + # after a successful rollout, so they share one fate: whenever a later tick + # reads `recorded == latest`, the applied marker it gates on is present too. + # The first-observation arm stamps first_observed_key instead (it applies + # nothing), and the two markers are what let the off-digest arm above tell a + # fresh install from an established edge without stranding the legacy fleet. + annotate_args="$annotate_args ${key}=${latest} ${applied_key}=1" restart_needed=1 # The reference we pin: the SAME registry the digest above was resolved diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index 1ef868ce..f7f14497 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -49,7 +49,8 @@ spec: # `Always` re-pulled on every restart even when the image was already # cached, so an offline Docker/WSL restart landed in ImagePullBackOff. # - # This Deployment runs the SAME docker.io/tracebloc/jobs-manager image as + # This Deployment runs the SAME tracebloc/jobs-manager image (from the + # tracebloc.tbRegistry registry) as # the jobs-manager Deployment, and image-refresh now reconciles both to # the same digest in one tick. Before #569 it reconciled only # `-jobs-manager`, so requests-proxy silently skewed: it stayed diff --git a/client/tests/client_env_alias_test.yaml b/client/tests/client_env_alias_test.yaml index 87754470..56363f68 100644 --- a/client/tests/client_env_alias_test.yaml +++ b/client/tests/client_env_alias_test.yaml @@ -43,7 +43,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager:stg + value: ghcr.io/tracebloc/jobs-manager:stg - it: "staging resolves to :stg for the pods-monitor sidecar" template: templates/jobs-manager-deployment.yaml @@ -52,7 +52,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[1].image - value: docker.io/tracebloc/pods-monitor:stg + value: ghcr.io/tracebloc/pods-monitor:stg - it: "staging resolves to :stg for requests-proxy (which runs the jobs-manager image)" template: templates/requests-proxy-deployment.yaml @@ -61,7 +61,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager:stg + value: ghcr.io/tracebloc/jobs-manager:stg - it: "staging resolves to :stg for the resource-monitor daemonset" template: templates/resource-monitor-daemonset.yaml @@ -70,7 +70,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/resource-monitor:stg + value: ghcr.io/tracebloc/resource-monitor:stg - it: "production and development resolve too, not just staging" template: templates/jobs-manager-deployment.yaml @@ -79,7 +79,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager:prod + value: ghcr.io/tracebloc/jobs-manager:prod - it: "a canonical value is unchanged by resolution" template: templates/jobs-manager-deployment.yaml @@ -88,7 +88,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager:stg + value: ghcr.io/tracebloc/jobs-manager:stg # ---- the auto-upgrade CronJob --------------------------------------- # The most consequential site: this is what keeps an installed edge diff --git a/client/tests/global_image_registry_test.yaml b/client/tests/global_image_registry_test.yaml index 1c6f1e49..19c17915 100644 --- a/client/tests/global_image_registry_test.yaml +++ b/client/tests/global_image_registry_test.yaml @@ -5,7 +5,8 @@ suite: global.imageRegistry private-mirror re-homing (#585) # convention. Two invariants are pinned here: # 1. Set -> every image (tracebloc/*, the spawned ingestor + training jobs, # and the alpine/*, ubuntu/squid utility images) carries the mirror prefix. -# 2. Unset -> byte-identical to the pre-#585 chart: tracebloc/* + squid on +# 2. Unset -> the chart's own defaults, untouched: tracebloc/* on +# images.traceblocRegistry (ghcr.io since the GHCR migration), squid on # docker.io, ingestor on ghcr.io, alpine/* unprefixed (docker.io implicit). # Precedence guards: an explicit images.ingestor.repository still wins over the # mirror (someone who names a full repo means it); an explicit per-image @@ -116,14 +117,14 @@ tests: value: "mirror.corp.example/" # --------------------------------------------------------------------------- - # 2. Mirror UNSET -> byte-identical to pre-#585 defaults + # 2. Mirror UNSET -> the chart's own defaults, untouched # --------------------------------------------------------------------------- - - it: leaves tracebloc.image on docker.io when no mirror is set + - it: leaves tracebloc.image on the chart default (ghcr.io) when no mirror is set template: templates/jobs-manager-deployment.yaml asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager" + pattern: "^ghcr\\.io/tracebloc/jobs-manager" - it: leaves the ingestor repository on ghcr.io when no mirror is set template: templates/jobs-manager-deployment.yaml diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index e1f8ddda..ad457ebf 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -692,14 +692,14 @@ tests: name: NODE_AGENTS_NAMESPACE value: "tracebloc-node-agents" - - it: IMAGE_REGISTRY defaults to docker.io, not the empty string + - it: IMAGE_REGISTRY defaults to ghcr.io, not the empty string # Regression guard with teeth. values.yaml ships `global.imageRegistry: ""`, - # so `dig "imageRegistry" "docker.io" .Values.global` returns "" — the key - # EXISTS, so dig's own default never applies. Rendering "" here would make - # the script's mirror guard see "" != "docker.io" and go INERT on every - # default install, silently disabling auto-refresh fleet-wide. The template - # needs a trailing `| default "docker.io"`; this test is what catches its - # removal. + # so a bare `dig "imageRegistry" .Values.global` returns "" — the + # key EXISTS, so dig's own default never applies. Rendering "" here would + # make the script's resolvable guard go INERT on every default install, + # silently disabling auto-refresh fleet-wide. tracebloc.tbRegistry owns the + # `| default` chain that ends in the chart default (ghcr.io since the GHCR + # migration); this test is what catches its removal. template: templates/image-refresh-cronjob.yaml documentIndex: 1 asserts: @@ -707,7 +707,7 @@ tests: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: IMAGE_REGISTRY - value: "docker.io" + value: "ghcr.io" - it: IMAGE_REGISTRY follows a configured private mirror template: templates/image-refresh-cronjob.yaml @@ -972,3 +972,51 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: 'workload already on the pinned digest; no-op' + + - it: a fresh install is gated OUT of the re-pin roll, but the legacy fleet is not + # Guards #1008 item 2. With `recorded == latest` and the workload on :tag, the + # loop must tell a FRESH install (watched born on :tag here) from a helm-revert + # of an applied pin AND from a pre-marker/legacy edge, and skip ONLY the fresh + # one -- else either every fresh install pays the #563 flap-path / Recreate + # cost for byte-identical content, or (if it skipped on absence alone) the + # whole existing fleet is stranded on a stale :tag on the upgrade hop. Two + # markers discriminate: `first-observed-` (stamped by first-observation) + # and `digest-applied-` (stamped only on a successful re-image). Skip = + # first_observed present AND applied absent. image-refresh-repin-on-revert.bats + # asserts the BEHAVIOUR across all three shapes (fresh / established / legacy); + # these lock the code shapes so a refactor cannot silently drop the gate. + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + # both marker keys, derived per image with the same suffix as stale-pin keys + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'applied_key="tracebloc\.io/digest-applied-\$\{repo#\*/\}"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'first_observed_key="tracebloc\.io/first-observed-\$\{repo#\*/\}"' + # the skip fires ONLY on positive fresh-install evidence, not on absence + - matchRegex: + path: data["image-refresh.sh"] + pattern: '\[ -n "\$first_observed" \] && \[ -z "\$applied" \]' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'this is a fresh install .* -- NOT rolling' + # first-observation stamps first_observed (the positive fresh-install signal) + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'annotate_args="\$annotate_args \$\{key\}=\$\{latest\} \$\{first_observed_key\}=1"' + # a re-image stamps applied alongside the digest record (one shared fate) + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'annotate_args="\$annotate_args \$\{key\}=\$\{latest\} \$\{applied_key\}=1"' + # the digest-record read is FAIL-CLOSED: a read error must not collapse to "" + # and enter the first-observation arm (which stamps first_observed and + # suppresses the #199 repair on a legacy edge) -- Bugbot on #1008. The + # fail-open `|| true` form must be gone; the guarded read must be present. + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'if ! recorded="\$\(get_annotation "\$key"\)"; then' + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'recorded="\$\(get_annotation "\$key" \|\| true\)"' diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index ef289d79..1c4d4234 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -100,11 +100,11 @@ tests: path: spec.template.spec.containers[0].env[?(@.name=="CLIENT_ID")].valueFrom.secretKeyRef.name value: RELEASE-NAME-secrets - - it: should use docker.io registry by default + - it: should pull the control-plane images from ghcr.io by default (the GHCR migration) asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/" + pattern: "^ghcr\\.io/tracebloc/" - it: should set JOB_IMAGE_HOST to docker.io asserts: @@ -1131,7 +1131,7 @@ tests: # tracebloc.controlPlanePullPolicy helper. # ===================================================================== - - it: both containers are IfNotPresent by default (refresh on, docker.io) + - it: both containers are IfNotPresent by default (refresh on, ghcr.io) # The offline-restart fix: `Always` forced a registry round-trip on every # (re)start, so a Docker/WSL restart without the registry landed in # ImagePullBackOff on a cached image. Safe here because image-refresh drives @@ -1145,7 +1145,7 @@ tests: value: IfNotPresent - it: falls back to Always behind a private mirror (#569, Bugbot High) - # The reconcile resolves digests from docker.io, so it goes inert under a + # The reconcile resolves digests on the public registry, so it goes inert under a # mirror rather than pinning a digest the mirror may not hold. If the pods # were IfNotPresent anyway, that edge would have NO update path at all: # syncing the mirror and restarting would keep serving the cached tag, and diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml index 5161e1d0..3e1fb973 100644 --- a/client/tests/requests_proxy_test.yaml +++ b/client/tests/requests_proxy_test.yaml @@ -73,11 +73,11 @@ tests: path: spec.template.spec.containers[0].securityContext.capabilities.drop content: "ALL" - - it: should pull the jobs-manager image from docker.io + - it: should pull the jobs-manager image from ghcr.io by default (the GHCR migration) asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager[:@]" + pattern: "^ghcr\\.io/tracebloc/jobs-manager[:@]" # #569: still floats on the tag by default, but the pull policy is now # IfNotPresent REGARDLESS of pinning. `Always` forced a registry round-trip on @@ -89,7 +89,7 @@ tests: asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager:" + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent @@ -104,7 +104,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager@sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8 + value: ghcr.io/tracebloc/jobs-manager@sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8 - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent @@ -211,7 +211,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + value: ghcr.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent @@ -226,7 +226,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: docker.io/tracebloc/jobs-manager@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + value: ghcr.io/tracebloc/jobs-manager@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb - it: still renders (no nil-pointer) when the images block is entirely absent # --reuse-values replay from a release predating these keys. @@ -235,7 +235,7 @@ tests: asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager:" + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent diff --git a/client/tests/resource_monitor_test.yaml b/client/tests/resource_monitor_test.yaml index 6aa1f1bd..2e9c7d24 100644 --- a/client/tests/resource_monitor_test.yaml +++ b/client/tests/resource_monitor_test.yaml @@ -19,7 +19,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: "docker.io/tracebloc/resource-monitor:prod" + value: "ghcr.io/tracebloc/resource-monitor:prod" - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent @@ -33,14 +33,16 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: "docker.io/tracebloc/resource-monitor@sha256:0000000000000000000000000000000000000000000000000000000000000000" + value: "ghcr.io/tracebloc/resource-monitor@sha256:0000000000000000000000000000000000000000000000000000000000000000" - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent # Regression guard: `helm upgrade --reuse-values` from <1.0.7 carries forward # stored values that never included images.resourceMonitor. The template must - # tolerate the whole images map being missing without nil-pointer. + # tolerate the whole images map being missing without nil-pointer. With no + # images.traceblocRegistry key at all, the registry comes from the + # tracebloc.tbRegistry helper's own fallback, which is the chart default. - it: should render when images block is entirely absent template: templates/resource-monitor-daemonset.yaml set: @@ -48,7 +50,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: "docker.io/tracebloc/resource-monitor:prod" + value: "ghcr.io/tracebloc/resource-monitor:prod" - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent diff --git a/client/tests/tracebloc_registry_test.yaml b/client/tests/tracebloc_registry_test.yaml index a3386a61..a42aeef4 100644 --- a/client/tests/tracebloc_registry_test.yaml +++ b/client/tests/tracebloc_registry_test.yaml @@ -4,7 +4,13 @@ suite: images.traceblocRegistry — the tracebloc-only registry knob (tracebloc. # published to both docker.io and ghcr.io at the same digests. ONE helper, # tracebloc.tbRegistry, decides which registry the chart pulls them from, as one # precedence chain: global.imageRegistry (a mirror re-homes everything) wins, -# then images.traceblocRegistry, then the chart default. +# then images.traceblocRegistry, then the chart default -- ghcr.io since the +# GHCR migration, with docker.io the documented per-edge rollback. +# +# Because ghcr.io IS the default, a knob set to ghcr.io exercises nothing: every +# override test below sets the knob to docker.io (the rollback) or quay.example +# (a registry the refresh script has no token arm for), never to the default. +# That is what lets a helper that ignores the knob go red here. # # Pinned here, each of them a machine check for a sentence in values.yaml: # 1. The knob moves EXACTLY the tracebloc control-plane sites: busybox, the @@ -33,38 +39,42 @@ set: clientId: "test-id" clientPassword: "test" tests: - - it: the chart default pulls the jobs-manager Deployment's images from docker.io + - it: the chart default pulls the jobs-manager Deployment's images from ghcr.io template: templates/jobs-manager-deployment.yaml asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager:" + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - matchRegex: path: spec.template.spec.containers[1].image - pattern: "^docker\\.io/tracebloc/pods-monitor:" + pattern: "^ghcr\\.io/tracebloc/pods-monitor:" - - it: the chart default pulls the requests-proxy image from docker.io + - it: the chart default pulls the requests-proxy image from ghcr.io template: templates/requests-proxy-deployment.yaml asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager:" + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - - it: the chart default pulls the resource-monitor image from docker.io + - it: the chart default pulls the resource-monitor image from ghcr.io template: templates/resource-monitor-daemonset.yaml asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/resource-monitor:" + pattern: "^ghcr\\.io/tracebloc/resource-monitor:" - it: an EMPTY knob renders the default registry on the pods, not an empty one (the dig-empty trap) # values.yaml ships every registry key as a string, so `dig` finds the key # and returns "" instead of its fallback; an edge that clears the knob, or a # --reuse-values replay from before it existed, must still get a registry. - # The pods are guarded twice (tracebloc.image applies its own default to an - # empty registry), so this test alone would stay green if the helper lost - # its `| default` chain -- the IMAGE_REGISTRY test right below is the one - # with no second net, and it is what reddens under that mutation. + # The helper's fallback literal RESTATES values.yaml's default; this test + # reads the literal and the chart-default tests above read values.yaml, and + # both pin ghcr.io, so the two cannot drift apart unseen. (tracebloc.image's + # own empty-registry default is docker.io, so a helper that lost its + # `| default` chain reddens here as well as in the IMAGE_REGISTRY test.) template: templates/jobs-manager-deployment.yaml set: images: @@ -72,7 +82,7 @@ tests: asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^docker\\.io/tracebloc/jobs-manager:" + pattern: "^ghcr\\.io/tracebloc/jobs-manager:" - it: an EMPTY knob renders the default registry into IMAGE_REGISTRY, not an empty one template: templates/image-refresh-cronjob.yaml @@ -85,55 +95,55 @@ tests: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: IMAGE_REGISTRY - value: "docker.io" + value: "ghcr.io" - - it: images.traceblocRegistry re-homes the jobs-manager Deployment (init, api, pods-monitor) + - it: images.traceblocRegistry re-homes the jobs-manager Deployment (init, api, pods-monitor) -- the docker.io rollback template: templates/jobs-manager-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegex: path: spec.template.spec.initContainers[0].image - pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + pattern: "^docker\\.io/tracebloc/jobs-manager:" - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + pattern: "^docker\\.io/tracebloc/jobs-manager:" - matchRegex: path: spec.template.spec.containers[1].image - pattern: "^ghcr\\.io/tracebloc/pods-monitor:" + pattern: "^docker\\.io/tracebloc/pods-monitor:" - - it: images.traceblocRegistry re-homes the requests-proxy + - it: images.traceblocRegistry re-homes the requests-proxy -- the docker.io rollback template: templates/requests-proxy-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^ghcr\\.io/tracebloc/jobs-manager:" + pattern: "^docker\\.io/tracebloc/jobs-manager:" - - it: images.traceblocRegistry re-homes the resource-monitor + - it: images.traceblocRegistry re-homes the resource-monitor -- the docker.io rollback template: templates/resource-monitor-daemonset.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "^ghcr\\.io/tracebloc/resource-monitor:" + pattern: "^docker\\.io/tracebloc/resource-monitor:" - - it: a digest pin keeps the knob's registry (repo@digest, not repo:tag) + - it: a digest pin keeps the knob's registry (repo@digest, not repo:tag) -- the docker.io rollback template: templates/jobs-manager-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io jobsManager: digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" asserts: - equal: path: spec.template.spec.containers[0].image - value: "ghcr.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + value: "docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - it: the knob does NOT move mysql-client (frozen, digest-pinned, docker.io only) # The open question on mysql-client, recorded on the epic: it stays on @@ -142,7 +152,7 @@ tests: template: templates/mysql-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: quay.example asserts: - matchRegex: path: spec.template.spec.containers[0].image @@ -184,7 +194,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegex: path: spec.template.spec.containers[0].image @@ -196,7 +206,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegex: path: spec.template.spec.containers[0].image @@ -209,7 +219,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env @@ -257,46 +267,51 @@ tests: path: data["image-refresh.sh"] pattern: 'ghcr\.io\) _host="ghcr\.io" ;;' - - it: on docker.io the reconcile is resolvable + - it: at the chart default the CronJob resolves on ghcr.io and the reconcile is resolvable template: templates/image-refresh-cronjob.yaml documentIndex: 1 asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: IMAGE_REGISTRY + value: "ghcr.io" - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: IMAGE_REGISTRY_RESOLVABLE value: "true" - - it: on docker.io the pods are IfNotPresent + - it: at the chart default (ghcr.io) the pods are IfNotPresent template: templates/jobs-manager-deployment.yaml asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent - - it: on ghcr.io the reconcile is resolvable and IMAGE_REGISTRY follows + - it: on the docker.io rollback the reconcile is resolvable and IMAGE_REGISTRY follows template: templates/image-refresh-cronjob.yaml documentIndex: 1 set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: IMAGE_REGISTRY - value: "ghcr.io" + value: "docker.io" - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: IMAGE_REGISTRY_RESOLVABLE value: "true" - - it: on ghcr.io the jobs-manager pods are IfNotPresent + - it: on the docker.io rollback the jobs-manager pods are IfNotPresent template: templates/jobs-manager-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy @@ -305,21 +320,21 @@ tests: path: spec.template.spec.containers[1].imagePullPolicy value: IfNotPresent - - it: on ghcr.io the requests-proxy is IfNotPresent + - it: on the docker.io rollback the requests-proxy is IfNotPresent template: templates/requests-proxy-deployment.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent - - it: on ghcr.io the resource-monitor is IfNotPresent + - it: on the docker.io rollback the resource-monitor is IfNotPresent template: templates/resource-monitor-daemonset.yaml set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy @@ -389,7 +404,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env @@ -403,7 +418,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy @@ -413,18 +428,18 @@ tests: template: templates/NOTES.txt asserts: - matchRegexRaw: - pattern: "Image registry:.*docker\\.io.*\\(tracebloc control-plane images\\)" + pattern: "Image registry:.*ghcr\\.io.*\\(tracebloc control-plane images\\)" - - it: NOTES follows images.traceblocRegistry + - it: NOTES follows images.traceblocRegistry -- the docker.io rollback template: templates/NOTES.txt set: images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegexRaw: - pattern: "Image registry:.*ghcr\\.io" - - notMatchRegexRaw: pattern: "Image registry:.*docker\\.io" + - notMatchRegexRaw: + pattern: "Image registry:.*ghcr\\.io" - it: NOTES follows the global mirror over the knob template: templates/NOTES.txt @@ -432,7 +447,7 @@ tests: global: imageRegistry: mirror.corp.example images: - traceblocRegistry: ghcr.io + traceblocRegistry: docker.io asserts: - matchRegexRaw: pattern: "Image registry:.*mirror\\.corp\\.example" diff --git a/client/values.yaml b/client/values.yaml index 4e62469c..d1e33b0c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -934,14 +934,22 @@ images: # nowhere else -- see images.mysqlClient). The training-image host # (JOB_IMAGE_HOST) is not routed through it yet; that moves in its own step. # - # The tracebloc images are published to BOTH docker.io and ghcr.io, at the - # same digests. Setting this to "ghcr.io" on one edge is how to try the - # ghcr.io copies before the chart default moves; setting it back is the - # per-edge rollback once it has. Image-refresh follows it: the CronJob - # resolves digests on this registry (it can do so anonymously on docker.io - # and ghcr.io; any other value makes the reconcile inert and the pods + # ghcr.io is where the control-plane images are published since the GHCR + # migration, and it is the chart default. They are still dual-published to + # docker.io at the same digests for now, so "docker.io" remains a valid value + # and is the per-edge rollback (`--set images.traceblocRegistry=docker.io`; + # user-supplied, so it survives the fleet auto-upgrade until cleared). Digest + # pins (images.*.digest) are registry-agnostic: the same digest exists on + # both. No new egress is needed: ghcr.io (+ pkg-containers.githubusercontent.com + # for layer redirects) is already required for the ingestor image and probed + # by the installer preflight. Image-refresh follows it: the CronJob resolves + # digests on this registry (it can do so anonymously on ghcr.io and docker.io; + # any other value makes the reconcile inert and the pods # imagePullPolicy=Always, exactly as a global.imageRegistry mirror does). - traceblocRegistry: "docker.io" + # The tracebloc.tbRegistry helper's own fallback for an EMPTY or absent key + # restates this default; tests/tracebloc_registry_test.yaml pins both to one + # value so they cannot drift apart unseen. + traceblocRegistry: "ghcr.io" jobsManager: digest: "" # -- TRAINING IMAGES: the digest-pinned spawn path (RFC-1246 P2, landed for diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 0480884b..7b43ad6d 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -33,8 +33,8 @@ The standalone installer runs a **preflight** check that verifies this connectiv | Host | Why | |---|---| -| `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox + the tracebloc client images | -| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | k3d node images + the ingestor image; the tracebloc control-plane images when `images.traceblocRegistry` points there | +| `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox + the tracebloc training images; the control-plane images too, only if you roll them back with `images.traceblocRegistry=docker.io` | +| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | the tracebloc control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy — the default since the GHCR migration) + the ingestor image + k3d node images | | `api.tracebloc.io` (`dev-api`/`stg-api` for non-prod) | client credential check + the running client's platform connection | | `tracebloc.github.io` | the tracebloc Helm chart repository | @@ -55,7 +55,7 @@ Some sites hard-block Docker Hub / GHCR outright — the images aren't reachable The chart follows the **`global.imageRegistry`** convention: set it once and **every** image the chart pulls — the tracebloc services, the spawned ingestor, the training-job images, and the `alpine/*`, `ubuntu/squid`, `busybox`, `curl` helper images — is re-homed onto your registry. No per-image overrides. -**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. To point just the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) at a different registry — they are published to both `docker.io` and `ghcr.io` at the same digests — set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset. `global.imageRegistry` always wins when both are set. The image-refresh CronJob follows the same value and can resolve digests anonymously on `docker.io` and `ghcr.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the training-image host yet. +**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. The tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) pull from `ghcr.io` by default and are also published to `docker.io` at the same digests. To point just those images at a different registry, set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset; `global.imageRegistry` always wins when both are set. Rolling the control plane back to Docker Hub is one flag — `--set images.traceblocRegistry=docker.io` — and, being user-supplied, it persists across the fleet auto-upgrade until you clear it. The image-refresh CronJob follows the same value and can resolve digests anonymously on `ghcr.io` and `docker.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the training-image host yet. **1. A private/mirror registry your site *can* reach.** diff --git a/scripts/tests/image-refresh-repin-on-revert.bats b/scripts/tests/image-refresh-repin-on-revert.bats index 9ebfb874..e838ef2a 100644 --- a/scripts/tests/image-refresh-repin-on-revert.bats +++ b/scripts/tests/image-refresh-repin-on-revert.bats @@ -70,10 +70,18 @@ teardown() { rm -rf "$TMP"; } # Runs the shipped re-pin branch with the registry HEAD (already known: recorded # == latest) and the two LIVE-image reads stubbed. -# $1 = STUB_API what workload_image_for_repo returns ("" = unreadable) -# $2 = STUB_PROXY what requests_proxy_image returns ("" = unreadable) -# $3 = RP_PINNED "1" opts the requests-proxy out of following the digest -# $4 = PENDING the ATTEMPT_KEY value carried in (0 = no unfinished re-image) +# $1 = STUB_API what workload_image_for_repo returns ("" = unreadable) +# $2 = STUB_PROXY what requests_proxy_image returns ("" = unreadable) +# $3 = RP_PINNED "1" opts the requests-proxy out of following the digest +# $4 = PENDING the ATTEMPT_KEY value carried in (0 = no unfinished re-image) +# $5 = STUB_APPLIED get_annotation value for ${applied_key} +# (non-empty = a digest was rolled here before) +# $6 = STUB_FIRST_OBSERVED get_annotation value for ${first_observed_key} +# (non-empty = we recorded this workload's first, +# annotation-less tick here) +# The off-digest arm skips ONLY on first_observed present AND applied absent +# (fresh install). applied present rolls (established edge); NEITHER marker rolls +# (pre-marker/legacy edge -- not stranded on the upgrade hop). #1008. # # The branch is wrapped in a ONE-ITERATION loop so its `continue` statements run # as they ship, rather than being stripped (which would change control flow). @@ -82,6 +90,8 @@ run_branch() { set -eu repo="tracebloc/jobs-manager" key="tracebloc.io/last-refreshed-jobs-manager-digest" +applied_key="tracebloc.io/digest-applied-jobs-manager" +first_observed_key="tracebloc.io/first-observed-jobs-manager" IMAGE_REGISTRY="docker.io" IMAGE_TAG="dev" REQUESTS_PROXY_DEPLOYMENT="t-requests-proxy" @@ -91,6 +101,8 @@ recorded="sha256:aaa" STUB_API="\${1:-}" STUB_PROXY="\${2:-}" pending_attempt="\${4:-0}" +STUB_APPLIED="\${5:-}" +STUB_FIRST_OBSERVED="\${6:-}" MAX_REFRESH_ATTEMPTS=3 restart_needed=0 annotate_args="" @@ -105,6 +117,17 @@ log() { printf '%s\n' "\$*"; } kubectl() { printf 'KUBECTL:%s\n' "\$*"; } workload_image_for_repo() { [ -n "\$STUB_API" ] && printf '%s' "\$STUB_API"; } requests_proxy_image() { [ -n "\$STUB_PROXY" ] && printf '%s' "\$STUB_PROXY"; } +# Only the two markers are read inside this branch. Model the real get_annotation: +# return 0 with the value (empty = annotation ABSENT), never non-zero -- a +# non-zero return means a kubectl/jq READ ERROR, which the branch handles +# separately. Using \`[ -n ] && printf\` here would return non-zero on an empty +# stub and be misread as a read error. +get_annotation() { + case "\$1" in + "\$applied_key") printf '%s' "\$STUB_APPLIED" ;; + "\$first_observed_key") printf '%s' "\$STUB_FIRST_OBSERVED" ;; + esac +} for _once in 1; do $(sed 's/^/ /' "$TMP/branch.sh") done @@ -113,15 +136,52 @@ printf 'JM:%s\n' "\$jm_set_args" printf 'RP:%s\n' "\$rp_set_args" printf 'ANNOTATE:%s\n' "\$annotate_args" EOF - sh "$TMP/harness.sh" "${1:-}" "${2:-}" "${3:-}" "${4:-0}" + sh "$TMP/harness.sh" "${1:-}" "${2:-}" "${3:-}" "${4:-0}" "${5:-}" "${6:-}" } -@test "workload reverted to :tag re-pins the digest (restart_needed=1)" { - run run_branch "docker.io/tracebloc/jobs-manager:dev" "" "1" +@test "ESTABLISHED edge reverted to :tag re-pins the digest (restart_needed=1)" { + # A digest was applied here before (applied marker present, $5="1"), then a helm + # re-render reverted the workload onto :tag -- the client-runtime#199 repair + # must roll. #1008. + run run_branch "docker.io/tracebloc/jobs-manager:dev" "" "1" "0" "1" [ "$status" -eq 0 ] || return 1 [[ "$output" == *"RESTART:1"* ]] || return 1 [[ "$output" == *"api=docker.io/tracebloc/jobs-manager@sha256:aaa"* ]] || return 1 [[ "$output" == *"re-pinning the digest"* ]] || return 1 + # the re-image records the digest AND stamps the applied marker, in one annotate + [[ "$output" == *"tracebloc.io/digest-applied-jobs-manager=1"* ]] || return 1 +} + +@test "FRESH install (first-observed here, never applied) does NOT roll -- stays on :tag" { + # #1008. recorded == latest, workload on :tag, first_observed set ($6="1") and + # applied absent ($5="") -- a workload we watched born on :tag here. Rolling + # would pay the full #563 flap-path / Recreate cost for byte-identical content + # the install already pulled. The tick must leave it on :tag. + run run_branch "docker.io/tracebloc/jobs-manager:dev" "" "1" "0" "" "1" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"fresh install"* ]] || return 1 + [[ "$output" == *"NOT rolling"* ]] || return 1 + [[ "$output" == *"RESTART:0"* ]] || return 1 + # nothing queued for a rollout, and no digest/marker write this tick + [[ "$output" != *"JM:api="* ]] || return 1 + [[ "$output" != *"re-pinning the digest"* ]] || return 1 + [[ "$output" != *"digest-applied-jobs-manager=1"* ]] || return 1 +} + +@test "PRE-MARKER / legacy edge (NEITHER marker) still gets the repair roll" { + # LukasWodka on #1008: an edge pinned by a version predating these markers has + # neither ($5="" $6=""). The upgrade shipping this chart reverts it to :tag, so + # its first post-upgrade tick is byte-for-byte the fresh shape. Skipping on + # absence alone would strand the whole existing fleet on a possibly-stale :tag + # until the next upstream digest change -- the exact #199 exposure. A legacy + # edge must therefore ROLL (repair), which then stamps the applied marker. + run run_branch "docker.io/tracebloc/jobs-manager:dev" "" "1" "0" "" "" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"RESTART:1"* ]] || return 1 + [[ "$output" == *"predates the fresh-install markers"* ]] || return 1 + [[ "$output" == *"api=docker.io/tracebloc/jobs-manager@sha256:aaa"* ]] || return 1 + [[ "$output" == *"tracebloc.io/digest-applied-jobs-manager=1"* ]] || return 1 + [[ "$output" != *"fresh install"* ]] || return 1 } @test "api and proxy both on the digest is a no-op (restart_needed=0, no set args)" {