release-train: develop -> staging

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.109
-appVersion: "1.9.109"
+version: 1.9.110
+appVersion: "1.9.110"
 keywords:
   - tracebloc
   - kubernetes

client/tests/image_refresh_test.yaml

@@ -223,6 +223,21 @@ tests:
       - matchRegex:
           path: data["image-refresh.sh"]
           pattern: "pinned by digest in values"
+      # Regression guard (Bugbot #1008): the requests-proxy is a SEPARATE
+      # deployment running the SAME jobs-manager image, so the no-op "already on
+      # the pinned digest" decision MUST also read the proxy and fall through
+      # when it is off the digest -- else a partial re-pin (api pinned, proxy
+      # still on :tag) is declared unchanged forever off the api match alone and
+      # never retried. Lock the reader and the per-workload digest check in
+      # place. (The BEHAVIOUR -- that an inverted check reddens -- is asserted in
+      # scripts/tests/image-refresh-repin-on-revert.bats; this only pins that the
+      # two pieces still exist in the shipped script.)
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'requests_proxy_image\(\)'
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'proxy_on_digest'
       # Regression guard: the script must HEAD the manifest with all
       # four Accept media types in a SINGLE comma-separated Accept
       # header per the Docker registry v2 spec (some proxies have been
@@ -935,3 +950,23 @@ tests:
       - matchRegex:
           path: data["image-refresh.sh"]
           pattern: 'rm_set_args tracebloc-resource-monitor='
+
+  - it: reconcile re-pins when a helm re-render reverted the workload off the digest
+    # Guards the client-runtime#199 fix: with `recorded == latest` the loop must
+    # NOT unconditionally no-op -- it must read the live workload image and
+    # re-pin when it is not `repo@latest` (a `helm upgrade --reset-then-reuse-values`
+    # reverted the pin onto the bare :tag, where a stale node cache serves an old
+    # image). The helper + the fall-through into the re-image path are the fix.
+    template: templates/image-refresh-cronjob.yaml
+    documentIndex: 0
+    asserts:
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'workload_image_for_repo\(\)'
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'have="\$\(workload_image_for_repo "\$repo"'
+      # the true no-op now requires BOTH digest-unchanged AND workload-on-digest
+      - matchRegex:
+          path: data["image-refresh.sh"]
+          pattern: 'workload already on the pinned digest; no-op'

client/values.yaml

@@ -1718,11 +1718,16 @@ autoUpgrade:
 #   - First observation (annotation absent on a fresh install): record
 #     the current digest without re-imaging. Rewriting repo:tag to
 #     repo@digest for byte-identical content would roll every workload —
-#     including the DaemonSet on every node — for nothing. The cost is
-#     that a fresh edge runs repo:tag until the first real digest change:
-#     still restart-safe offline, just not yet reproducible.
-#   - Idle-cheap: when the recorded digest matches today's digest, the
-#     script exits without touching anything. Steady state is one HEAD
+#     including the DaemonSet on every node — for nothing. The NEXT tick,
+#     seeing the digest recorded but the workload still on repo:tag, pins
+#     it (client-runtime#199) — so a fresh edge becomes reproducible ~one interval
+#     post-install, not at the next upstream release. Restart-safe offline
+#     throughout.
+#   - Idle-cheap: when the recorded digest matches today's digest AND the
+#     workload already runs that digest, the script exits without touching
+#     anything. If a helm re-render reverted the pin back to repo:tag it
+#     re-pins that one tick (compared on the @sha256 digest, so a mirror
+#     prefix rewrite is not mistaken for a revert). Steady state is one HEAD
 #     per image per tick, well under Docker Hub's 100/6h anonymous
 #     pull-rate limit.
 #   - Private mirrors (global.imageRegistry): the script resolves digests

scripts/lib/cluster.sh

@@ -1188,13 +1188,26 @@ create_cluster() {
 ensure_cluster_autostart() {
   if [[ -n "${TRACEBLOC_NO_AUTOSTART:-}" ]]; then return 0; fi
 
-  local nodes node
+  local nodes node _nodes_rc=0
   # BOUNDED (client#984, LukasWodka): this is a daemon read on the main install
   # path, and it ran unbounded while its `docker info` neighbours did not — the gap
-  # check-style rule 5 could not see until it was widened past `info`. `|| return 0`
-  # already treats an unreadable engine as "nothing to autostart", so a 124 lands in
-  # the branch this function was written for.
-  nodes=$(_bounded "${TB_DOCKER_PROBE_TIMEOUT:-10}" docker ps -a --filter "name=k3d-${CLUSTER_NAME}-" --format '{{.Names}}' 2>/dev/null) || return 0
+  # check-style rule 5 could not see until it was widened past `info`.
+  #
+  # DO NOT `|| return 0` here (Bugbot Medium, off the client#1011 promotion
+  # review): this read feeds ONLY the node restart-policy loop below, but the Linux
+  # docker.service boot-enable further down does NOT depend on the node list.
+  # Bailing out of the whole function on a 124 left the operator a finished
+  # install whose docker.service was never enabled on boot — the cluster would
+  # not come back after a reboot, with no warning. A failed/timed-out read means
+  # "we couldn't enumerate nodes", so skip the loop (k3d already sets
+  # --restart unless-stopped at create time, so the policy still holds — the same
+  # rationale the Windows twin Set-ClusterAutostart states) and fall through to
+  # the boot-enable step.
+  nodes=$(_bounded "${TB_DOCKER_PROBE_TIMEOUT:-10}" docker ps -a --filter "name=k3d-${CLUSTER_NAME}-" --format '{{.Names}}' 2>/dev/null) || _nodes_rc=$?
+  if [[ "$_nodes_rc" -ne 0 ]]; then
+    nodes=""
+    log "Could not read k3d nodes for the restart policy (docker ps exit ${_nodes_rc}); leaving k3d's own --restart policy in place and continuing to the boot-enable step."
+  fi
   if [[ -n "$nodes" ]]; then
     for node in $nodes; do
       docker update --restart unless-stopped "$node" >/dev/null 2>&1 || true

scripts/manifest.sha256

@@ -7,7 +7,7 @@ a61f5bac3786a3283b5fa08fea9e522f7cea5f7fa44799a2b38b5caea42469d8  scripts/lib/de
 b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c  scripts/lib/gpu-amd.sh
 95209eeca22917db32e3f352af3f394773978709c73f474d1694c0a42dedc7de  scripts/lib/setup-macos.sh
 d9a372308bf53b25fb39b404bd78cf044563425e3eda750f04583d517114a8ea  scripts/lib/setup-linux.sh
-e1737a4a7d76bb871e07937b1149ea187baf7185977fde4011cd585ed2c7c1b2  scripts/lib/cluster.sh
+669274f425058421ba6346d672d5ee339ba3faac33197dd6af3cc15d9f17b54c  scripts/lib/cluster.sh
 84ed9d9b3ab4633bfaf07b256c066ed43f96a0025ec6b1a34db23fdef75f0f62  scripts/lib/gpu-plugins.sh
 320a3d04d7127849c92d372d5c7942f86f19256a4ea8e2f0afe48b3a5149c53a  scripts/lib/install-client-helm.sh
 1b3e11d06e4be983ec5cecd8f55b16034b76bdb0476f3e141c025d383ddc043a  scripts/lib/install-cli.sh

scripts/tests/bounded-reads-propagate.bats

@@ -102,7 +102,10 @@ _bounded_capture_read
   # Why each is out of scope rather than wrong: every one is a best-effort read
   # whose timeout branch is already a `|| return 0` / `|| true` no-op on a path
   # that reconciles anyway (the _check_existing_cluster_* drift probes,
-  # _generate_node_cdi_specs, ensure_cluster_autostart), a yes/no liveness probe
+  # _generate_node_cdi_specs), or falls through logging the skip without ever
+  # claiming a machine state it could not read (ensure_cluster_autostart, whose
+  # timed-out node read now skips only the restart-policy loop and still runs the
+  # boot-enable), a yes/no liveness probe
   # that is already tri-state or has no third state to lose (_docker_answers,
   # _k3d_cluster_running, _assess_runtime_down, _docker_default_runtime_is_nvidia),
   # or a preflight/install step that reports its own failure to the operator

scripts/tests/cluster.bats

@@ -932,6 +932,26 @@ _cc_mocks() {   # $1 = "real-handle" to leave _handle_existing_cluster UNstubbed
   [[ "$output" != *"docker update"* ]] || return 1
 }
 
+# A TIMED-OUT nodes read must skip only the node restart-policy loop, NOT abandon
+# the Linux docker.service boot-enable below it — that step does not depend on the
+# node list, and bailing out left a finished install whose cluster never came back
+# after a reboot, with no warning (Bugbot Medium, off the client#1011 promotion review).
+@test "ensure_cluster_autostart: nodes read times out -> still enables docker.service, skips node loop, logs" {
+  OS=Linux
+  LOG_FILE="$BATS_TEST_TMPDIR/autostart.log"
+  _bounded() { return 124; }                         # docker ps -a (nodes read) times out
+  docker() { record "docker $*"; }                   # any docker update would be recorded
+  sudo()   { record "sudo $*"; }
+  systemctl() { record "systemctl $*"; return 1; }   # not already enabled on boot
+  has()    { return 0; }
+  run ensure_cluster_autostart
+  [ "$status" -eq 0 ] || return 1
+  run mock_calls
+  [[ "$output" != *"docker update"* ]] || return 1                 # node loop skipped on the timeout
+  [[ "$output" == *"sudo systemctl enable docker"* ]] || return 1  # INDEPENDENT boot-enable still ran
+  grep -q "leaving k3d's own --restart policy in place" "$LOG_FILE" || return 1   # logged to LOG_FILE, not silent
+}
+
 # ── bounded create (#426) ────────────────────────────────────────────────────
 @test "k3d create is bounded: --wait always pairs with --timeout (#426)" {
   grep -q -- '--wait --timeout' "$BATS_TEST_DIRNAME/../lib/cluster.sh"

scripts/tests/customer-copy-no-ticket-refs.bats

@@ -348,3 +348,140 @@ FX
   [ "$status" -eq 2 ] || { echo "$output"; return 1; }
   [[ "$output" == *"derived ZERO copy-emitting bash helpers"* ]] || { echo "$output"; return 1; }
 }
+
+# --- second review round (client#1020): escapes, here-document text, one lexer ---
+
+@test "derivation: an escaped quote around an unbalanced brace inside a string does not move the depth (bash)" {
+  # Old walker: `\"` toggled quote state, so the `{` counted as a real brace and the
+  # first helper never balanced (guard error), while a `}` closed the second early.
+  printf 'say_escaped() {\n  echo "open \\"{ deeper\\" now"\n}\nsay_escaped_close() {\n  echo "close \\"} early\\" now"\n  echo "$*"\n}\nafter_escaped() { echo "$*"; }\n' >> "$WORK/scripts/lib/cluster.sh"
+  grep -qF 'echo "open \"{ deeper\" now"' "$WORK/scripts/lib/cluster.sh" || return 1   # anchor applied
+  plant scripts/lib/cluster.sh 'after_escaped "planted after an escaped quote (backend#15)"'
+  run run_guard "$WORK" --print-vocab bash
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  for fn in say_escaped say_escaped_close after_escaped; do
+    grep -qx "$fn" <<<"$output" || { echo "missing $fn"; echo "$output"; return 1; }
+  done
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"planted after an escaped quote (backend#15)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "derivation: a backtick-escaped or doubled quote around a brace does not move the depth (PowerShell)" {
+  printf 'function Say-Escaped {\n  Write-Host "he said `"go { deeper`" now"\n}\nfunction Say-Doubled {\n  Write-Host "he said ""go } early"" now"\n}\nfunction After-Escaped($m) { Write-Host $m }\n' >> "$WORK/scripts/install-k8s.ps1"
+  grep -qF 'Write-Host "he said `"go { deeper`" now"' "$WORK/scripts/install-k8s.ps1" || return 1   # anchor applied
+  plant scripts/install-k8s.ps1 'After-Escaped "planted after an escaped quote (RFC-9908)"'
+  run run_guard "$WORK" --print-vocab ps
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  for fn in Say-Escaped Say-Doubled After-Escaped; do
+    grep -qx "$fn" <<<"$output" || { echo "missing $fn"; echo "$output"; return 1; }
+  done
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"planted after an escaped quote (RFC-9908)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "mutation: a # inside a PRINTED here-document body is text the customer reads, not a comment" {
+  printf "help_hash() {\n  cat <<'HELP'\n  # migration required, see backend#16\n  Some line   # tracked in backend#17\nHELP\n}\n" >> "$WORK/scripts/lib/cluster.sh"
+  grep -q 'tracked in backend#17' "$WORK/scripts/lib/cluster.sh" || return 1   # anchor applied
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"migration required, see backend#16"* ]] || { echo "$output"; return 1; }
+  [[ "$output" == *"tracked in backend#17"* ]] || { echo "$output"; return 1; }
+  [[ "$output" == *"2 user-visible line(s)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "a # inside a here-document that GENERATES A FILE is that file's comment; its other lines are still copy" {
+  # The values.yaml the installer writes carries the rationale for its defaults
+  # as YAML comments -- out of scope like every other comment.
+  printf 'write_values() {\n  cat <<EOF > "$1"\n# rationale for this default (backend#19)\nreplicas: 1   # see RFC-9909\nEOF\n}\n' >> "$WORK/scripts/lib/cluster.sh"
+  grep -q 'rationale for this default (backend#19)' "$WORK/scripts/lib/cluster.sh" || return 1   # anchor applied
+  run run_guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  # ...but a non-comment line of the generated file is text the customer can open.
+  printf 'write_values_token() {\n  cat <<EOF > "$1"\nnote: planted in a generated file (backend#20)\nEOF\n}\n' >> "$WORK/scripts/lib/cluster.sh"
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"planted in a generated file (backend#20)"* ]] || { echo "$output"; return 1; }
+  [[ "$output" != *"backend#19"* ]] || { echo "$output"; return 1; }
+}
+
+@test "a # inside an ASSIGNED PowerShell here-string is that file's comment; its other lines are still copy" {
+  printf 'function Write-Values {\n  $values = @"\n# rationale for this default (backend#21)\nreplicas: 1\n"@\n  Set-Content -Path $p -Value $values\n}\n' >> "$WORK/scripts/install-k8s.ps1"
+  grep -q 'rationale for this default (backend#21)' "$WORK/scripts/install-k8s.ps1" || return 1   # anchor applied
+  run run_guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  printf 'function Write-Values-Token {\n  $values = @"\nnote: planted in an assigned here-string (backend#23)\n"@\n}\n' >> "$WORK/scripts/install-k8s.ps1"
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"planted in an assigned here-string (backend#23)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "mutation: the org's RFC-<AREA>-<nnn> form with three digits is caught" {
+  plant scripts/lib/cluster.sh 'warn "planted short rfc (RFC-BACKEND-664)"'
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"RFC-BACKEND-664"* ]] || { echo "$output"; return 1; }
+}
+
+@test "a here-document body line that starts with an emitter word is reported once, not twice" {
+  printf "help_once() {\n  cat <<'HELP'\n  echo is what this prints, see backend#18\nHELP\n}\n" >> "$WORK/scripts/lib/cluster.sh"
+  grep -q 'echo is what this prints, see backend#18' "$WORK/scripts/lib/cluster.sh" || return 1   # anchor applied
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [ "$(grep -c 'backend#18' <<<"$output")" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"1 user-visible line(s)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "derivation: a PowerShell here-string closed by \"@.Trim() does not swallow every later function" {
+  # The old closer rule wanted the closer ALONE on its line, so `"@.Trim()` in
+  # install-k8s.ps1 left the here-string open for ~2,750 lines and hid eleven
+  # emitting helpers from the vocabulary (a silent miss).
+  printf 'function Get-Script {\n  return @"\necho hi\n"@.Trim()\n}\nfunction After-Trim($m) { Write-Host $m }\n' >> "$WORK/scripts/install-k8s.ps1"
+  grep -qF '"@.Trim()' "$WORK/scripts/install-k8s.ps1" || return 1   # anchor applied
+  plant scripts/install-k8s.ps1 'After-Trim "planted after a trimmed here-string (RFC-9910)"'
+  run run_guard "$WORK" --print-vocab ps
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  grep -qx After-Trim <<<"$output" || { echo "$output"; return 1; }
+  ! grep -qx Get-Script <<<"$output" || { echo "Get-Script emits nothing"; echo "$output"; return 1; }
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"planted after a trimmed here-string (RFC-9910)"* ]] || { echo "$output"; return 1; }
+}
+
+@test "one lexer: the quote/comment walk and every here-document rule (open AND close) are defined once" {
+  # The census names each shared rule, opener and closer alike: the closer is the
+  # half that was wrong before (a second, anchored closer rule left ~2,750 lines
+  # of install-k8s.ps1 as here-string body), so a second copy of it must redden
+  # this test too (Bugbot on client#1022).
+  for fn in lex code_only heredoc_delim herestring_closer closes; do
+    [ "$(grep -c "function $fn(" "$GUARD")" -eq 1 ] || { echo "$fn defined $(grep -c "function $fn(" "$GUARD") times"; return 1; }
+  done
+  [ "$(grep -c 'sub(/\.\*<<-?' "$GUARD")" -eq 1 ] || return 1
+  # ...and the closer TEST is spelled exactly once, inside closes(): a state
+  # machine re-spelling `"^[ \t]*" closer` inline is a second closer rule.
+  [ "$(grep -cE '"\^\[ \\t\]\*" (heredoc|closer)' "$GUARD")" -eq 1 ] || { echo "an inline closer regex exists outside closes()"; return 1; }
+}
+
+# --- Bugbot round on the follow-up: redirect classification, scratch dir ------
+
+@test "a printed here-document with a stderr or /dev redirect is still text, not a generated file" {
+  printf "help_quiet() {\n  cat <<'HELP' 2>/dev/null\n  # see backend#24 before upgrading\nHELP\n}\nhelp_err() {\n  cat <<'HELP' >/dev/stderr\n  # see backend#25 before upgrading\nHELP\n}\nhelp_fd() {\n  cat <<'HELP' >&2\n  # see backend#26 before upgrading\nHELP\n}\n" >> "$WORK/scripts/lib/cluster.sh"
+  grep -q 'see backend#26 before upgrading' "$WORK/scripts/lib/cluster.sh" || return 1   # anchor applied
+  run run_guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  for t in 'backend#24' 'backend#25' 'backend#26'; do
+    [[ "$output" == *"$t before upgrading"* ]] || { echo "missing $t"; echo "$output"; return 1; }
+  done
+  [[ "$output" == *"3 user-visible line(s)"* ]] || { echo "$output"; return 1; }
+  # ...while an explicit stdout-to-file redirect (`1>`) is a generated file.
+  printf 'write_one() {\n  cat <<EOF 1> "$1"\n# rationale (backend#27)\nEOF\n}\n' >> "$WORK/scripts/lib/cluster.sh"
+  run run_guard
+  [[ "$output" != *"backend#27"* ]] || { echo "$output"; return 1; }
+}
+
+@test "fail closed: a scratch directory that cannot be created is a guard error, never a cleanup of /" {
+  TMPDIR="$WORK/does-not-exist" run run_guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"could not create a scratch directory"* ]] || { echo "$output"; return 1; }
+}