From da0811bccbc6ed13279ad4563f451c51f912920e Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 12:26:55 -0700 Subject: [PATCH 1/7] fix(recipes): raise K8s floors to clear the DRA chart's kubeVersion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every recipe inherits nvidia-dra-driver-gpu from base.yaml, and the pinned chart declares kubeVersion: ">=1.32.0-0". Helm refuses the install below that, so 29 overlays declaring a lower K8s.server.version admitted clusters that pass every recipe-time check and then fail at `helm install`. recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason; its comment recorded the diagnosis but the rest of the catalog was never reconciled. Every declaration is raised, not just base.yaml: constraints merge by name with the later overlay winning and no max comparison, so a leaf declaring ">= 1.30" silently overwrites a higher floor inherited from base. That is visible in the golden churn — the 16 leaves that moved are those inheriting a raised floor, while leaves declaring their own >= 1.34 or >= 1.32.4 were already clear and are unchanged. Adds a guard asserting no overlay or mixin declares a floor below the chart's, so the reconciliation cannot drift back. Control verified: reverting one leaf to 1.31 fails the guard. It also fails closed when no floors match, so it cannot go vacuous. Signed-off-by: Yuan Chen --- pkg/recipe/dra_k8s_floor_test.go | 114 ++++++++++++++++++ recipes/overlays/a100-eks-training.yaml | 2 +- .../a100-eks-ubuntu-training-kubeflow.yaml | 2 +- .../overlays/a100-eks-ubuntu-training.yaml | 2 +- .../a100-gke-cos-training-kubeflow.yaml | 2 +- recipes/overlays/a100-gke-cos-training.yaml | 2 +- recipes/overlays/a100-oke-training.yaml | 2 +- .../a100-oke-ubuntu-training-kubeflow.yaml | 2 +- .../overlays/a100-oke-ubuntu-training.yaml | 2 +- recipes/overlays/base.yaml | 2 +- recipes/overlays/eks-inference.yaml | 2 +- recipes/overlays/eks-training.yaml | 2 +- recipes/overlays/eks.yaml | 2 +- recipes/overlays/gke-cos-inference.yaml | 2 +- recipes/overlays/gke-cos-training.yaml | 2 +- recipes/overlays/gke-cos.yaml | 2 +- recipes/overlays/kind-inference.yaml | 2 +- recipes/overlays/kind.yaml | 2 +- recipes/overlays/l40s-oke-inference.yaml | 2 +- recipes/overlays/l40s-oke-training.yaml | 2 +- recipes/overlays/lke-inference.yaml | 2 +- recipes/overlays/lke-training.yaml | 2 +- recipes/overlays/lke.yaml | 2 +- recipes/overlays/oke-ol-inference.yaml | 2 +- recipes/overlays/oke-ol-training.yaml | 2 +- recipes/overlays/oke-ol.yaml | 2 +- .../overlays/rtx-pro-6000-lke-inference.yaml | 2 +- .../overlays/rtx-pro-6000-lke-training.yaml | 2 +- .../rtx-pro-6000-lke-ubuntu-inference.yaml | 2 +- .../rtx-pro-6000-lke-ubuntu-training.yaml | 2 +- 30 files changed, 143 insertions(+), 29 deletions(-) create mode 100644 pkg/recipe/dra_k8s_floor_test.go diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go new file mode 100644 index 000000000..195f0c1d7 --- /dev/null +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strconv" + "strings" + "testing" +) + +// draChartKubeVersionMinor is the minor version floor the pinned +// nvidia-dra-driver-gpu chart declares: +// +// kubeVersion: ">=1.32.0-0" +// +// Helm REJECTS the install outright when the cluster is below it, so a recipe +// declaring a lower K8s.server.version admits clusters that pass every +// recipe-time check and then fail at `helm install`. +// +// Sourced from the chart, not from a running cluster. If the DRA chart pin in +// recipes/registry.yaml moves to a version with a different kubeVersion, this +// constant and the affected overlays must move together — that coupling is the +// point of the guard. +const draChartKubeVersionMinor = 32 + +// k8sFloorRE captures the minor version from a K8s.server.version constraint +// expressed as a floor. Only `>=` forms are checked: an exact pin or a range is +// a deliberate statement that should be reviewed on its own terms. +var k8sFloorRE = regexp.MustCompile(`(?s)- name: K8s\.server\.version\s*\n\s*value: "\s*>=\s*1\.(\d+)`) + +// TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a +// Kubernetes floor below the DRA chart's own kubeVersion. +// +// Every recipe inherits nvidia-dra-driver-gpu from base.yaml — no overlay +// removes or disables it — so the chart's floor applies catalog-wide. +// +// Why every declaration and not just base.yaml: constraints merge by name with +// the LATER overlay winning and no max comparison (see mergeValidation in +// validation.go). A leaf declaring ">= 1.30" silently overwrites a higher floor +// inherited from base, so raising base alone would not hold. This is the same +// last-wins hazard documented for driver floors in #2438. +// +// recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason +// before the rest of the catalog was reconciled; its comment records the +// diagnosis. +func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + if !strings.Contains(path, "overlays/") && !strings.Contains(path, "mixins/") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + m := k8sFloorRE.FindStringSubmatch(string(raw)) + if m == nil { + return nil + } + checked++ + minor, convErr := strconv.Atoi(m[1]) + if convErr != nil { + t.Errorf("%s: could not parse K8s.server.version minor %q: %v", path, m[1], convErr) + return nil + } + if minor < draChartKubeVersionMinor { + t.Errorf("%s declares K8s.server.version \">= 1.%d\", below the pinned "+ + "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + path, minor, draChartKubeVersionMinor, draChartKubeVersionMinor) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if nothing matched, the guard is inert. + if checked == 0 { + t.Fatal("no K8s.server.version floors found in overlays or mixins — this guard " + + "is vacuous. Either the constraint name changed or the embed pattern no " + + "longer covers the recipe tree.") + } + t.Logf("verified %d K8s.server.version floor(s) clear the DRA chart floor", checked) +} diff --git a/recipes/overlays/a100-eks-training.yaml b/recipes/overlays/a100-eks-training.yaml index c2e2c1159..f9fc790c7 100644 --- a/recipes/overlays/a100-eks-training.yaml +++ b/recipes/overlays/a100-eks-training.yaml @@ -32,7 +32,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from eks-training). diff --git a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml index 95cdd1eed..368c2afa9 100644 --- a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-eks-ubuntu-training.yaml b/recipes/overlays/a100-eks-ubuntu-training.yaml index d9a461d80..c6625021e 100644 --- a/recipes/overlays/a100-eks-ubuntu-training.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml index 6eb940a08..0df5add20 100644 --- a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml +++ b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml @@ -32,7 +32,7 @@ spec: # Constraints for A100 on GKE COS for Kubeflow training workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" # Kubeflow Training Operator for TrainJob support. # Declared inline (not via the platform-kubeflow mixin) to match the GKE COS diff --git a/recipes/overlays/a100-gke-cos-training.yaml b/recipes/overlays/a100-gke-cos-training.yaml index 0f8f716a4..ae92d247d 100644 --- a/recipes/overlays/a100-gke-cos-training.yaml +++ b/recipes/overlays/a100-gke-cos-training.yaml @@ -32,7 +32,7 @@ spec: # the GKE COS training baseline rather than the H100 1.32 floor. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from gke-cos-training). diff --git a/recipes/overlays/a100-oke-training.yaml b/recipes/overlays/a100-oke-training.yaml index e929c2ce5..4b6a8ddab 100644 --- a/recipes/overlays/a100-oke-training.yaml +++ b/recipes/overlays/a100-oke-training.yaml @@ -34,7 +34,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml index ca77b4cda..d7ed0c31c 100644 --- a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-oke-ubuntu-training.yaml b/recipes/overlays/a100-oke-ubuntu-training.yaml index dcb5f5867..bb298d2da 100644 --- a/recipes/overlays/a100-oke-ubuntu-training.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/base.yaml b/recipes/overlays/base.yaml index 473b2629a..67fc3dbb8 100644 --- a/recipes/overlays/base.yaml +++ b/recipes/overlays/base.yaml @@ -22,7 +22,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" componentRefs: - name: nfd diff --git a/recipes/overlays/eks-inference.yaml b/recipes/overlays/eks-inference.yaml index b5fb62a03..c85e8e890 100644 --- a/recipes/overlays/eks-inference.yaml +++ b/recipes/overlays/eks-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/eks-training.yaml b/recipes/overlays/eks-training.yaml index fe436bcf7..d7014a611 100644 --- a/recipes/overlays/eks-training.yaml +++ b/recipes/overlays/eks-training.yaml @@ -29,7 +29,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the training-optimized GPU Operator values diff --git a/recipes/overlays/eks.yaml b/recipes/overlays/eks.yaml index 3b9459d06..195186014 100644 --- a/recipes/overlays/eks.yaml +++ b/recipes/overlays/eks.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # EKS-specific components componentRefs: diff --git a/recipes/overlays/gke-cos-inference.yaml b/recipes/overlays/gke-cos-inference.yaml index d659c931f..1240b9304 100644 --- a/recipes/overlays/gke-cos-inference.yaml +++ b/recipes/overlays/gke-cos-inference.yaml @@ -33,6 +33,6 @@ spec: # Inference specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/gke-cos-training.yaml b/recipes/overlays/gke-cos-training.yaml index 3cea0c4bd..e0e1fb19d 100644 --- a/recipes/overlays/gke-cos-training.yaml +++ b/recipes/overlays/gke-cos-training.yaml @@ -29,7 +29,7 @@ spec: # Training specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the GKE-COS training-optimized GPU Operator values diff --git a/recipes/overlays/gke-cos.yaml b/recipes/overlays/gke-cos.yaml index 29e42499c..2dab8f638 100644 --- a/recipes/overlays/gke-cos.yaml +++ b/recipes/overlays/gke-cos.yaml @@ -28,7 +28,7 @@ spec: # GKE-specific constraints constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # The GPU stack shape is a configuration profile (ADR-015, issue # #1761): one qualified GKE cluster state per value — how the driver is diff --git a/recipes/overlays/kind-inference.yaml b/recipes/overlays/kind-inference.yaml index 0ebbf84b1..3467d5b17 100644 --- a/recipes/overlays/kind-inference.yaml +++ b/recipes/overlays/kind-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/kind.yaml b/recipes/overlays/kind.yaml index e8a20d5a7..08afd5c5b 100644 --- a/recipes/overlays/kind.yaml +++ b/recipes/overlays/kind.yaml @@ -27,7 +27,7 @@ spec: # Kind-specific constraints constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" # Kind-specific component overrides componentRefs: diff --git a/recipes/overlays/l40s-oke-inference.yaml b/recipes/overlays/l40s-oke-inference.yaml index dc9830e97..710394771 100644 --- a/recipes/overlays/l40s-oke-inference.yaml +++ b/recipes/overlays/l40s-oke-inference.yaml @@ -29,7 +29,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/l40s-oke-training.yaml b/recipes/overlays/l40s-oke-training.yaml index 6314819d2..957e43b5a 100644 --- a/recipes/overlays/l40s-oke-training.yaml +++ b/recipes/overlays/l40s-oke-training.yaml @@ -32,7 +32,7 @@ spec: # requirement, so the recipe keeps the OKE training baseline K8s floor. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # L40S-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/lke-inference.yaml b/recipes/overlays/lke-inference.yaml index d4b34a107..54d8b0e97 100644 --- a/recipes/overlays/lke-inference.yaml +++ b/recipes/overlays/lke-inference.yaml @@ -32,6 +32,6 @@ spec: # Inference specific constraints for LKE workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke-training.yaml b/recipes/overlays/lke-training.yaml index 7e4812c8c..0e7ee9f45 100644 --- a/recipes/overlays/lke-training.yaml +++ b/recipes/overlays/lke-training.yaml @@ -29,6 +29,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke.yaml b/recipes/overlays/lke.yaml index da811a074..795883ace 100644 --- a/recipes/overlays/lke.yaml +++ b/recipes/overlays/lke.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" # LKE-specific components componentRefs: diff --git a/recipes/overlays/oke-ol-inference.yaml b/recipes/overlays/oke-ol-inference.yaml index 24b212de5..c46c6295c 100644 --- a/recipes/overlays/oke-ol-inference.yaml +++ b/recipes/overlays/oke-ol-inference.yaml @@ -30,6 +30,6 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/oke-ol-training.yaml b/recipes/overlays/oke-ol-training.yaml index 0d26f3dd5..097819f05 100644 --- a/recipes/overlays/oke-ol-training.yaml +++ b/recipes/overlays/oke-ol-training.yaml @@ -27,7 +27,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/oke-ol.yaml b/recipes/overlays/oke-ol.yaml index 57f4687d9..0944a5bdd 100644 --- a/recipes/overlays/oke-ol.yaml +++ b/recipes/overlays/oke-ol.yaml @@ -26,7 +26,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # OKE-specific components componentRefs: diff --git a/recipes/overlays/rtx-pro-6000-lke-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-inference.yaml index bdeb89112..2fba32e06 100644 --- a/recipes/overlays/rtx-pro-6000-lke-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-inference.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-training.yaml b/recipes/overlays/rtx-pro-6000-lke-training.yaml index e397e2b3f..94fb6e0fa 100644 --- a/recipes/overlays/rtx-pro-6000-lke-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-training.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml index 5b8d518a2..f74fbe601 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for inference workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml index d90aeba8a..1554736d2 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for training workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] From 1ad6d05c2c2bc9e162af56ab26420a135a8510a9 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:11:56 -0700 Subject: [PATCH 2/7] test(recipe): check every K8s floor declaration and fail closed on unknown forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard used FindStringSubmatch, so only the first K8s.server.version declaration in a file was checked, and its regex matched only ">= 1.", so an exact pin or a range was skipped entirely. Both would admit clusters below the DRA chart's kubeVersion just as effectively. The original comment rationalised the second hole — "an exact pin or a range is a deliberate statement that should be reviewed on its own terms" — which is the wrong instinct for a guard whose only job is catching a future author deviating from the established shape. Now iterates every declaration and fails closed on any form it cannot interpret, naming the value and asking for either a >= floor or an extension to the guard. Controls verified: an exact pin of "== 1.30" fails as uninterpretable, and a second declaration of ">= 1.29" appended after a valid one fails as below the floor. Neither was caught before. Signed-off-by: Yuan Chen --- pkg/recipe/dra_k8s_floor_test.go | 68 +++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 195f0c1d7..054ad7377 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -37,10 +37,14 @@ import ( // point of the guard. const draChartKubeVersionMinor = 32 -// k8sFloorRE captures the minor version from a K8s.server.version constraint -// expressed as a floor. Only `>=` forms are checked: an exact pin or a range is -// a deliberate statement that should be reviewed on its own terms. -var k8sFloorRE = regexp.MustCompile(`(?s)- name: K8s\.server\.version\s*\n\s*value: "\s*>=\s*1\.(\d+)`) +// k8sConstraintRE captures EVERY K8s.server.version declaration in a file and +// its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry +// more than one declaration, and checking only the first would let a later, +// lower one through. +var k8sConstraintRE = regexp.MustCompile(`- name: K8s\.server\.version\s*\n\s*value: "([^"]*)"`) + +// geFloorRE matches the `>= 1.` form this guard can reason about. +var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a // Kubernetes floor below the DRA chart's own kubeVersion. @@ -77,26 +81,42 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { if readErr != nil { return readErr } - m := k8sFloorRE.FindStringSubmatch(string(raw)) - if m == nil { - return nil - } - checked++ - minor, convErr := strconv.Atoi(m[1]) - if convErr != nil { - t.Errorf("%s: could not parse K8s.server.version minor %q: %v", path, m[1], convErr) - return nil - } - if minor < draChartKubeVersionMinor { - t.Errorf("%s declares K8s.server.version \">= 1.%d\", below the pinned "+ - "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - path, minor, draChartKubeVersionMinor, draChartKubeVersionMinor) + for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { + checked++ + value := m[1] + + g := geFloorRE.FindStringSubmatch(value) + if g == nil { + // Fail closed on any form this guard cannot interpret — an exact + // pin (== 1.30), a range, or a bare version would each be just as + // capable of admitting a sub-floor cluster, and silently skipping + // them would make the guard weakest exactly where a future author + // deviates from the established shape. + t.Errorf("%s declares K8s.server.version %q, a form this guard cannot verify.\n"+ + " It can only reason about \">= 1.\". Any other form may admit clusters\n"+ + " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ + " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ + " this guard to understand the new form. See #2402.", + path, value, draChartKubeVersionMinor) + continue + } + + minor, convErr := strconv.Atoi(g[1]) + if convErr != nil { + t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) + continue + } + if minor < draChartKubeVersionMinor { + t.Errorf("%s declares K8s.server.version %q, below the pinned "+ + "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + path, value, draChartKubeVersionMinor, draChartKubeVersionMinor) + } } return nil }) From 6d9a3ed229db3582c87c3d59ddef4bd1a718b029 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:16:55 -0700 Subject: [PATCH 3/7] test(recipe): couple the K8s floor guard to the registry pins and finish the sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps from review. The guard's value regex required double quotes, so a single-quoted or plain scalar was not misparsed but INVISIBLE — the declaration was never counted or checked. It now matches any YAML scalar style and trims quoting. The floor was a hardcoded constant that never read the registry, so a DRA chart bump raising kubeVersion would leave the guard green at a stale 1.32 despite a comment claiming the two move together. Replaced with an audited component/version/floor table plus TestDRAChartFloorAuditIsCurrent, following the ownsCRDs version-audit pattern. Both DRA components are enrolled: the earlier comment wrongly claimed no overlay disables the generic one, but ocp.yaml sets enabled: false and substitutes nvidia-dra-driver-gpu-ocp, so covering only the generic entry left the OCP chain unguarded. Controls verified: a single-quoted ">= 1.29" now fails where it was previously invisible, and pointing an audited entry at a version the registry does not pin fails the audit test. Also finishes the floor-reference sweep — the CLI reference constraint examples, the component-catalog Topology Updater note, the OKE L40S demo claim that the floor drops to 1.30, and two test comments naming the old kind 1.25 floor. Left alone: the GB200 table in demos/images/recipe.md claimed >= 1.28 before this PR while GB200 already required 1.34, so it is pre-existing drift rather than this change's to correct. Signed-off-by: Yuan Chen --- demos/query.md | 6 +- docs/user/cli-reference.md | 4 +- docs/user/component-catalog.md | 2 +- pkg/cli/touched_invariant_test.go | 2 +- pkg/client/v1/relax_test.go | 2 +- pkg/recipe/dra_k8s_floor_test.go | 91 ++++++++++++++++++++++++------- 6 files changed, 81 insertions(+), 26 deletions(-) diff --git a/demos/query.md b/demos/query.md index 17cb7bddd..f98c23bcd 100644 --- a/demos/query.md +++ b/demos/query.md @@ -110,14 +110,16 @@ aicr query --service gke --accelerator h100 --intent training --os cos \ value: '>= 1.32' ``` -L40S (on OKE) relaxes K8s further (older accelerators run on older clusters): +L40S (on OKE) resolves a different component set; its K8s floor now matches the +catalog-wide minimum (every recipe carries the DRA driver, whose chart requires +1.32): ```shell aicr query --service oke --accelerator l40s --intent training --os ol \ --selector constraints ``` -> K8s minimum drops from `1.32.4` to `1.30`. +> K8s minimum relaxes from `1.32.4` to `1.32`. ### Component set differs by intent diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 0e2dc13f9..f957a7c04 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -1016,7 +1016,7 @@ aicr query --service eks --accelerator gb200 --intent training \ --selector components.nodewright-customizations.values # Watch constraints tighten as you add specificity -# Just "EKS" → 1 constraint (K8s >= 1.28) +# Just "EKS" → 1 constraint (K8s >= 1.32) aicr query --service eks --selector constraints # Add GPU + intent + OS → 4 constraints (K8s >= 1.32.4, Ubuntu 24.04, kernel >= 6.8) aicr query --service eks --accelerator h100 --intent training --os ubuntu \ @@ -1141,7 +1141,7 @@ Supported operators: | Operator | Example | Description | |----------|---------|-------------| -| `>=` | `>= 1.30` | Greater than or equal (version comparison) | +| `>=` | `>= 1.32` | Greater than or equal (version comparison) | | `<=` | `<= 1.33` | Less than or equal (version comparison) | | `>` | `> 1.30` | Greater than (version comparison) | | `<` | `< 2.0` | Less than (version comparison) | diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index 0ee6a04ad..9110b177a 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -72,7 +72,7 @@ Not every component appears in every recipe. The recipe engine selects component Production GPU leaf recipes (H100, GB200, RTX Pro 6000 on EKS / AKS / GKE / OKE / LKE) enable the NFD Topology Updater. It publishes per-node `NodeResourceTopology` CRDs that describe NUMA zones, GPU-to-NUMA affinity, and NIC-to-NUMA affinity. Runtime consumers (NUMA-aware schedulers, debugging via `kubectl get noderesourcetopologies`) can read these CRDs without further configuration. -The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints on the affected leaves require K8s ≥ 1.30 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. +The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. See the upstream [Topology Updater docs](https://kubernetes-sigs.github.io/node-feature-discovery/stable/usage/nfd-topology-updater.html) for runtime consumer examples. diff --git a/pkg/cli/touched_invariant_test.go b/pkg/cli/touched_invariant_test.go index 6cb84eddc..bae5635d0 100644 --- a/pkg/cli/touched_invariant_test.go +++ b/pkg/cli/touched_invariant_test.go @@ -244,7 +244,7 @@ func TestRecipeCmd_Snapshot_StatedDimensionNotRelaxed(t *testing.T) { } // constraintFailingKindSnapshotYAML fingerprints to service=kind on a -// Kubernetes version below the kind overlay's `K8s.server.version >= 1.25` +// Kubernetes version below the kind overlay's `K8s.server.version >= 1.32` // constraint, so the only overlay covering service=kind is excluded by // constraint evaluation rather than absent from the catalog. const constraintFailingKindSnapshotYAML = `kind: Snapshot diff --git a/pkg/client/v1/relax_test.go b/pkg/client/v1/relax_test.go index 23555c4a7..bd26aae7e 100644 --- a/pkg/client/v1/relax_test.go +++ b/pkg/client/v1/relax_test.go @@ -486,7 +486,7 @@ func kindUbuntuSnapshot() *snapshotter.Snapshot { } // constraintFailingKindSnapshot fingerprints to service=kind on a Kubernetes -// version below the kind overlay's `K8s.server.version >= 1.25` constraint, so +// version below the kind overlay's `K8s.server.version >= 1.32` constraint, so // constraint evaluation excludes the only overlay that covers service=kind. func constraintFailingKindSnapshot() *snapshotter.Snapshot { return &snapshotter.Snapshot{ diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 054ad7377..c1464cf93 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -22,26 +22,77 @@ import ( "testing" ) -// draChartKubeVersionMinor is the minor version floor the pinned -// nvidia-dra-driver-gpu chart declares: +// auditedDRAChartFloors records, per DRA component, the chart version whose +// kubeVersion was read and the Kubernetes minor it declares. // -// kubeVersion: ">=1.32.0-0" +// Both catalog DRA components are enrolled. OCP disables the generic +// nvidia-dra-driver-gpu (recipes/overlays/ocp.yaml sets enabled: false) and +// substitutes nvidia-dra-driver-gpu-ocp, so covering only the generic one would +// leave the OCP chain unguarded. // -// Helm REJECTS the install outright when the cluster is below it, so a recipe -// declaring a lower K8s.server.version admits clusters that pass every -// recipe-time check and then fail at `helm install`. -// -// Sourced from the chart, not from a running cluster. If the DRA chart pin in -// recipes/registry.yaml moves to a version with a different kubeVersion, this -// constant and the affected overlays must move together — that coupling is the -// point of the guard. -const draChartKubeVersionMinor = 32 +// Re-audit procedure when a pin moves: read `kubeVersion` from the chart at the +// new version and update the entry. TestDRAChartFloorAuditIsCurrent fails until +// you do, which is what couples this guard to the registry rather than letting +// it sit green at a stale floor. +var auditedDRAChartFloors = map[string]struct { + version string + minor int +}{ + // oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu + // kubeVersion: ">=1.32.0-0" + "nvidia-dra-driver-gpu": {version: "0.4.1", minor: 32}, + "nvidia-dra-driver-gpu-ocp": {version: "0.4.1", minor: 32}, +} + +// draChartKubeVersionMinor is the highest audited floor across the enrolled DRA +// components — the value every recipe must clear, since each recipe carries one +// of them. +func draChartKubeVersionMinorFn() int { + highest := 0 + for _, a := range auditedDRAChartFloors { + if a.minor > highest { + highest = a.minor + } + } + return highest +} + +// TestDRAChartFloorAuditIsCurrent fails when a DRA chart pin in registry.yaml +// moves away from the version whose kubeVersion was audited, so a chart bump +// cannot silently leave the floor guard asserting a stale minor. +func TestDRAChartFloorAuditIsCurrent(t *testing.T) { + t.Parallel() + + registry, err := GetComponentRegistry() + if err != nil { + t.Fatalf("GetComponentRegistry: %v", err) + } + + for name, audited := range auditedDRAChartFloors { + cfg := registry.Get(name) + if cfg == nil { + t.Errorf("audited DRA component %q is not in the registry; remove it from "+ + "auditedDRAChartFloors or restore the component", name) + continue + } + if cfg.Helm.DefaultVersion != audited.version { + t.Errorf("%s is pinned at %q but its kubeVersion was audited at %q.\n"+ + " Read `kubeVersion` from the chart at %s, update auditedDRAChartFloors,\n"+ + " and raise the affected overlay floors if it moved. See #2402.", + name, cfg.Helm.DefaultVersion, audited.version, cfg.Helm.DefaultVersion) + } + } +} // k8sConstraintRE captures EVERY K8s.server.version declaration in a file and // its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry // more than one declaration, and checking only the first would let a later, // lower one through. -var k8sConstraintRE = regexp.MustCompile(`- name: K8s\.server\.version\s*\n\s*value: "([^"]*)"`) +// The value is matched irrespective of quoting: YAML accepts double-quoted, +// single-quoted and plain scalars, and a guard that only sees one style would +// not merely misparse the others — it would not see the declaration at all. +var k8sConstraintRE = regexp.MustCompile( + `- name: K8s\.server\.version\s*\n\s*value:[ \t]*("[^"]*"|'[^']*'|[^\n#]*)`) // geFloorRE matches the `>= 1.` form this guard can reason about. var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) @@ -49,8 +100,10 @@ var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a // Kubernetes floor below the DRA chart's own kubeVersion. // -// Every recipe inherits nvidia-dra-driver-gpu from base.yaml — no overlay -// removes or disables it — so the chart's floor applies catalog-wide. +// Every recipe carries a DRA driver: base.yaml declares nvidia-dra-driver-gpu, +// and OCP disables that one and substitutes nvidia-dra-driver-gpu-ocp. Both +// resolve to the same upstream chart and the same kubeVersion, so the floor +// applies catalog-wide either way. // // Why every declaration and not just base.yaml: constraints merge by name with // the LATER overlay winning and no max comparison (see mergeValidation in @@ -83,7 +136,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { } for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { checked++ - value := m[1] + value := strings.Trim(strings.TrimSpace(m[1]), `"'`) g := geFloorRE.FindStringSubmatch(value) if g == nil { @@ -97,7 +150,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ " this guard to understand the new form. See #2402.", - path, value, draChartKubeVersionMinor) + path, value, draChartKubeVersionMinorFn()) continue } @@ -106,7 +159,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) continue } - if minor < draChartKubeVersionMinor { + if minor < draChartKubeVersionMinorFn() { t.Errorf("%s declares K8s.server.version %q, below the pinned "+ "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ @@ -115,7 +168,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ " with no max comparison, so a lower leaf value overwrites a higher\n"+ " inherited one. See #2402.", - path, value, draChartKubeVersionMinor, draChartKubeVersionMinor) + path, value, draChartKubeVersionMinorFn(), draChartKubeVersionMinorFn()) } } return nil From e7052639b5993bea2bdc37a3ca16fb4981c605d1 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:39:58 -0700 Subject: [PATCH 4/7] test(recipe): evaluate K8s floors typed instead of pattern-matching YAML text Decode each overlay and mixin into RecipeMetadata and evaluate every K8s.server.version constraint with the shipping parser and evaluator, so the guard no longer depends on YAML key order, quoting, or the expression's surface form. Also correct two stale comments: the MirrorDefaultKubeVersion note naming the old ">= 1.25" base floor, and the A100 GKE contrast with an H100 floor the recipe now shares. Signed-off-by: Yuan Chen --- pkg/defaults/timeouts.go | 2 +- pkg/recipe/dra_k8s_floor_test.go | 252 ++++++++++++++++---- recipes/overlays/a100-gke-cos-training.yaml | 5 +- 3 files changed, 203 insertions(+), 56 deletions(-) diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index f2425897e..c1cb49943 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -1272,7 +1272,7 @@ const ( // // This is a render-safe floor, not a support floor. This constant must // stay at or above the strictest kubeVersion any bundled chart declares. - // Do NOT lower it to match the ">= 1.25" recipe floor in + // Do NOT lower it to match the ">= 1.32" recipe floor in // recipes/overlays/base.yaml: recipes are validated against their own // constraints, while mirror discovery raises lower versions to this // value solely for Helm rendering (see mirror.KubeVersionFromConstraints). diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index c1464cf93..e06cfa352 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -12,16 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package recipe +// This guard lives in the external recipe_test package so it can import +// pkg/constraints — the production constraint parser and evaluator — which +// itself imports pkg/recipe. An in-package test could not, and would be forced +// to reimplement comparison logic that the shipping evaluator already owns. +package recipe_test import ( + "fmt" "io/fs" - "regexp" - "strconv" "strings" "testing" + + "gopkg.in/yaml.v3" + + "github.com/NVIDIA/aicr/pkg/constraints" + "github.com/NVIDIA/aicr/pkg/recipe" ) +// k8sServerVersionConstraint is the measurement path whose floor this guard +// audits. +const k8sServerVersionConstraint = "K8s.server.version" + // auditedDRAChartFloors records, per DRA component, the chart version whose // kubeVersion was read and the Kubernetes minor it declares. // @@ -47,7 +59,7 @@ var auditedDRAChartFloors = map[string]struct { // draChartKubeVersionMinor is the highest audited floor across the enrolled DRA // components — the value every recipe must clear, since each recipe carries one // of them. -func draChartKubeVersionMinorFn() int { +func draChartKubeVersionMinor() int { highest := 0 for _, a := range auditedDRAChartFloors { if a.minor > highest { @@ -63,7 +75,7 @@ func draChartKubeVersionMinorFn() int { func TestDRAChartFloorAuditIsCurrent(t *testing.T) { t.Parallel() - registry, err := GetComponentRegistry() + registry, err := recipe.GetComponentRegistry() if err != nil { t.Fatalf("GetComponentRegistry: %v", err) } @@ -84,21 +96,110 @@ func TestDRAChartFloorAuditIsCurrent(t *testing.T) { } } -// k8sConstraintRE captures EVERY K8s.server.version declaration in a file and -// its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry -// more than one declaration, and checking only the first would let a later, -// lower one through. -// The value is matched irrespective of quoting: YAML accepts double-quoted, -// single-quoted and plain scalars, and a guard that only sees one style would -// not merely misparse the others — it would not see the declaration at all. -var k8sConstraintRE = regexp.MustCompile( - `- name: K8s\.server\.version\s*\n\s*value:[ \t]*("[^"]*"|'[^']*'|[^\n#]*)`) +// subFloorProbeVersions returns the Kubernetes version strings that MUST NOT +// satisfy any catalog floor: every minor below the DRA chart's kubeVersion, +// rendered in each shape a real cluster reading or a recipe author's exact pin +// can take. +// +// Probing the production evaluator with concrete readings — rather than +// pattern-matching the expression text — is what makes this guard independent +// of the expression's *form*. A prefix match on ">= 1." is defeated by a +// compound expression (">= 1.32 || >= 1.29" begins with a safe floor and is +// still satisfied by 1.29.7, because the shipping parser treats "||" as OR), +// and a bare-string exact pin ("1.30") carries no operator to match at all. +// Both are caught here because both admit a sub-floor reading. +func subFloorProbeVersions(floorMinor int) []string { + probes := make([]string, 0, 2+4*floorMinor) + probes = append(probes, "0.99", "0.99.99") + for minor := range floorMinor { + probes = append(probes, + fmt.Sprintf("1.%d", minor), + fmt.Sprintf("1.%d.0", minor), + fmt.Sprintf("1.%d.99", minor), + fmt.Sprintf("v1.%d.0", minor), + ) + } + return probes +} + +// supportedProbeVersions returns readings at or above the floor, used only to +// prove a constraint is not vacuous. An expression satisfied by nothing admits +// no sub-floor cluster and would pass the sub-floor sweep trivially, but it +// also rejects every cluster the catalog claims to support — a typo, not a +// floor. Failing on it keeps the guard closed against expressions it cannot +// show are meaningful. +func supportedProbeVersions(floorMinor int) []string { + var probes []string + for minor := floorMinor; minor <= 60; minor++ { + probes = append(probes, fmt.Sprintf("1.%d.0", minor)) + } + return probes +} -// geFloorRE matches the `>= 1.` form this guard can reason about. -var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) +// k8sFloorDeclaration is one typed K8s.server.version constraint located in the +// catalog, with the structural path it was found at for error reporting. +type k8sFloorDeclaration struct { + file string + location string + value string +} + +// collectK8sFloorDeclarations decodes one recipe metadata document and returns +// every K8s.server.version constraint it declares, from every field that can +// carry one: spec.constraints, each validation phase, and each profile value's +// constraints and readinessConstraints. +// +// Decoding to the typed form is what closes the YAML-layout hole: a mapping +// written "value:" before "name:" is the same Constraint after unmarshalling, +// so key order, quoting style, comments, and indentation are all invisible to +// this guard by construction rather than by widening a regex. +func collectK8sFloorDeclarations(file string, raw []byte) ([]k8sFloorDeclaration, error) { + var metadata recipe.RecipeMetadata + if err := yaml.Unmarshal(raw, &metadata); err != nil { + return nil, err + } + + var found []k8sFloorDeclaration + collect := func(location string, cs []recipe.Constraint) { + for _, c := range cs { + if c.Name != k8sServerVersionConstraint { + continue + } + found = append(found, k8sFloorDeclaration{file: file, location: location, value: c.Value}) + } + } + + spec := metadata.Spec + collect("spec.constraints", spec.Constraints) + + if v := spec.Validation; v != nil { + for _, phase := range []struct { + name string + p *recipe.ValidationPhase + }{ + {"readiness", v.Readiness}, + {"deployment", v.Deployment}, + {"performance", v.Performance}, + {"conformance", v.Conformance}, + } { + if phase.p != nil { + collect("spec.validation."+phase.name+".constraints", phase.p.Constraints) + } + } + } + + if p := spec.Profile; p != nil { + for valueName, pv := range p.Values { + collect(fmt.Sprintf("spec.profile.values[%s].constraints", valueName), pv.Constraints) + collect(fmt.Sprintf("spec.profile.values[%s].readinessConstraints", valueName), pv.ReadinessConstraints) + } + } + + return found, nil +} // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a -// Kubernetes floor below the DRA chart's own kubeVersion. +// Kubernetes floor that admits a cluster below the DRA chart's own kubeVersion. // // Every recipe carries a DRA driver: base.yaml declares nvidia-dra-driver-gpu, // and OCP disables that one and substitutes nvidia-dra-driver-gpu-ocp. Both @@ -114,10 +215,23 @@ var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason // before the rest of the catalog was reconciled; its comment records the // diagnosis. +// +// How it checks, and why not by reading the expression: each declaration is +// decoded typed, parsed with the shipping parser +// (constraints.ParseCompoundConstraint), and then EVALUATED against concrete +// sub-floor readings with the shipping evaluator. The guard therefore asserts +// the property that actually matters — "no supported-but-too-old cluster +// satisfies this" — instead of asserting the expression is spelled a +// particular way. It fails closed on any expression the parser rejects and on +// any expression no supported reading satisfies. func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Parallel() - efs := GetEmbeddedFS() + floorMinor := draChartKubeVersionMinor() + subFloor := subFloorProbeVersions(floorMinor) + supported := supportedProbeVersions(floorMinor) + + efs := recipe.GetEmbeddedFS() var checked int err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { @@ -134,42 +248,15 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { if readErr != nil { return readErr } - for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { - checked++ - value := strings.Trim(strings.TrimSpace(m[1]), `"'`) - - g := geFloorRE.FindStringSubmatch(value) - if g == nil { - // Fail closed on any form this guard cannot interpret — an exact - // pin (== 1.30), a range, or a bare version would each be just as - // capable of admitting a sub-floor cluster, and silently skipping - // them would make the guard weakest exactly where a future author - // deviates from the established shape. - t.Errorf("%s declares K8s.server.version %q, a form this guard cannot verify.\n"+ - " It can only reason about \">= 1.\". Any other form may admit clusters\n"+ - " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ - " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ - " this guard to understand the new form. See #2402.", - path, value, draChartKubeVersionMinorFn()) - continue - } + decls, decodeErr := collectK8sFloorDeclarations(path, raw) + if decodeErr != nil { + t.Errorf("%s: could not decode recipe metadata: %v", path, decodeErr) + return nil + } - minor, convErr := strconv.Atoi(g[1]) - if convErr != nil { - t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) - continue - } - if minor < draChartKubeVersionMinorFn() { - t.Errorf("%s declares K8s.server.version %q, below the pinned "+ - "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - path, value, draChartKubeVersionMinorFn(), draChartKubeVersionMinorFn()) - } + for _, decl := range decls { + checked++ + verifyK8sFloorDeclaration(t, decl, floorMinor, subFloor, supported) } return nil }) @@ -185,3 +272,62 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { } t.Logf("verified %d K8s.server.version floor(s) clear the DRA chart floor", checked) } + +// verifyK8sFloorDeclaration checks one declaration against the DRA chart floor +// using the production parser and evaluator. +func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, subFloor, supported []string) { + t.Helper() + + parsed, err := constraints.ParseCompoundConstraint(decl.value) + if err != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping constraint\n"+ + " parser rejects: %v\n"+ + " An expression aicr cannot parse cannot be shown to clear the pinned\n"+ + " nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\". See #2402.", + decl.file, decl.location, decl.value, err, floorMinor) + return + } + + for _, reading := range subFloor { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ + " could not evaluate against the Kubernetes reading %q: %v\n"+ + " The guard fails closed: an expression whose result is unknown may admit a\n"+ + " cluster below the chart floor \">=1.%d.0-0\". See #2402.", + decl.file, decl.location, decl.value, reading, evalErr, floorMinor) + return + } + if satisfied { + t.Errorf("%s (%s) declares K8s.server.version %q, which is SATISFIED by a\n"+ + " Kubernetes %s cluster — below the pinned nvidia-dra-driver-gpu chart's\n"+ + " kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + decl.file, decl.location, decl.value, reading, floorMinor, floorMinor) + return + } + } + + for _, reading := range supported { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ + " could not evaluate against the Kubernetes reading %q: %v. See #2402.", + decl.file, decl.location, decl.value, reading, evalErr) + return + } + if satisfied { + return + } + } + + t.Errorf("%s (%s) declares K8s.server.version %q, which no Kubernetes release from\n"+ + " 1.%d through 1.60 satisfies. It admits no cluster the catalog supports, so this\n"+ + " guard cannot show it is a floor rather than a typo, and fails closed. See #2402.", + decl.file, decl.location, decl.value, floorMinor) +} diff --git a/recipes/overlays/a100-gke-cos-training.yaml b/recipes/overlays/a100-gke-cos-training.yaml index ae92d247d..c3c6c69c9 100644 --- a/recipes/overlays/a100-gke-cos-training.yaml +++ b/recipes/overlays/a100-gke-cos-training.yaml @@ -28,8 +28,9 @@ spec: intent: training # Specific constraints for A100 on GKE COS training workloads. - # A100 has no IMEX/NVLink ComputeDomain requirement, so the recipe keeps - # the GKE COS training baseline rather than the H100 1.32 floor. + # A100 has no IMEX/NVLink ComputeDomain requirement, so this floor is not + # driven by a ComputeDomain need as the H100 recipes' is; it simply carries + # the catalog-wide 1.32 baseline the DRA driver chart's kubeVersion sets. constraints: - name: K8s.server.version value: ">= 1.32" From ef60b6b9805baa8373a1015b585c6d296ae753fc Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 15:16:21 -0700 Subject: [PATCH 5/7] test(recipe): prove the K8s floor symbolically instead of sampling versions The guard proved 'no too-old cluster satisfies this floor' by evaluating each declared expression against a fixed list of probe versions. That is sampling, and the production grammar supports arbitrary OR-of-AND ranges, so no finite probe list can cover it. '>= 1.32 || > 1.31.0 < 1.31.2' is a supported shape that passed the guard while the production evaluator accepts a Kubernetes 1.31.1 cluster that Helm's '>=1.32.0-0' rejects. Walk the parsed structure from constraints.ParseCompoundConstraint instead and prove the effective lower bound. An AND group's satisfying set is the intersection of its terms, so the group clears the floor as soon as any one term does; a compound's satisfying set is the union of its groups, so every group must clear it. Only >=, >, ==, and bare exact match place a lower bound; <, <=, and != place none. Anything else - an unparseable value, a major-only precision, an unknown operator - fails closed rather than being waved through. Symbolic proof was chosen over restricting the catalog to a simple '>= X.Y' form because it keeps the per-track GKE range expressions the parser already supports (see #1985) provable rather than banned, and it is exact where a grammar restriction is merely conservative. The defeating expression is kept as a permanent regression control in TestProveExpressionClearsFloor, with an adversarial control asserting the production evaluator really does admit 1.31.1 for it - so a prover bug that rejected everything cannot make the table green. Also corrects the comment attributing top-level constraint last-wins merging to mergeValidation in validation.go: RecipeMetadataSpec.Merge in metadata.go is what merges spec.constraints; mergeValidationPhase handles phase constraints. Signed-off-by: Yuan Chen --- pkg/recipe/dra_k8s_floor_test.go | 285 ++++++++++++++++++++++++------- 1 file changed, 219 insertions(+), 66 deletions(-) diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index e06cfa352..3c56fe88c 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -28,6 +28,7 @@ import ( "github.com/NVIDIA/aicr/pkg/constraints" "github.com/NVIDIA/aicr/pkg/recipe" + "github.com/NVIDIA/aicr/pkg/version" ) // k8sServerVersionConstraint is the measurement path whose floor this guard @@ -96,30 +97,108 @@ func TestDRAChartFloorAuditIsCurrent(t *testing.T) { } } -// subFloorProbeVersions returns the Kubernetes version strings that MUST NOT -// satisfy any catalog floor: every minor below the DRA chart's kubeVersion, -// rendered in each shape a real cluster reading or a recipe author's exact pin -// can take. +// floorMajor is the Kubernetes major version the DRA chart floors sit on. The +// audited kubeVersion is ">=1.32.0-0", so every bound is compared as +// (major, minor) against (floorMajor, floorMinor). +const floorMajor = 1 + +// termClearsFloor reports whether a single parsed term, on its own, confines +// every version that satisfies it to at or above Kubernetes +// floorMajor.floorMinor.0. +// +// Only the lower-bounding operators can do that: +// +// - ">= v" admits exactly [v, inf) +// - "> v" admits (v, inf); requiring v itself to clear the floor is one +// patch conservative and never fails open, since the patch component is +// unbounded (there is no "last" 1.31.x to fall back on) +// - "== v" and a bare exact match admit only v +// +// "<", "<=", and "!=" place no lower bound at all, so they return false: they +// can narrow an alternative but can never be what lifts it above the floor. // -// Probing the production evaluator with concrete readings — rather than -// pattern-matching the expression text — is what makes this guard independent -// of the expression's *form*. A prefix match on ">= 1." is defeated by a -// compound expression (">= 1.32 || >= 1.29" begins with a safe floor and is -// still satisfied by 1.29.7, because the shipping parser treats "||" as OR), -// and a bare-string exact pin ("1.30") carries no operator to match at all. -// Both are caught here because both admit a sub-floor reading. -func subFloorProbeVersions(floorMinor int) []string { - probes := make([]string, 0, 2+4*floorMinor) - probes = append(probes, "0.99", "0.99.99") - for minor := range floorMinor { - probes = append(probes, - fmt.Sprintf("1.%d", minor), - fmt.Sprintf("1.%d.0", minor), - fmt.Sprintf("1.%d.99", minor), - fmt.Sprintf("v1.%d.0", minor), - ) +// A value the shipping version parser cannot read, or one written with less +// than major.minor precision (">= 1"), returns an error. Comparing "1" against +// "1.32.0" at min-precision would report equal and wave a floorless expression +// through, so the guard refuses to reason about it instead. +func termClearsFloor(pc constraints.ParsedConstraint, floorMinor int) (bool, error) { + switch pc.Operator { + case constraints.OperatorLT, constraints.OperatorLTE, constraints.OperatorNE: + return false, nil + case constraints.OperatorGTE, constraints.OperatorGT, constraints.OperatorEQ, constraints.OperatorExact: + default: + return false, fmt.Errorf("unknown operator %q; the guard cannot prove a lower bound for it", pc.Operator) } - return probes + + v, err := version.ParseVersion(pc.Value) + if err != nil { + return false, fmt.Errorf("value %q is not a version the shipping parser can read: %w", pc.Value, err) + } + if v.Precision < 2 { + return false, fmt.Errorf("value %q has only major precision; a floor must name at least major.minor", pc.Value) + } + if v.Major > floorMajor { + return true, nil + } + return v.Major == floorMajor && v.Minor >= floorMinor, nil +} + +// alternativeString renders one AND group for error messages. +func alternativeString(group []constraints.ParsedConstraint) string { + terms := make([]string, 0, len(group)) + for i := range group { + terms = append(terms, group[i].String()) + } + return strings.Join(terms, " ") +} + +// proveExpressionClearsFloor proves, symbolically, that no Kubernetes cluster +// below floorMajor.floorMinor.0 can satisfy expr. It returns nil only when the +// proof succeeds, and a describing error otherwise — including for any +// expression it cannot reason about, so the guard fails closed. +// +// Why symbolic and not by probing readings: the production grammar admits +// arbitrary OR-of-AND range expressions, so no finite list of probe versions +// covers it. ">= 1.32 || > 1.31.0 < 1.31.2" is a supported shape that a probe +// sweep over 1.N / 1.N.0 / 1.N.99 misses entirely while the production +// evaluator happily accepts a 1.31.1 cluster that Helm's ">=1.32.0-0" rejects. +// +// The proof: an AND group's satisfying set is the intersection of its terms, so +// the group clears the floor as soon as ANY ONE of its terms does — a single +// ">= 1.32" makes the whole group safe no matter what the others say. A +// compound expression's satisfying set is the union of its groups, so EVERY +// group must clear the floor; one loose alternative admits a sub-floor cluster +// regardless of how strict its siblings are. +func proveExpressionClearsFloor(expr string, floorMinor int) error { + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + return fmt.Errorf("the shipping constraint parser rejects it: %w", err) + } + if len(parsed.Alternatives) == 0 { + return fmt.Errorf("it parsed to zero OR alternatives, so nothing bounds it") + } + + for i, group := range parsed.Alternatives { + if len(group) == 0 { + return fmt.Errorf("OR alternative %d has no terms, so nothing bounds it", i+1) + } + cleared := false + for j := range group { + ok, termErr := termClearsFloor(group[j], floorMinor) + if termErr != nil { + return fmt.Errorf("OR alternative %d (%q): %w", i+1, alternativeString(group), termErr) + } + if ok { + cleared = true + } + } + if !cleared { + return fmt.Errorf("OR alternative %d (%q) carries no lower bound at or above %d.%d.0, "+ + "so at least one cluster below the chart floor satisfies it", + i+1, alternativeString(group), floorMajor, floorMinor) + } + } + return nil } // supportedProbeVersions returns readings at or above the floor, used only to @@ -207,28 +286,32 @@ func collectK8sFloorDeclarations(file string, raw []byte) ([]k8sFloorDeclaration // applies catalog-wide either way. // // Why every declaration and not just base.yaml: constraints merge by name with -// the LATER overlay winning and no max comparison (see mergeValidation in -// validation.go). A leaf declaring ">= 1.30" silently overwrites a higher floor -// inherited from base, so raising base alone would not hold. This is the same +// the LATER overlay winning and no max comparison (see RecipeMetadataSpec.Merge +// in metadata.go; validation-phase constraints merge the same way in +// mergeValidationPhase in validation.go). A leaf declaring ">= 1.30" silently +// overwrites a higher floor inherited from base, so raising base alone would +// not hold. This is the same // last-wins hazard documented for driver floors in #2438. // // recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason // before the rest of the catalog was reconciled; its comment records the // diagnosis. // -// How it checks, and why not by reading the expression: each declaration is -// decoded typed, parsed with the shipping parser -// (constraints.ParseCompoundConstraint), and then EVALUATED against concrete -// sub-floor readings with the shipping evaluator. The guard therefore asserts -// the property that actually matters — "no supported-but-too-old cluster -// satisfies this" — instead of asserting the expression is spelled a -// particular way. It fails closed on any expression the parser rejects and on -// any expression no supported reading satisfies. +// How it checks: each declaration is decoded typed, parsed with the shipping +// parser (constraints.ParseCompoundConstraint), and then PROVEN — symbolically, +// over the parsed OR-of-AND structure — to carry a lower bound at or above the +// chart floor on every alternative. See proveExpressionClearsFloor. +// +// Sampling the evaluator with a list of sub-floor readings was tried and is not +// sufficient: the grammar admits arbitrary ranges, and ">= 1.32 || > 1.31.0 +// < 1.31.2" slips past any fixed probe list while admitting a 1.31.1 cluster. +// The guard fails closed on any expression the parser rejects, on any operator +// or value it cannot reason about, and on any expression no supported reading +// satisfies. func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Parallel() floorMinor := draChartKubeVersionMinor() - subFloor := subFloorProbeVersions(floorMinor) supported := supportedProbeVersions(floorMinor) efs := recipe.GetEmbeddedFS() @@ -256,7 +339,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { for _, decl := range decls { checked++ - verifyK8sFloorDeclaration(t, decl, floorMinor, subFloor, supported) + verifyK8sFloorDeclaration(t, decl, floorMinor, supported) } return nil }) @@ -275,42 +358,29 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { // verifyK8sFloorDeclaration checks one declaration against the DRA chart floor // using the production parser and evaluator. -func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, subFloor, supported []string) { +func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, supported []string) { t.Helper() - parsed, err := constraints.ParseCompoundConstraint(decl.value) - if err != nil { - t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping constraint\n"+ - " parser rejects: %v\n"+ - " An expression aicr cannot parse cannot be shown to clear the pinned\n"+ - " nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\". See #2402.", - decl.file, decl.location, decl.value, err, floorMinor) + if err := proveExpressionClearsFloor(decl.value, floorMinor); err != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which this guard cannot prove\n"+ + " clears the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\":\n"+ + " %v\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so a recipe that admits a lower cluster\n"+ + " validates clean and then fails at `helm install`. Raise it to \">= 1.%d\",\n"+ + " or, if the expression is genuinely safe in a form the guard cannot yet\n"+ + " prove, extend proveExpressionClearsFloor rather than loosening it.\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + decl.file, decl.location, decl.value, floorMinor, err, floorMinor) return } - for _, reading := range subFloor { - satisfied, evalErr := parsed.Evaluate(reading) - if evalErr != nil { - t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ - " could not evaluate against the Kubernetes reading %q: %v\n"+ - " The guard fails closed: an expression whose result is unknown may admit a\n"+ - " cluster below the chart floor \">=1.%d.0-0\". See #2402.", - decl.file, decl.location, decl.value, reading, evalErr, floorMinor) - return - } - if satisfied { - t.Errorf("%s (%s) declares K8s.server.version %q, which is SATISFIED by a\n"+ - " Kubernetes %s cluster — below the pinned nvidia-dra-driver-gpu chart's\n"+ - " kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - decl.file, decl.location, decl.value, reading, floorMinor, floorMinor) - return - } + parsed, err := constraints.ParseCompoundConstraint(decl.value) + if err != nil { + t.Errorf("%s (%s): %v", decl.file, decl.location, err) + return } for _, reading := range supported { @@ -331,3 +401,86 @@ func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMino " guard cannot show it is a floor rather than a typo, and fails closed. See #2402.", decl.file, decl.location, decl.value, floorMinor) } + +// TestProveExpressionClearsFloor pins the prover's behavior on the shapes the +// catalog guard has to withstand. Every "must fail" row is a permanent +// regression control: each one is an expression a real author could write that +// the production evaluator accepts for a sub-floor cluster. +// +// The last row is the shape that defeated the previous probe-sampling guard: a +// safe first alternative followed by a narrow sub-floor range. It is kept here +// permanently so no future rewrite can reintroduce sampling and stay green. +func TestProveExpressionClearsFloor(t *testing.T) { + t.Parallel() + + const floorMinor = 32 + + tests := []struct { + name string + expr string + wantErr bool + }{ + {"simple floor at the chart minor", ">= 1.32", false}, + {"simple floor with patch", ">= 1.32.4", false}, + {"floor above the chart minor", ">= 1.33.0", false}, + {"major above the floor", ">= 2.0", false}, + {"range whose lower bound clears the floor", ">= 1.32.4 < 1.35.0", false}, + {"every alternative clears the floor", ">= 1.34.3-gke.1318000 < 1.35.0 || >= 1.35.0-gke.2745000", false}, + {"upper-bounded term does not lift a cleared group", ">= 1.32 < 1.33", false}, + + {"floor below the chart minor", ">= 1.30", true}, + {"simple compound with a low alternative", ">= 1.32 || >= 1.29", true}, + {"exact pin below the chart minor", "== 1.30", true}, + {"bare exact pin below the chart minor", "1.30", true}, + {"greater-than below the chart minor", "> 1.31", true}, + {"only an upper bound", "< 1.40", true}, + {"only a not-equal", "!= 1.30", true}, + {"major-only precision", ">= 1", true}, + {"non-version value", ">= stable", true}, + {"empty expression", "", true}, + {"empty OR clause", ">= 1.32 ||", true}, + // The control: accepted by the production evaluator for a 1.31.1 + // cluster, which Helm's ">=1.32.0-0" rejects. A probe sweep over + // 1.N / 1.N.0 / 1.N.99 / v1.N.0 misses it entirely. + {"narrow sub-floor range hidden behind a safe alternative", ">= 1.32 || > 1.31.0 < 1.31.2", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := proveExpressionClearsFloor(tt.expr, floorMinor) + if (err != nil) != tt.wantErr { + t.Fatalf("proveExpressionClearsFloor(%q, %d) error = %v, wantErr %v", + tt.expr, floorMinor, err, tt.wantErr) + } + }) + } +} + +// TestProveExpressionRejectsWhatTheEvaluatorAdmits is the adversarial control +// for the row above: it proves independently that the production evaluator +// really does accept a sub-floor cluster for that expression, so the "must +// fail" verdict is grounded in behavior rather than in the prover's own +// opinion. Without this, a prover bug that rejected everything would still +// make the table green. +func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { + t.Parallel() + + const expr = ">= 1.32 || > 1.31.0 < 1.31.2" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("ParseCompoundConstraint(%q): %v", expr, err) + } + satisfied, err := parsed.Evaluate("1.31.1") + if err != nil { + t.Fatalf("Evaluate(1.31.1): %v", err) + } + if !satisfied { + t.Fatalf("expected the production evaluator to accept 1.31.1 for %q; if this "+ + "changed, the control in TestProveExpressionClearsFloor needs rebasing", expr) + } + if err := proveExpressionClearsFloor(expr, 32); err == nil { + t.Fatalf("prover accepted %q even though the evaluator admits a 1.31.1 cluster", expr) + } +} From 2b9d355ac9a6c4fff7a3943176e57480d9e86e66 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 16:37:18 -0700 Subject: [PATCH 6/7] test(recipe): probe declared bounds so patch-precision floors are satisfiable The probe sweep tried only minor-precision readings, which no patch-precision range can satisfy: ">= 1.34.3 < 1.35.0" clears the prover, yet 1.34.0 is below its lower bound and 1.35.0 is excluded by its upper one, so a correct floor was reported as admitting no supported release. The catalog declares no such range today, so the defect was latent and fail-closed rather than fail-open. Extract probeReadings, which appends each declared bound to the minor-precision probes and returns a fresh slice - appending to the shared probe slice in place would write into a backing array reused by every declaration under test. TestProbeSetAdmitsPatchPrecisionRange pins this: reverting probeReadings to return the supported readings unchanged fails it. Also drop a redundant comparison in the component catalog. Signed-off-by: Yuan Chen --- docs/user/component-catalog.md | 2 +- pkg/recipe/dra_k8s_floor_test.go | 65 +++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index 9110b177a..8471b83f7 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -72,7 +72,7 @@ Not every component appears in every recipe. The recipe engine selects component Production GPU leaf recipes (H100, GB200, RTX Pro 6000 on EKS / AKS / GKE / OKE / LKE) enable the NFD Topology Updater. It publishes per-node `NodeResourceTopology` CRDs that describe NUMA zones, GPU-to-NUMA affinity, and NIC-to-NUMA affinity. Runtime consumers (NUMA-aware schedulers, debugging via `kubectl get noderesourcetopologies`) can read these CRDs without further configuration. -The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. +The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. See the upstream [Topology Updater docs](https://kubernetes-sigs.github.io/node-feature-discovery/stable/usage/nfd-topology-updater.html) for runtime consumer examples. diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 3c56fe88c..0754148da 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -207,6 +207,26 @@ func proveExpressionClearsFloor(expr string, floorMinor int) error { // also rejects every cluster the catalog claims to support — a typo, not a // floor. Failing on it keeps the guard closed against expressions it cannot // show are meaningful. +// probeReadings returns the Kubernetes readings used to show that a +// declaration admits at least one cluster. Minor-precision probes alone are +// not enough: a patch-precision range such as ">= 1.34.3 < 1.35.0" is +// satisfied by no "1.N.0" reading, so a correct floor would be reported as +// admitting nothing. The declared bounds are therefore probed as well. +// +// The result is a fresh slice. Appending to `supported` in place would write +// into a backing array shared by every declaration under test whenever it has +// spare capacity. +func probeReadings(supported []string, parsed *constraints.CompoundConstraint) []string { + readings := make([]string, 0, len(supported)) + readings = append(readings, supported...) + for _, group := range parsed.Alternatives { + for i := range group { + readings = append(readings, group[i].Value) + } + } + return readings +} + func supportedProbeVersions(floorMinor int) []string { var probes []string for minor := floorMinor; minor <= 60; minor++ { @@ -383,7 +403,7 @@ func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMino return } - for _, reading := range supported { + for _, reading := range probeReadings(supported, parsed) { satisfied, evalErr := parsed.Evaluate(reading) if evalErr != nil { t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ @@ -484,3 +504,46 @@ func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { t.Fatalf("prover accepted %q even though the evaluator admits a 1.31.1 cluster", expr) } } + +// TestProbeSetAdmitsPatchPrecisionRange is the regression control for the +// probe sweep in assertDeclarationIsSatisfiable. The sweep used to try only +// "1.N.0" readings, which no patch-precision range can satisfy: ">= 1.34.3 +// < 1.35.0" clears proveExpressionClearsFloor, yet 1.34.0 is below its lower +// bound and 1.35.0 is excluded by its upper one. A correct floor was therefore +// reported as admitting no supported release. +// +// Reverting probeReadings to return `supported` unchanged makes this test +// fail, which is the point: the catalog currently declares no patch-precision +// range, so the defect is latent and nothing else would catch a regression. +func TestProbeSetAdmitsPatchPrecisionRange(t *testing.T) { + const expr = ">= 1.34.3 < 1.35.0" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("parsing %q: %v", expr, err) + } + + for _, reading := range supportedProbeVersions(32) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against %q: %v", expr, reading, evalErr) + } + if satisfied { + t.Fatalf("expected no minor-precision reading to satisfy %q, but %q did;\n"+ + " this test no longer proves the bound-probing loop is required", expr, reading) + } + } + + var admitted bool + for _, reading := range probeReadings(supportedProbeVersions(32), parsed) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against probe %q: %v", expr, reading, evalErr) + } + admitted = admitted || satisfied + } + if !admitted { + t.Errorf("no declared bound of %q satisfies it, so the probe sweep would still\n"+ + " fail a correct floor. See #2402.", expr) + } +} From 62ae16ca5d5fdd58d6228a9e0aa73fafcc1e93be Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Wed, 2 Sep 2026 11:36:05 -0700 Subject: [PATCH 7/7] fix(review): clarify DRA floor guard probes Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 34 +++++++++---------- pkg/recipe/dra_k8s_floor_test.go | 26 +++++++------- .../testdata/catalog_parity_golden.yaml | 34 +++++++++---------- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index b3b35a9a8..a452bc46f 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). a100-aks-ubuntu-training-kubeflow: b8161ed651945695164e296dbda3465d288ea19dcdf8d0ef6e84a1cc92af5cdd -a100-any: 3fda24847acd0e9e1fd1761b250222414d4c3890b827c451ab849923fb69f6a2 -a100-eks-ubuntu-training-kubeflow: efe1f32719432689bd496d826d8b673e588d2d45b8698a141e713f2d29927192 -a100-gke-cos-training-kubeflow: 3835ccd38d6fc1cece73abed272ffa2f88b2de7271c19c0471c92aa6bad157dd -a100-oke-ubuntu-training-kubeflow: 89b090edae168a9a2a8ee3f7fc7330e1417ebcd9ddd22c279ad48e48d4def1c9 -b200-any: 0f1d82bf9d57bdc62b5bca97e4dd51512888a66875499155ef31ab5cbd186dfc +a100-any: 8f6e700544dfd93a8be31a148badb32829924807aa1d13c2db2b80527e94c365 +a100-eks-ubuntu-training-kubeflow: 9d87274b22b1e3a3c35cee9f884ee00aa8bf2d98c75c7961be6d055ca8b2b4c4 +a100-gke-cos-training-kubeflow: 5bfae64e203fdc096e7e5df3d912d45fbf98e79543248c380c73e59562e059f6 +a100-oke-ubuntu-training-kubeflow: 5c46cfdaf69b6f38d2b3b377fc5641b5aa6c84b1030fe97426dc4874ed0a5819 +b200-any: b0a2e105824bd5042396bbdc91529a5986a62d7e0e2be5fd41a4ee31bc3ab24d b200-gke-cos-inference-dynamo: 22f2ee25485c9cba13ce6b566354fee292bf7ed744aa292b9892e2f6f603d3fc b200-gke-cos-training-kubeflow: 0d72edc6fe724e87aaa278a60e31fd852b995b978ed7c7768b1ea1ed8cb71864 bcm-inference: 0de8f37c9026519148af9d1f915e0cb35cef0799d39eb19a9f10736f54170742 -gb200-any: 90d5b0863d470b5e3e46a73e4cfba208e64075f001004684c53f701fe46f203c +gb200-any: 5523fb162d04b85010ea14adcacde848823051530ecb2e0784d27bc5c0e10e1a gb200-eks-ubuntu-inference-dynamo: 750f8596ceaabd823c3de31a619bae6a69314bdac0de353792b7cbc40d2fb14a gb200-eks-ubuntu-training-kubeflow: f9de625240b1836d2d940cef2da8b989bd42d78e73273230ea7bd1b693345f9f gb200-eks-ubuntu-training-slurm: 52c6c3858c36742d53f23ee36c5cb47859ef5b61ff2c9bbc49e932570b617f62 gb200-oke-ubuntu-inference-dynamo: bcbe7c746baa48e9ea619e2bcaf3ed06999b9424418cd316bdc616a7dd41959d gb200-oke-ubuntu-training-kubeflow: 090e678f7ff902fdce9a06afcb5b30aea11a4b90c86979a6ca9059fbd9c14605 -gb300-any: 1c5c2d474532a7221483053967d56dcdca25d748c01370b478c22946a8973fc7 +gb300-any: 357198d4ae6a0e73528cf0ceda23258213e7eddba5f83aa25f133562bdaf65b0 gb300-eks-ubuntu-inference-dynamo: 4f6177e36fa86210e66d627ab92ce44338866796a96609e4461e1cc760046421 gb300-eks-ubuntu-training-kubeflow: 0d6b69f81dd4919d02ebad13324b5a1efe3b3efdb783bf91f5454eca194c57fc h100-aks-ubuntu-inference-dynamo: 7c7dab309ed6f940c49a08914ad55886dbd28a77036ba244f65624ef4cc079f7 h100-aks-ubuntu-training-kubeflow: 5f5d9d468a54571bded9030a968a684a43779161dd0a487521283355bd19560d h100-aks-ubuntu-training-slurm: 8544a761f71c8577952126a19c9a973b0890826d1680c89ec2ff63540d1b1ef6 -h100-any: db2ac4995dd1699023f5f470029250e997fb5b1a742084a8fac86416ddac5632 +h100-any: 9a1e512c6d7fec5f9920ffc2d4b1f036fe59ba80322d4de286d931685bac70d5 h100-bcm-ubuntu-training: cba1fe3491fe878683a4a99fe0af5bf08673b3d4ad21669351d6ce719b313ca3 h100-eks-ubuntu-inference-dynamo: 0339ba1f42f68cc1d683ee8cbc5cbd35806c0524d13251107024f63cd9b13af4 h100-eks-ubuntu-inference-nim: 6b6446a298885359b76f697970c33b80ace95de9cc0a4ab549fe58b8e579ec80 @@ -36,19 +36,19 @@ h100-gke-cos-training-slurm: edbbfa71d6dd096bf32d73412929f5779eafd5e172c99083909 h100-kind-inference-dynamo: 41f81a662fe2563a84de83964e88d17b0f5babba19f55e81540f18d7ccdd0902 h100-kind-training-kubeflow: 1c589e48629c86e5c8b7db3de708fb502bed7a4e48468a635e5da92cb83c3c1d h100-kind-training-slurm: c8f780ca26d66c5191ba97409fdf9f49c5226eb78fee6ed54e06d05490b2215a -h200-any: 40cd70c567e4010fafc2336f0a97441009b005dd7eab3ddf917965ca64bacb5a +h200-any: 8f30849eac9a8489ce02ea195d538f6f76ce8afb0bcf1c8515843869bd304024 h200-eks-inference: dbbeb66fc64382271916197fb24d951d10a5d4972c59aa7931438e5e3ec151f0 h200-eks-training: 14db15267d453edbe3e4121fd1933cae633f2377c3aa6c198e97dc076f8cd464 -l40-any: 069343d9a016841bbfef280ab123040750a553814c07c8f770fd2009f6c2dde8 -l40s-any: dcbf859b1cf9522b7a4d1b12fbb8f01b7dfa2c159d93d446bd4e4527a44b1285 -l40s-oke-inference: 48a74d265f7ba8315a346c2934c2e868deb54f85171ccfb2907537dfce0d500b -l40s-oke-training: 2e7feb830c090f03bdcb6003c4e155bce249030b30ccbef92fc602fe044be667 -monitoring-hpa: 56c321574b7f8a5e94a71d1d8d9f35d8091f7303c3063c2c00eaabe02a6171bb +l40-any: da7bb3a2831930ddb224927d3ebfb5a5dd65d3c9a128c632aad413fb29c0e28b +l40s-any: 8c2f0d3ff1e2f0b5df3869650d3e74ec3808484f52949726bfd5db9c409c8199 +l40s-oke-inference: 69696a6ec37fc930d5fe728d28f979badb9ef9f6a4fa45a39a58b99c981c93ec +l40s-oke-training: 4afd8779e7c706530cd2936fdf0ce7537f86c9cbc019e02c4e05a6787cb31310 +monitoring-hpa: 1e4f15813317d9b472f66cedd8a280a1555a0a7e3142ba9c3dbfb014e578fbcd ocp-inference-nim: 67cb04b6ad946fe56ce423972ec0692ebe8ad66f571f29e94536ae0124eee91c ocp-training: 1082d9d2f5b8dd302d498ab9cb6c936dd6f95996f65e5ce41746ff84aefed04e -rtx-pro-6000-any: ba7efb6b6b1aa683e15e562748bb3f6ca8ac497217e81c643de2268ee7b63966 +rtx-pro-6000-any: ca387e660b770165b828770102eaa0396b8a51f7157d78a2c714c631d5ecf70d rtx-pro-6000-eks-ubuntu-inference-dynamo: f9a9755ecd50b698af1d51fe08f107543c442538cad3b4f1e033ccb9c7128947 rtx-pro-6000-eks-ubuntu-inference-nim: 53c99f5cde980798745ed84acff78b18035ee649e78e945bac8bc1029a32f986 rtx-pro-6000-eks-ubuntu-training-kubeflow: 6c6ad11d9bfb44518abdf37e576c0df08eb432a2d47b01a79b3477ee63e80775 -rtx-pro-6000-lke-ubuntu-inference: 0b62aab1bb38500db14723597997cfc1f9dc0e449b13102179027822ba58df64 -rtx-pro-6000-lke-ubuntu-training: db349f2e58ab99e9374da166419e7011e5411ef9d513d81b2fd0c4ebcc733acb +rtx-pro-6000-lke-ubuntu-inference: ce3fb000f39d710548db9b056613367ce0a8c87253417777a03c594afa10e864 +rtx-pro-6000-lke-ubuntu-training: e1f3fb5a0a31c8169490a589bca3cdc9b4b6b80adb862280cbac45cfaf7040ca diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 0754148da..5baaf2056 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -110,8 +110,9 @@ const floorMajor = 1 // // - ">= v" admits exactly [v, inf) // - "> v" admits (v, inf); requiring v itself to clear the floor is one -// patch conservative and never fails open, since the patch component is -// unbounded (there is no "last" 1.31.x to fall back on) +// unit conservative at v's own precision (one whole minor for a +// minor-precision value) and never fails open, since the patch component +// is unbounded (there is no "last" 1.31.x to fall back on) // - "== v" and a bare exact match admit only v // // "<", "<=", and "!=" place no lower bound at all, so they return false: they @@ -201,12 +202,6 @@ func proveExpressionClearsFloor(expr string, floorMinor int) error { return nil } -// supportedProbeVersions returns readings at or above the floor, used only to -// prove a constraint is not vacuous. An expression satisfied by nothing admits -// no sub-floor cluster and would pass the sub-floor sweep trivially, but it -// also rejects every cluster the catalog claims to support — a typo, not a -// floor. Failing on it keeps the guard closed against expressions it cannot -// show are meaningful. // probeReadings returns the Kubernetes readings used to show that a // declaration admits at least one cluster. Minor-precision probes alone are // not enough: a patch-precision range such as ">= 1.34.3 < 1.35.0" is @@ -227,6 +222,12 @@ func probeReadings(supported []string, parsed *constraints.CompoundConstraint) [ return readings } +// supportedProbeVersions returns readings at or above the floor, used only to +// prove a constraint is not vacuous. An expression satisfied by nothing admits +// no sub-floor cluster and would pass the sub-floor sweep trivially, but it +// also rejects every cluster the catalog claims to support — a typo, not a +// floor. Failing on it keeps the guard closed against expressions it cannot +// show are meaningful. func supportedProbeVersions(floorMinor int) []string { var probes []string for minor := floorMinor; minor <= 60; minor++ { @@ -416,9 +417,10 @@ func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMino } } - t.Errorf("%s (%s) declares K8s.server.version %q, which no Kubernetes release from\n"+ - " 1.%d through 1.60 satisfies. It admits no cluster the catalog supports, so this\n"+ - " guard cannot show it is a floor rather than a typo, and fails closed. See #2402.", + t.Errorf("%s (%s) declares K8s.server.version %q, which none of the probed Kubernetes\n"+ + " readings (1.%d through 1.60 plus the expression's declared bounds) satisfies.\n"+ + " It admits no cluster the catalog supports, so this guard cannot show it is a\n"+ + " floor rather than a typo, and fails closed. See #2402.", decl.file, decl.location, decl.value, floorMinor) } @@ -506,7 +508,7 @@ func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { } // TestProbeSetAdmitsPatchPrecisionRange is the regression control for the -// probe sweep in assertDeclarationIsSatisfiable. The sweep used to try only +// probe sweep used by verifyK8sFloorDeclaration. The sweep used to try only // "1.N.0" readings, which no patch-precision range can satisfy: ">= 1.34.3 // < 1.35.0" clears proveExpressionClearsFloor, yet 1.34.0 is below its lower // bound and 1.35.0 is excluded by its upper one. A correct floor was therefore diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 609eba8c1..e03a684ae 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. a100-aks-ubuntu-training-kubeflow: 47f2c604ad1491346fc4923dacafe6d636fce9fc62b7312e1972f2286e7a1184 -a100-any: 3a9e12086e343324236f49d0d4e4e344f9b9309eb2fb5680b7de9265eda6a73b -a100-eks-ubuntu-training-kubeflow: e9ba3616f5a49924fda0d94f1035932c1d44b286d9d642f5b9823b6b4e98f04b -a100-gke-cos-training-kubeflow: ffc46e4ee6998281d70835c05e0782a8dc531dfc6d35c7df37d0f19c61291a0f -a100-oke-ubuntu-training-kubeflow: c202ac0aef742bcefb7252b3e5c073d94f3fbbbff3a3fc2572565d318b58d46c -b200-any: b1da720ca5ca65a854f86cf643b669dab0bc1ec4642d8b58a1b2e049d7e916c9 +a100-any: 5abf30208c87a34a0f7a8ace6fb04b878de14233bb8691fe7344f1500985eed3 +a100-eks-ubuntu-training-kubeflow: 3cda91f7acec765b159ce307509479e30a9035acff615da06eab3a0b5d6d24c4 +a100-gke-cos-training-kubeflow: 7632830f5d0b5b669f6f4d509233c4314efc29f57152230d376fb2473eca7186 +a100-oke-ubuntu-training-kubeflow: 6619d20510945e05270ede31d1124313d45959f9cdfb189d63e5cd0f9eea7b16 +b200-any: c38ec9744d305fe7de0a5a590d445f028e38b0047786fe80c6a3c349ea95f02b b200-gke-cos-inference-dynamo: 185567cd067fcfe1288b544b5676b8586eb4960332593b13b2af7a2b865c68e6 b200-gke-cos-training-kubeflow: 62eec84f2f80c6f08fbfb2dadbd859e2ba07e787e43b58c5bdcc8360cf4aee71 bcm-inference: 799569869ffff42841c5e758d6da1bc6f474af41a3cca5f74f806597e5be2f10 -gb200-any: bcc0972c4fe7b0a9e325d36f3dd1f1118119ecbe5b1c8a54da97e239029e775c +gb200-any: 0dfa723aa4cc4850bea174f575e2853d4760c56035efb4acffa4d69b3ad2b886 gb200-eks-ubuntu-inference-dynamo: 71e0707b3669b3084fb6cec75a8bd1d6c9493d9148fec8cddc5771be10680c91 gb200-eks-ubuntu-training-kubeflow: 36468d7c502b13e3ae6e441eecd0c6da7901fc2c89dfeac014d5df184cd74700 gb200-eks-ubuntu-training-slurm: c660bbe8b1a97e250cdc7a61e75ea58be49dc96061626715f516bcbf885f2d13 gb200-oke-ubuntu-inference-dynamo: 12da75460cf668b2822557eccc998d815395649d6dfec42bea68e31e5480c621 gb200-oke-ubuntu-training-kubeflow: a2828e176878a36533d39a6ba720418bb6cb3eff34a6de7b2e7db83197248d69 -gb300-any: 1274f6c19c26b29e064885f14b518e18c8469f2361caaac4832813af12fbb779 +gb300-any: 4d22bb7186907f546733386a08efb32c8824552890b2e8d2c90cc717d201fb0d gb300-eks-ubuntu-inference-dynamo: 631e54ea1f2297e33a7884fb173fd60a3244b0d7954ebbdda92e0d51ac4aee88 gb300-eks-ubuntu-training-kubeflow: 33b8adf01bc622198ba9d46731c23efb88ab44991b84c8e37eea4ddf19aaeea9 h100-aks-ubuntu-inference-dynamo: fa53ee5ecde84329429fd43b194a0828eeed354e471a1744b1b073f6fbbc189a h100-aks-ubuntu-training-kubeflow: 3be5d233181d6dea5bf7127f554be827c579bac2fc893defca81621bbcc7ee77 h100-aks-ubuntu-training-slurm: 2d797149b58ce853e7150d908be331c61dfdb477220fae9c15f6ebea22f9fe22 -h100-any: 233b2e31cbd94e7a5c1ac647648dce5f29fe07a4cc1674745be0e8847775f3fc +h100-any: 2896754239e04cf8490a29c1a86b9a7d543cf982eb557c363f1e586668c1fe12 h100-bcm-ubuntu-training: 85c125e8440059a5087e3835551edb0196bf9256f27f84cb4fa094b3517536e9 h100-eks-ubuntu-inference-dynamo: fa55376d388a78d961aa76b7343379f2bdbfad4d827e32af19eb3795962e4ffe h100-eks-ubuntu-inference-nim: 53a96e375f75eed6508e57978e1e07fd05f740688dc4ca5a0e2fd04b90d716f3 @@ -36,19 +36,19 @@ h100-gke-cos-training-slurm: a4aaadffe6e17e6d01f2bdea68e5ea2d76fff2fc364b27ed811 h100-kind-inference-dynamo: 7188e00985d9e89992ceae579cf60b2e373846678d51d69aa78cfec15b748e46 h100-kind-training-kubeflow: 5dd6d4efb2585c41c58dba1f373a5738c19be44618d109d5e31a53cf56482bd8 h100-kind-training-slurm: 355c088589f5c948e025483d71468509addec60544b137653bf4df07f222ccc1 -h200-any: cea4403e292b8a2b3e44562c6770d6ca7366f4e99d409aece4c7e965bb6d2b20 +h200-any: 15d07296d4050a66f0df218d3a41435541f69bb965c643869bc775e41988b4b5 h200-eks-inference: 68b97d51d0cbc982564175114cb8626824e7643ac2fb6babf62073311c7eeb60 h200-eks-training: 6db44e06178a0f74fab4d554406f988fe2e6c119b9523e458eb470ab3ed0cc8d -l40-any: 773378e1c6912ada54256eb6813124fcfe986cbce470aa99b17697f2e84b6591 -l40s-any: 78724a03fa25315bf8a7ef600dd5e869f8e25bc3c1d98500b5356e23a9daeffb -l40s-oke-inference: fbffa7c7d2e73b281b98fc010bb198549f4ccf5358830cefcc88579aad9102d2 -l40s-oke-training: b5af79f5f556627326759a7e8c9d44c35d215956b6d7c61cb959e26473164eed -monitoring-hpa: f45fa89ce40731a71cffe175fae6eb7d84788d8864f05329d2095688a0b6c807 +l40-any: abcbb63d3cd95efac239582f156b5af2c116079e3c48407fd275c889d0b2ad12 +l40s-any: 1bd1432c216e2e9f7b9ef061916e12e667aae1eb98fb3311857e2bc5c6dee44e +l40s-oke-inference: 0e0ef1669b6cb89ff3596a4b4c66f809b607d13ab264b1a02b82ad797c61b2bb +l40s-oke-training: bc8f2c4b08609739102969c4cc610045073834024b57b8121d2a715dbe8768c2 +monitoring-hpa: ca01840738fc06b47eb6713d630304ac2ca40693ed6bfbac9c57f7c8da4294a4 ocp-inference-nim: fe0fc043ef8da714a0b3ff20b589faeba23a1bf7507380c3cf4c3f8ea6c37987 ocp-training: 7c5beb2c2e001147b01e507a3d7e9cb1b2fa8c1cc20d6f86b02ebb271c00e390 -rtx-pro-6000-any: af194881071cad5943d884e2f2a54b930cf1c384ce69e8023356a701bef9ebe1 +rtx-pro-6000-any: 945f6d4425ee52df4491fc74da78184a4d5f04f0204db4950694486993cc9244 rtx-pro-6000-eks-ubuntu-inference-dynamo: f3d27f5f3bd7bf6cedd53a73df14984478d1351fa4422d245b8a3b4b13580e7a rtx-pro-6000-eks-ubuntu-inference-nim: 11f5cfe19a40b3e37ac63fb8c324ab4aa3eb74d05da50e4e0b10328e1426633e rtx-pro-6000-eks-ubuntu-training-kubeflow: d2b5ceb741c24ed111457174755f0fe7e25c72dc8844112f0ce28954a7b7c76b -rtx-pro-6000-lke-ubuntu-inference: 0093d27aea8993870fe0c449eae9b7851660d7cdbec4fd1fe7182fbe0fc33a5e -rtx-pro-6000-lke-ubuntu-training: dfc4de4e93e1cab191c88a57ba969393e4acab024e86fffd57fd82b5c625988b +rtx-pro-6000-lke-ubuntu-inference: 24a807d451b8f14513c47069d0c4a77f7b51b45c5762184328fd12edae7edc7a +rtx-pro-6000-lke-ubuntu-training: b2e12bddf4562fb347616e37e206e975550b43fb0303cf69c1a6bdb068e27daf