diff --git a/demos/query.md b/demos/query.md index 17cb7bddd..f98c23bcd 100644 --- a/demos/query.md +++ b/demos/query.md @@ -110,14 +110,16 @@ aicr query --service gke --accelerator h100 --intent training --os cos \ value: '>= 1.32' ``` -L40S (on OKE) relaxes K8s further (older accelerators run on older clusters): +L40S (on OKE) resolves a different component set; its K8s floor now matches the +catalog-wide minimum (every recipe carries the DRA driver, whose chart requires +1.32): ```shell aicr query --service oke --accelerator l40s --intent training --os ol \ --selector constraints ``` -> K8s minimum drops from `1.32.4` to `1.30`. +> K8s minimum relaxes from `1.32.4` to `1.32`. ### Component set differs by intent diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 0e2dc13f9..f957a7c04 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -1016,7 +1016,7 @@ aicr query --service eks --accelerator gb200 --intent training \ --selector components.nodewright-customizations.values # Watch constraints tighten as you add specificity -# Just "EKS" → 1 constraint (K8s >= 1.28) +# Just "EKS" → 1 constraint (K8s >= 1.32) aicr query --service eks --selector constraints # Add GPU + intent + OS → 4 constraints (K8s >= 1.32.4, Ubuntu 24.04, kernel >= 6.8) aicr query --service eks --accelerator h100 --intent training --os ubuntu \ @@ -1141,7 +1141,7 @@ Supported operators: | Operator | Example | Description | |----------|---------|-------------| -| `>=` | `>= 1.30` | Greater than or equal (version comparison) | +| `>=` | `>= 1.32` | Greater than or equal (version comparison) | | `<=` | `<= 1.33` | Less than or equal (version comparison) | | `>` | `> 1.30` | Greater than (version comparison) | | `<` | `< 2.0` | Less than (version comparison) | diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index 0ee6a04ad..8471b83f7 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -72,7 +72,7 @@ Not every component appears in every recipe. The recipe engine selects component Production GPU leaf recipes (H100, GB200, RTX Pro 6000 on EKS / AKS / GKE / OKE / LKE) enable the NFD Topology Updater. It publishes per-node `NodeResourceTopology` CRDs that describe NUMA zones, GPU-to-NUMA affinity, and NIC-to-NUMA affinity. Runtime consumers (NUMA-aware schedulers, debugging via `kubectl get noderesourcetopologies`) can read these CRDs without further configuration. -The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints on the affected leaves require K8s ≥ 1.30 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. +The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. See the upstream [Topology Updater docs](https://kubernetes-sigs.github.io/node-feature-discovery/stable/usage/nfd-topology-updater.html) for runtime consumer examples. diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index b3b35a9a8..a452bc46f 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). a100-aks-ubuntu-training-kubeflow: b8161ed651945695164e296dbda3465d288ea19dcdf8d0ef6e84a1cc92af5cdd -a100-any: 3fda24847acd0e9e1fd1761b250222414d4c3890b827c451ab849923fb69f6a2 -a100-eks-ubuntu-training-kubeflow: efe1f32719432689bd496d826d8b673e588d2d45b8698a141e713f2d29927192 -a100-gke-cos-training-kubeflow: 3835ccd38d6fc1cece73abed272ffa2f88b2de7271c19c0471c92aa6bad157dd -a100-oke-ubuntu-training-kubeflow: 89b090edae168a9a2a8ee3f7fc7330e1417ebcd9ddd22c279ad48e48d4def1c9 -b200-any: 0f1d82bf9d57bdc62b5bca97e4dd51512888a66875499155ef31ab5cbd186dfc +a100-any: 8f6e700544dfd93a8be31a148badb32829924807aa1d13c2db2b80527e94c365 +a100-eks-ubuntu-training-kubeflow: 9d87274b22b1e3a3c35cee9f884ee00aa8bf2d98c75c7961be6d055ca8b2b4c4 +a100-gke-cos-training-kubeflow: 5bfae64e203fdc096e7e5df3d912d45fbf98e79543248c380c73e59562e059f6 +a100-oke-ubuntu-training-kubeflow: 5c46cfdaf69b6f38d2b3b377fc5641b5aa6c84b1030fe97426dc4874ed0a5819 +b200-any: b0a2e105824bd5042396bbdc91529a5986a62d7e0e2be5fd41a4ee31bc3ab24d b200-gke-cos-inference-dynamo: 22f2ee25485c9cba13ce6b566354fee292bf7ed744aa292b9892e2f6f603d3fc b200-gke-cos-training-kubeflow: 0d72edc6fe724e87aaa278a60e31fd852b995b978ed7c7768b1ea1ed8cb71864 bcm-inference: 0de8f37c9026519148af9d1f915e0cb35cef0799d39eb19a9f10736f54170742 -gb200-any: 90d5b0863d470b5e3e46a73e4cfba208e64075f001004684c53f701fe46f203c +gb200-any: 5523fb162d04b85010ea14adcacde848823051530ecb2e0784d27bc5c0e10e1a gb200-eks-ubuntu-inference-dynamo: 750f8596ceaabd823c3de31a619bae6a69314bdac0de353792b7cbc40d2fb14a gb200-eks-ubuntu-training-kubeflow: f9de625240b1836d2d940cef2da8b989bd42d78e73273230ea7bd1b693345f9f gb200-eks-ubuntu-training-slurm: 52c6c3858c36742d53f23ee36c5cb47859ef5b61ff2c9bbc49e932570b617f62 gb200-oke-ubuntu-inference-dynamo: bcbe7c746baa48e9ea619e2bcaf3ed06999b9424418cd316bdc616a7dd41959d gb200-oke-ubuntu-training-kubeflow: 090e678f7ff902fdce9a06afcb5b30aea11a4b90c86979a6ca9059fbd9c14605 -gb300-any: 1c5c2d474532a7221483053967d56dcdca25d748c01370b478c22946a8973fc7 +gb300-any: 357198d4ae6a0e73528cf0ceda23258213e7eddba5f83aa25f133562bdaf65b0 gb300-eks-ubuntu-inference-dynamo: 4f6177e36fa86210e66d627ab92ce44338866796a96609e4461e1cc760046421 gb300-eks-ubuntu-training-kubeflow: 0d6b69f81dd4919d02ebad13324b5a1efe3b3efdb783bf91f5454eca194c57fc h100-aks-ubuntu-inference-dynamo: 7c7dab309ed6f940c49a08914ad55886dbd28a77036ba244f65624ef4cc079f7 h100-aks-ubuntu-training-kubeflow: 5f5d9d468a54571bded9030a968a684a43779161dd0a487521283355bd19560d h100-aks-ubuntu-training-slurm: 8544a761f71c8577952126a19c9a973b0890826d1680c89ec2ff63540d1b1ef6 -h100-any: db2ac4995dd1699023f5f470029250e997fb5b1a742084a8fac86416ddac5632 +h100-any: 9a1e512c6d7fec5f9920ffc2d4b1f036fe59ba80322d4de286d931685bac70d5 h100-bcm-ubuntu-training: cba1fe3491fe878683a4a99fe0af5bf08673b3d4ad21669351d6ce719b313ca3 h100-eks-ubuntu-inference-dynamo: 0339ba1f42f68cc1d683ee8cbc5cbd35806c0524d13251107024f63cd9b13af4 h100-eks-ubuntu-inference-nim: 6b6446a298885359b76f697970c33b80ace95de9cc0a4ab549fe58b8e579ec80 @@ -36,19 +36,19 @@ h100-gke-cos-training-slurm: edbbfa71d6dd096bf32d73412929f5779eafd5e172c99083909 h100-kind-inference-dynamo: 41f81a662fe2563a84de83964e88d17b0f5babba19f55e81540f18d7ccdd0902 h100-kind-training-kubeflow: 1c589e48629c86e5c8b7db3de708fb502bed7a4e48468a635e5da92cb83c3c1d h100-kind-training-slurm: c8f780ca26d66c5191ba97409fdf9f49c5226eb78fee6ed54e06d05490b2215a -h200-any: 40cd70c567e4010fafc2336f0a97441009b005dd7eab3ddf917965ca64bacb5a +h200-any: 8f30849eac9a8489ce02ea195d538f6f76ce8afb0bcf1c8515843869bd304024 h200-eks-inference: dbbeb66fc64382271916197fb24d951d10a5d4972c59aa7931438e5e3ec151f0 h200-eks-training: 14db15267d453edbe3e4121fd1933cae633f2377c3aa6c198e97dc076f8cd464 -l40-any: 069343d9a016841bbfef280ab123040750a553814c07c8f770fd2009f6c2dde8 -l40s-any: dcbf859b1cf9522b7a4d1b12fbb8f01b7dfa2c159d93d446bd4e4527a44b1285 -l40s-oke-inference: 48a74d265f7ba8315a346c2934c2e868deb54f85171ccfb2907537dfce0d500b -l40s-oke-training: 2e7feb830c090f03bdcb6003c4e155bce249030b30ccbef92fc602fe044be667 -monitoring-hpa: 56c321574b7f8a5e94a71d1d8d9f35d8091f7303c3063c2c00eaabe02a6171bb +l40-any: da7bb3a2831930ddb224927d3ebfb5a5dd65d3c9a128c632aad413fb29c0e28b +l40s-any: 8c2f0d3ff1e2f0b5df3869650d3e74ec3808484f52949726bfd5db9c409c8199 +l40s-oke-inference: 69696a6ec37fc930d5fe728d28f979badb9ef9f6a4fa45a39a58b99c981c93ec +l40s-oke-training: 4afd8779e7c706530cd2936fdf0ce7537f86c9cbc019e02c4e05a6787cb31310 +monitoring-hpa: 1e4f15813317d9b472f66cedd8a280a1555a0a7e3142ba9c3dbfb014e578fbcd ocp-inference-nim: 67cb04b6ad946fe56ce423972ec0692ebe8ad66f571f29e94536ae0124eee91c ocp-training: 1082d9d2f5b8dd302d498ab9cb6c936dd6f95996f65e5ce41746ff84aefed04e -rtx-pro-6000-any: ba7efb6b6b1aa683e15e562748bb3f6ca8ac497217e81c643de2268ee7b63966 +rtx-pro-6000-any: ca387e660b770165b828770102eaa0396b8a51f7157d78a2c714c631d5ecf70d rtx-pro-6000-eks-ubuntu-inference-dynamo: f9a9755ecd50b698af1d51fe08f107543c442538cad3b4f1e033ccb9c7128947 rtx-pro-6000-eks-ubuntu-inference-nim: 53c99f5cde980798745ed84acff78b18035ee649e78e945bac8bc1029a32f986 rtx-pro-6000-eks-ubuntu-training-kubeflow: 6c6ad11d9bfb44518abdf37e576c0df08eb432a2d47b01a79b3477ee63e80775 -rtx-pro-6000-lke-ubuntu-inference: 0b62aab1bb38500db14723597997cfc1f9dc0e449b13102179027822ba58df64 -rtx-pro-6000-lke-ubuntu-training: db349f2e58ab99e9374da166419e7011e5411ef9d513d81b2fd0c4ebcc733acb +rtx-pro-6000-lke-ubuntu-inference: ce3fb000f39d710548db9b056613367ce0a8c87253417777a03c594afa10e864 +rtx-pro-6000-lke-ubuntu-training: e1f3fb5a0a31c8169490a589bca3cdc9b4b6b80adb862280cbac45cfaf7040ca diff --git a/pkg/cli/touched_invariant_test.go b/pkg/cli/touched_invariant_test.go index 6cb84eddc..bae5635d0 100644 --- a/pkg/cli/touched_invariant_test.go +++ b/pkg/cli/touched_invariant_test.go @@ -244,7 +244,7 @@ func TestRecipeCmd_Snapshot_StatedDimensionNotRelaxed(t *testing.T) { } // constraintFailingKindSnapshotYAML fingerprints to service=kind on a -// Kubernetes version below the kind overlay's `K8s.server.version >= 1.25` +// Kubernetes version below the kind overlay's `K8s.server.version >= 1.32` // constraint, so the only overlay covering service=kind is excluded by // constraint evaluation rather than absent from the catalog. const constraintFailingKindSnapshotYAML = `kind: Snapshot diff --git a/pkg/client/v1/relax_test.go b/pkg/client/v1/relax_test.go index 23555c4a7..bd26aae7e 100644 --- a/pkg/client/v1/relax_test.go +++ b/pkg/client/v1/relax_test.go @@ -486,7 +486,7 @@ func kindUbuntuSnapshot() *snapshotter.Snapshot { } // constraintFailingKindSnapshot fingerprints to service=kind on a Kubernetes -// version below the kind overlay's `K8s.server.version >= 1.25` constraint, so +// version below the kind overlay's `K8s.server.version >= 1.32` constraint, so // constraint evaluation excludes the only overlay that covers service=kind. func constraintFailingKindSnapshot() *snapshotter.Snapshot { return &snapshotter.Snapshot{ diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index f2425897e..c1cb49943 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -1272,7 +1272,7 @@ const ( // // This is a render-safe floor, not a support floor. This constant must // stay at or above the strictest kubeVersion any bundled chart declares. - // Do NOT lower it to match the ">= 1.25" recipe floor in + // Do NOT lower it to match the ">= 1.32" recipe floor in // recipes/overlays/base.yaml: recipes are validated against their own // constraints, while mirror discovery raises lower versions to this // value solely for Helm rendering (see mirror.KubeVersionFromConstraints). diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go new file mode 100644 index 000000000..5baaf2056 --- /dev/null +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -0,0 +1,551 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This guard lives in the external recipe_test package so it can import +// pkg/constraints — the production constraint parser and evaluator — which +// itself imports pkg/recipe. An in-package test could not, and would be forced +// to reimplement comparison logic that the shipping evaluator already owns. +package recipe_test + +import ( + "fmt" + "io/fs" + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/NVIDIA/aicr/pkg/constraints" + "github.com/NVIDIA/aicr/pkg/recipe" + "github.com/NVIDIA/aicr/pkg/version" +) + +// k8sServerVersionConstraint is the measurement path whose floor this guard +// audits. +const k8sServerVersionConstraint = "K8s.server.version" + +// auditedDRAChartFloors records, per DRA component, the chart version whose +// kubeVersion was read and the Kubernetes minor it declares. +// +// Both catalog DRA components are enrolled. OCP disables the generic +// nvidia-dra-driver-gpu (recipes/overlays/ocp.yaml sets enabled: false) and +// substitutes nvidia-dra-driver-gpu-ocp, so covering only the generic one would +// leave the OCP chain unguarded. +// +// Re-audit procedure when a pin moves: read `kubeVersion` from the chart at the +// new version and update the entry. TestDRAChartFloorAuditIsCurrent fails until +// you do, which is what couples this guard to the registry rather than letting +// it sit green at a stale floor. +var auditedDRAChartFloors = map[string]struct { + version string + minor int +}{ + // oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu + // kubeVersion: ">=1.32.0-0" + "nvidia-dra-driver-gpu": {version: "0.4.1", minor: 32}, + "nvidia-dra-driver-gpu-ocp": {version: "0.4.1", minor: 32}, +} + +// draChartKubeVersionMinor is the highest audited floor across the enrolled DRA +// components — the value every recipe must clear, since each recipe carries one +// of them. +func draChartKubeVersionMinor() int { + highest := 0 + for _, a := range auditedDRAChartFloors { + if a.minor > highest { + highest = a.minor + } + } + return highest +} + +// TestDRAChartFloorAuditIsCurrent fails when a DRA chart pin in registry.yaml +// moves away from the version whose kubeVersion was audited, so a chart bump +// cannot silently leave the floor guard asserting a stale minor. +func TestDRAChartFloorAuditIsCurrent(t *testing.T) { + t.Parallel() + + registry, err := recipe.GetComponentRegistry() + if err != nil { + t.Fatalf("GetComponentRegistry: %v", err) + } + + for name, audited := range auditedDRAChartFloors { + cfg := registry.Get(name) + if cfg == nil { + t.Errorf("audited DRA component %q is not in the registry; remove it from "+ + "auditedDRAChartFloors or restore the component", name) + continue + } + if cfg.Helm.DefaultVersion != audited.version { + t.Errorf("%s is pinned at %q but its kubeVersion was audited at %q.\n"+ + " Read `kubeVersion` from the chart at %s, update auditedDRAChartFloors,\n"+ + " and raise the affected overlay floors if it moved. See #2402.", + name, cfg.Helm.DefaultVersion, audited.version, cfg.Helm.DefaultVersion) + } + } +} + +// floorMajor is the Kubernetes major version the DRA chart floors sit on. The +// audited kubeVersion is ">=1.32.0-0", so every bound is compared as +// (major, minor) against (floorMajor, floorMinor). +const floorMajor = 1 + +// termClearsFloor reports whether a single parsed term, on its own, confines +// every version that satisfies it to at or above Kubernetes +// floorMajor.floorMinor.0. +// +// Only the lower-bounding operators can do that: +// +// - ">= v" admits exactly [v, inf) +// - "> v" admits (v, inf); requiring v itself to clear the floor is one +// unit conservative at v's own precision (one whole minor for a +// minor-precision value) and never fails open, since the patch component +// is unbounded (there is no "last" 1.31.x to fall back on) +// - "== v" and a bare exact match admit only v +// +// "<", "<=", and "!=" place no lower bound at all, so they return false: they +// can narrow an alternative but can never be what lifts it above the floor. +// +// A value the shipping version parser cannot read, or one written with less +// than major.minor precision (">= 1"), returns an error. Comparing "1" against +// "1.32.0" at min-precision would report equal and wave a floorless expression +// through, so the guard refuses to reason about it instead. +func termClearsFloor(pc constraints.ParsedConstraint, floorMinor int) (bool, error) { + switch pc.Operator { + case constraints.OperatorLT, constraints.OperatorLTE, constraints.OperatorNE: + return false, nil + case constraints.OperatorGTE, constraints.OperatorGT, constraints.OperatorEQ, constraints.OperatorExact: + default: + return false, fmt.Errorf("unknown operator %q; the guard cannot prove a lower bound for it", pc.Operator) + } + + v, err := version.ParseVersion(pc.Value) + if err != nil { + return false, fmt.Errorf("value %q is not a version the shipping parser can read: %w", pc.Value, err) + } + if v.Precision < 2 { + return false, fmt.Errorf("value %q has only major precision; a floor must name at least major.minor", pc.Value) + } + if v.Major > floorMajor { + return true, nil + } + return v.Major == floorMajor && v.Minor >= floorMinor, nil +} + +// alternativeString renders one AND group for error messages. +func alternativeString(group []constraints.ParsedConstraint) string { + terms := make([]string, 0, len(group)) + for i := range group { + terms = append(terms, group[i].String()) + } + return strings.Join(terms, " ") +} + +// proveExpressionClearsFloor proves, symbolically, that no Kubernetes cluster +// below floorMajor.floorMinor.0 can satisfy expr. It returns nil only when the +// proof succeeds, and a describing error otherwise — including for any +// expression it cannot reason about, so the guard fails closed. +// +// Why symbolic and not by probing readings: the production grammar admits +// arbitrary OR-of-AND range expressions, so no finite list of probe versions +// covers it. ">= 1.32 || > 1.31.0 < 1.31.2" is a supported shape that a probe +// sweep over 1.N / 1.N.0 / 1.N.99 misses entirely while the production +// evaluator happily accepts a 1.31.1 cluster that Helm's ">=1.32.0-0" rejects. +// +// The proof: an AND group's satisfying set is the intersection of its terms, so +// the group clears the floor as soon as ANY ONE of its terms does — a single +// ">= 1.32" makes the whole group safe no matter what the others say. A +// compound expression's satisfying set is the union of its groups, so EVERY +// group must clear the floor; one loose alternative admits a sub-floor cluster +// regardless of how strict its siblings are. +func proveExpressionClearsFloor(expr string, floorMinor int) error { + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + return fmt.Errorf("the shipping constraint parser rejects it: %w", err) + } + if len(parsed.Alternatives) == 0 { + return fmt.Errorf("it parsed to zero OR alternatives, so nothing bounds it") + } + + for i, group := range parsed.Alternatives { + if len(group) == 0 { + return fmt.Errorf("OR alternative %d has no terms, so nothing bounds it", i+1) + } + cleared := false + for j := range group { + ok, termErr := termClearsFloor(group[j], floorMinor) + if termErr != nil { + return fmt.Errorf("OR alternative %d (%q): %w", i+1, alternativeString(group), termErr) + } + if ok { + cleared = true + } + } + if !cleared { + return fmt.Errorf("OR alternative %d (%q) carries no lower bound at or above %d.%d.0, "+ + "so at least one cluster below the chart floor satisfies it", + i+1, alternativeString(group), floorMajor, floorMinor) + } + } + return nil +} + +// probeReadings returns the Kubernetes readings used to show that a +// declaration admits at least one cluster. Minor-precision probes alone are +// not enough: a patch-precision range such as ">= 1.34.3 < 1.35.0" is +// satisfied by no "1.N.0" reading, so a correct floor would be reported as +// admitting nothing. The declared bounds are therefore probed as well. +// +// The result is a fresh slice. Appending to `supported` in place would write +// into a backing array shared by every declaration under test whenever it has +// spare capacity. +func probeReadings(supported []string, parsed *constraints.CompoundConstraint) []string { + readings := make([]string, 0, len(supported)) + readings = append(readings, supported...) + for _, group := range parsed.Alternatives { + for i := range group { + readings = append(readings, group[i].Value) + } + } + return readings +} + +// supportedProbeVersions returns readings at or above the floor, used only to +// prove a constraint is not vacuous. An expression satisfied by nothing admits +// no sub-floor cluster and would pass the sub-floor sweep trivially, but it +// also rejects every cluster the catalog claims to support — a typo, not a +// floor. Failing on it keeps the guard closed against expressions it cannot +// show are meaningful. +func supportedProbeVersions(floorMinor int) []string { + var probes []string + for minor := floorMinor; minor <= 60; minor++ { + probes = append(probes, fmt.Sprintf("1.%d.0", minor)) + } + return probes +} + +// k8sFloorDeclaration is one typed K8s.server.version constraint located in the +// catalog, with the structural path it was found at for error reporting. +type k8sFloorDeclaration struct { + file string + location string + value string +} + +// collectK8sFloorDeclarations decodes one recipe metadata document and returns +// every K8s.server.version constraint it declares, from every field that can +// carry one: spec.constraints, each validation phase, and each profile value's +// constraints and readinessConstraints. +// +// Decoding to the typed form is what closes the YAML-layout hole: a mapping +// written "value:" before "name:" is the same Constraint after unmarshalling, +// so key order, quoting style, comments, and indentation are all invisible to +// this guard by construction rather than by widening a regex. +func collectK8sFloorDeclarations(file string, raw []byte) ([]k8sFloorDeclaration, error) { + var metadata recipe.RecipeMetadata + if err := yaml.Unmarshal(raw, &metadata); err != nil { + return nil, err + } + + var found []k8sFloorDeclaration + collect := func(location string, cs []recipe.Constraint) { + for _, c := range cs { + if c.Name != k8sServerVersionConstraint { + continue + } + found = append(found, k8sFloorDeclaration{file: file, location: location, value: c.Value}) + } + } + + spec := metadata.Spec + collect("spec.constraints", spec.Constraints) + + if v := spec.Validation; v != nil { + for _, phase := range []struct { + name string + p *recipe.ValidationPhase + }{ + {"readiness", v.Readiness}, + {"deployment", v.Deployment}, + {"performance", v.Performance}, + {"conformance", v.Conformance}, + } { + if phase.p != nil { + collect("spec.validation."+phase.name+".constraints", phase.p.Constraints) + } + } + } + + if p := spec.Profile; p != nil { + for valueName, pv := range p.Values { + collect(fmt.Sprintf("spec.profile.values[%s].constraints", valueName), pv.Constraints) + collect(fmt.Sprintf("spec.profile.values[%s].readinessConstraints", valueName), pv.ReadinessConstraints) + } + } + + return found, nil +} + +// TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a +// Kubernetes floor that admits a cluster below the DRA chart's own kubeVersion. +// +// Every recipe carries a DRA driver: base.yaml declares nvidia-dra-driver-gpu, +// and OCP disables that one and substitutes nvidia-dra-driver-gpu-ocp. Both +// resolve to the same upstream chart and the same kubeVersion, so the floor +// applies catalog-wide either way. +// +// Why every declaration and not just base.yaml: constraints merge by name with +// the LATER overlay winning and no max comparison (see RecipeMetadataSpec.Merge +// in metadata.go; validation-phase constraints merge the same way in +// mergeValidationPhase in validation.go). A leaf declaring ">= 1.30" silently +// overwrites a higher floor inherited from base, so raising base alone would +// not hold. This is the same +// last-wins hazard documented for driver floors in #2438. +// +// recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason +// before the rest of the catalog was reconciled; its comment records the +// diagnosis. +// +// How it checks: each declaration is decoded typed, parsed with the shipping +// parser (constraints.ParseCompoundConstraint), and then PROVEN — symbolically, +// over the parsed OR-of-AND structure — to carry a lower bound at or above the +// chart floor on every alternative. See proveExpressionClearsFloor. +// +// Sampling the evaluator with a list of sub-floor readings was tried and is not +// sufficient: the grammar admits arbitrary ranges, and ">= 1.32 || > 1.31.0 +// < 1.31.2" slips past any fixed probe list while admitting a 1.31.1 cluster. +// The guard fails closed on any expression the parser rejects, on any operator +// or value it cannot reason about, and on any expression no supported reading +// satisfies. +func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { + t.Parallel() + + floorMinor := draChartKubeVersionMinor() + supported := supportedProbeVersions(floorMinor) + + efs := recipe.GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + if !strings.Contains(path, "overlays/") && !strings.Contains(path, "mixins/") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + decls, decodeErr := collectK8sFloorDeclarations(path, raw) + if decodeErr != nil { + t.Errorf("%s: could not decode recipe metadata: %v", path, decodeErr) + return nil + } + + for _, decl := range decls { + checked++ + verifyK8sFloorDeclaration(t, decl, floorMinor, supported) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if nothing matched, the guard is inert. + if checked == 0 { + t.Fatal("no K8s.server.version floors found in overlays or mixins — this guard " + + "is vacuous. Either the constraint name changed or the embed pattern no " + + "longer covers the recipe tree.") + } + t.Logf("verified %d K8s.server.version floor(s) clear the DRA chart floor", checked) +} + +// verifyK8sFloorDeclaration checks one declaration against the DRA chart floor +// using the production parser and evaluator. +func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, supported []string) { + t.Helper() + + if err := proveExpressionClearsFloor(decl.value, floorMinor); err != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which this guard cannot prove\n"+ + " clears the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\":\n"+ + " %v\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so a recipe that admits a lower cluster\n"+ + " validates clean and then fails at `helm install`. Raise it to \">= 1.%d\",\n"+ + " or, if the expression is genuinely safe in a form the guard cannot yet\n"+ + " prove, extend proveExpressionClearsFloor rather than loosening it.\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + decl.file, decl.location, decl.value, floorMinor, err, floorMinor) + return + } + + parsed, err := constraints.ParseCompoundConstraint(decl.value) + if err != nil { + t.Errorf("%s (%s): %v", decl.file, decl.location, err) + return + } + + for _, reading := range probeReadings(supported, parsed) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ + " could not evaluate against the Kubernetes reading %q: %v. See #2402.", + decl.file, decl.location, decl.value, reading, evalErr) + return + } + if satisfied { + return + } + } + + t.Errorf("%s (%s) declares K8s.server.version %q, which none of the probed Kubernetes\n"+ + " readings (1.%d through 1.60 plus the expression's declared bounds) satisfies.\n"+ + " It admits no cluster the catalog supports, so this guard cannot show it is a\n"+ + " floor rather than a typo, and fails closed. See #2402.", + decl.file, decl.location, decl.value, floorMinor) +} + +// TestProveExpressionClearsFloor pins the prover's behavior on the shapes the +// catalog guard has to withstand. Every "must fail" row is a permanent +// regression control: each one is an expression a real author could write that +// the production evaluator accepts for a sub-floor cluster. +// +// The last row is the shape that defeated the previous probe-sampling guard: a +// safe first alternative followed by a narrow sub-floor range. It is kept here +// permanently so no future rewrite can reintroduce sampling and stay green. +func TestProveExpressionClearsFloor(t *testing.T) { + t.Parallel() + + const floorMinor = 32 + + tests := []struct { + name string + expr string + wantErr bool + }{ + {"simple floor at the chart minor", ">= 1.32", false}, + {"simple floor with patch", ">= 1.32.4", false}, + {"floor above the chart minor", ">= 1.33.0", false}, + {"major above the floor", ">= 2.0", false}, + {"range whose lower bound clears the floor", ">= 1.32.4 < 1.35.0", false}, + {"every alternative clears the floor", ">= 1.34.3-gke.1318000 < 1.35.0 || >= 1.35.0-gke.2745000", false}, + {"upper-bounded term does not lift a cleared group", ">= 1.32 < 1.33", false}, + + {"floor below the chart minor", ">= 1.30", true}, + {"simple compound with a low alternative", ">= 1.32 || >= 1.29", true}, + {"exact pin below the chart minor", "== 1.30", true}, + {"bare exact pin below the chart minor", "1.30", true}, + {"greater-than below the chart minor", "> 1.31", true}, + {"only an upper bound", "< 1.40", true}, + {"only a not-equal", "!= 1.30", true}, + {"major-only precision", ">= 1", true}, + {"non-version value", ">= stable", true}, + {"empty expression", "", true}, + {"empty OR clause", ">= 1.32 ||", true}, + // The control: accepted by the production evaluator for a 1.31.1 + // cluster, which Helm's ">=1.32.0-0" rejects. A probe sweep over + // 1.N / 1.N.0 / 1.N.99 / v1.N.0 misses it entirely. + {"narrow sub-floor range hidden behind a safe alternative", ">= 1.32 || > 1.31.0 < 1.31.2", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := proveExpressionClearsFloor(tt.expr, floorMinor) + if (err != nil) != tt.wantErr { + t.Fatalf("proveExpressionClearsFloor(%q, %d) error = %v, wantErr %v", + tt.expr, floorMinor, err, tt.wantErr) + } + }) + } +} + +// TestProveExpressionRejectsWhatTheEvaluatorAdmits is the adversarial control +// for the row above: it proves independently that the production evaluator +// really does accept a sub-floor cluster for that expression, so the "must +// fail" verdict is grounded in behavior rather than in the prover's own +// opinion. Without this, a prover bug that rejected everything would still +// make the table green. +func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { + t.Parallel() + + const expr = ">= 1.32 || > 1.31.0 < 1.31.2" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("ParseCompoundConstraint(%q): %v", expr, err) + } + satisfied, err := parsed.Evaluate("1.31.1") + if err != nil { + t.Fatalf("Evaluate(1.31.1): %v", err) + } + if !satisfied { + t.Fatalf("expected the production evaluator to accept 1.31.1 for %q; if this "+ + "changed, the control in TestProveExpressionClearsFloor needs rebasing", expr) + } + if err := proveExpressionClearsFloor(expr, 32); err == nil { + t.Fatalf("prover accepted %q even though the evaluator admits a 1.31.1 cluster", expr) + } +} + +// TestProbeSetAdmitsPatchPrecisionRange is the regression control for the +// probe sweep used by verifyK8sFloorDeclaration. The sweep used to try only +// "1.N.0" readings, which no patch-precision range can satisfy: ">= 1.34.3 +// < 1.35.0" clears proveExpressionClearsFloor, yet 1.34.0 is below its lower +// bound and 1.35.0 is excluded by its upper one. A correct floor was therefore +// reported as admitting no supported release. +// +// Reverting probeReadings to return `supported` unchanged makes this test +// fail, which is the point: the catalog currently declares no patch-precision +// range, so the defect is latent and nothing else would catch a regression. +func TestProbeSetAdmitsPatchPrecisionRange(t *testing.T) { + const expr = ">= 1.34.3 < 1.35.0" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("parsing %q: %v", expr, err) + } + + for _, reading := range supportedProbeVersions(32) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against %q: %v", expr, reading, evalErr) + } + if satisfied { + t.Fatalf("expected no minor-precision reading to satisfy %q, but %q did;\n"+ + " this test no longer proves the bound-probing loop is required", expr, reading) + } + } + + var admitted bool + for _, reading := range probeReadings(supportedProbeVersions(32), parsed) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against probe %q: %v", expr, reading, evalErr) + } + admitted = admitted || satisfied + } + if !admitted { + t.Errorf("no declared bound of %q satisfies it, so the probe sweep would still\n"+ + " fail a correct floor. See #2402.", expr) + } +} diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 609eba8c1..e03a684ae 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. a100-aks-ubuntu-training-kubeflow: 47f2c604ad1491346fc4923dacafe6d636fce9fc62b7312e1972f2286e7a1184 -a100-any: 3a9e12086e343324236f49d0d4e4e344f9b9309eb2fb5680b7de9265eda6a73b -a100-eks-ubuntu-training-kubeflow: e9ba3616f5a49924fda0d94f1035932c1d44b286d9d642f5b9823b6b4e98f04b -a100-gke-cos-training-kubeflow: ffc46e4ee6998281d70835c05e0782a8dc531dfc6d35c7df37d0f19c61291a0f -a100-oke-ubuntu-training-kubeflow: c202ac0aef742bcefb7252b3e5c073d94f3fbbbff3a3fc2572565d318b58d46c -b200-any: b1da720ca5ca65a854f86cf643b669dab0bc1ec4642d8b58a1b2e049d7e916c9 +a100-any: 5abf30208c87a34a0f7a8ace6fb04b878de14233bb8691fe7344f1500985eed3 +a100-eks-ubuntu-training-kubeflow: 3cda91f7acec765b159ce307509479e30a9035acff615da06eab3a0b5d6d24c4 +a100-gke-cos-training-kubeflow: 7632830f5d0b5b669f6f4d509233c4314efc29f57152230d376fb2473eca7186 +a100-oke-ubuntu-training-kubeflow: 6619d20510945e05270ede31d1124313d45959f9cdfb189d63e5cd0f9eea7b16 +b200-any: c38ec9744d305fe7de0a5a590d445f028e38b0047786fe80c6a3c349ea95f02b b200-gke-cos-inference-dynamo: 185567cd067fcfe1288b544b5676b8586eb4960332593b13b2af7a2b865c68e6 b200-gke-cos-training-kubeflow: 62eec84f2f80c6f08fbfb2dadbd859e2ba07e787e43b58c5bdcc8360cf4aee71 bcm-inference: 799569869ffff42841c5e758d6da1bc6f474af41a3cca5f74f806597e5be2f10 -gb200-any: bcc0972c4fe7b0a9e325d36f3dd1f1118119ecbe5b1c8a54da97e239029e775c +gb200-any: 0dfa723aa4cc4850bea174f575e2853d4760c56035efb4acffa4d69b3ad2b886 gb200-eks-ubuntu-inference-dynamo: 71e0707b3669b3084fb6cec75a8bd1d6c9493d9148fec8cddc5771be10680c91 gb200-eks-ubuntu-training-kubeflow: 36468d7c502b13e3ae6e441eecd0c6da7901fc2c89dfeac014d5df184cd74700 gb200-eks-ubuntu-training-slurm: c660bbe8b1a97e250cdc7a61e75ea58be49dc96061626715f516bcbf885f2d13 gb200-oke-ubuntu-inference-dynamo: 12da75460cf668b2822557eccc998d815395649d6dfec42bea68e31e5480c621 gb200-oke-ubuntu-training-kubeflow: a2828e176878a36533d39a6ba720418bb6cb3eff34a6de7b2e7db83197248d69 -gb300-any: 1274f6c19c26b29e064885f14b518e18c8469f2361caaac4832813af12fbb779 +gb300-any: 4d22bb7186907f546733386a08efb32c8824552890b2e8d2c90cc717d201fb0d gb300-eks-ubuntu-inference-dynamo: 631e54ea1f2297e33a7884fb173fd60a3244b0d7954ebbdda92e0d51ac4aee88 gb300-eks-ubuntu-training-kubeflow: 33b8adf01bc622198ba9d46731c23efb88ab44991b84c8e37eea4ddf19aaeea9 h100-aks-ubuntu-inference-dynamo: fa53ee5ecde84329429fd43b194a0828eeed354e471a1744b1b073f6fbbc189a h100-aks-ubuntu-training-kubeflow: 3be5d233181d6dea5bf7127f554be827c579bac2fc893defca81621bbcc7ee77 h100-aks-ubuntu-training-slurm: 2d797149b58ce853e7150d908be331c61dfdb477220fae9c15f6ebea22f9fe22 -h100-any: 233b2e31cbd94e7a5c1ac647648dce5f29fe07a4cc1674745be0e8847775f3fc +h100-any: 2896754239e04cf8490a29c1a86b9a7d543cf982eb557c363f1e586668c1fe12 h100-bcm-ubuntu-training: 85c125e8440059a5087e3835551edb0196bf9256f27f84cb4fa094b3517536e9 h100-eks-ubuntu-inference-dynamo: fa55376d388a78d961aa76b7343379f2bdbfad4d827e32af19eb3795962e4ffe h100-eks-ubuntu-inference-nim: 53a96e375f75eed6508e57978e1e07fd05f740688dc4ca5a0e2fd04b90d716f3 @@ -36,19 +36,19 @@ h100-gke-cos-training-slurm: a4aaadffe6e17e6d01f2bdea68e5ea2d76fff2fc364b27ed811 h100-kind-inference-dynamo: 7188e00985d9e89992ceae579cf60b2e373846678d51d69aa78cfec15b748e46 h100-kind-training-kubeflow: 5dd6d4efb2585c41c58dba1f373a5738c19be44618d109d5e31a53cf56482bd8 h100-kind-training-slurm: 355c088589f5c948e025483d71468509addec60544b137653bf4df07f222ccc1 -h200-any: cea4403e292b8a2b3e44562c6770d6ca7366f4e99d409aece4c7e965bb6d2b20 +h200-any: 15d07296d4050a66f0df218d3a41435541f69bb965c643869bc775e41988b4b5 h200-eks-inference: 68b97d51d0cbc982564175114cb8626824e7643ac2fb6babf62073311c7eeb60 h200-eks-training: 6db44e06178a0f74fab4d554406f988fe2e6c119b9523e458eb470ab3ed0cc8d -l40-any: 773378e1c6912ada54256eb6813124fcfe986cbce470aa99b17697f2e84b6591 -l40s-any: 78724a03fa25315bf8a7ef600dd5e869f8e25bc3c1d98500b5356e23a9daeffb -l40s-oke-inference: fbffa7c7d2e73b281b98fc010bb198549f4ccf5358830cefcc88579aad9102d2 -l40s-oke-training: b5af79f5f556627326759a7e8c9d44c35d215956b6d7c61cb959e26473164eed -monitoring-hpa: f45fa89ce40731a71cffe175fae6eb7d84788d8864f05329d2095688a0b6c807 +l40-any: abcbb63d3cd95efac239582f156b5af2c116079e3c48407fd275c889d0b2ad12 +l40s-any: 1bd1432c216e2e9f7b9ef061916e12e667aae1eb98fb3311857e2bc5c6dee44e +l40s-oke-inference: 0e0ef1669b6cb89ff3596a4b4c66f809b607d13ab264b1a02b82ad797c61b2bb +l40s-oke-training: bc8f2c4b08609739102969c4cc610045073834024b57b8121d2a715dbe8768c2 +monitoring-hpa: ca01840738fc06b47eb6713d630304ac2ca40693ed6bfbac9c57f7c8da4294a4 ocp-inference-nim: fe0fc043ef8da714a0b3ff20b589faeba23a1bf7507380c3cf4c3f8ea6c37987 ocp-training: 7c5beb2c2e001147b01e507a3d7e9cb1b2fa8c1cc20d6f86b02ebb271c00e390 -rtx-pro-6000-any: af194881071cad5943d884e2f2a54b930cf1c384ce69e8023356a701bef9ebe1 +rtx-pro-6000-any: 945f6d4425ee52df4491fc74da78184a4d5f04f0204db4950694486993cc9244 rtx-pro-6000-eks-ubuntu-inference-dynamo: f3d27f5f3bd7bf6cedd53a73df14984478d1351fa4422d245b8a3b4b13580e7a rtx-pro-6000-eks-ubuntu-inference-nim: 11f5cfe19a40b3e37ac63fb8c324ab4aa3eb74d05da50e4e0b10328e1426633e rtx-pro-6000-eks-ubuntu-training-kubeflow: d2b5ceb741c24ed111457174755f0fe7e25c72dc8844112f0ce28954a7b7c76b -rtx-pro-6000-lke-ubuntu-inference: 0093d27aea8993870fe0c449eae9b7851660d7cdbec4fd1fe7182fbe0fc33a5e -rtx-pro-6000-lke-ubuntu-training: dfc4de4e93e1cab191c88a57ba969393e4acab024e86fffd57fd82b5c625988b +rtx-pro-6000-lke-ubuntu-inference: 24a807d451b8f14513c47069d0c4a77f7b51b45c5762184328fd12edae7edc7a +rtx-pro-6000-lke-ubuntu-training: b2e12bddf4562fb347616e37e206e975550b43fb0303cf69c1a6bdb068e27daf diff --git a/recipes/overlays/a100-eks-training.yaml b/recipes/overlays/a100-eks-training.yaml index c2e2c1159..f9fc790c7 100644 --- a/recipes/overlays/a100-eks-training.yaml +++ b/recipes/overlays/a100-eks-training.yaml @@ -32,7 +32,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from eks-training). diff --git a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml index 95cdd1eed..368c2afa9 100644 --- a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-eks-ubuntu-training.yaml b/recipes/overlays/a100-eks-ubuntu-training.yaml index d9a461d80..c6625021e 100644 --- a/recipes/overlays/a100-eks-ubuntu-training.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml index 6eb940a08..0df5add20 100644 --- a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml +++ b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml @@ -32,7 +32,7 @@ spec: # Constraints for A100 on GKE COS for Kubeflow training workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" # Kubeflow Training Operator for TrainJob support. # Declared inline (not via the platform-kubeflow mixin) to match the GKE COS diff --git a/recipes/overlays/a100-gke-cos-training.yaml b/recipes/overlays/a100-gke-cos-training.yaml index 0f8f716a4..c3c6c69c9 100644 --- a/recipes/overlays/a100-gke-cos-training.yaml +++ b/recipes/overlays/a100-gke-cos-training.yaml @@ -28,11 +28,12 @@ spec: intent: training # Specific constraints for A100 on GKE COS training workloads. - # A100 has no IMEX/NVLink ComputeDomain requirement, so the recipe keeps - # the GKE COS training baseline rather than the H100 1.32 floor. + # A100 has no IMEX/NVLink ComputeDomain requirement, so this floor is not + # driven by a ComputeDomain need as the H100 recipes' is; it simply carries + # the catalog-wide 1.32 baseline the DRA driver chart's kubeVersion sets. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from gke-cos-training). diff --git a/recipes/overlays/a100-oke-training.yaml b/recipes/overlays/a100-oke-training.yaml index e929c2ce5..4b6a8ddab 100644 --- a/recipes/overlays/a100-oke-training.yaml +++ b/recipes/overlays/a100-oke-training.yaml @@ -34,7 +34,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml index ca77b4cda..d7ed0c31c 100644 --- a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-oke-ubuntu-training.yaml b/recipes/overlays/a100-oke-ubuntu-training.yaml index dcb5f5867..bb298d2da 100644 --- a/recipes/overlays/a100-oke-ubuntu-training.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/base.yaml b/recipes/overlays/base.yaml index 473b2629a..67fc3dbb8 100644 --- a/recipes/overlays/base.yaml +++ b/recipes/overlays/base.yaml @@ -22,7 +22,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" componentRefs: - name: nfd diff --git a/recipes/overlays/eks-inference.yaml b/recipes/overlays/eks-inference.yaml index b5fb62a03..c85e8e890 100644 --- a/recipes/overlays/eks-inference.yaml +++ b/recipes/overlays/eks-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/eks-training.yaml b/recipes/overlays/eks-training.yaml index fe436bcf7..d7014a611 100644 --- a/recipes/overlays/eks-training.yaml +++ b/recipes/overlays/eks-training.yaml @@ -29,7 +29,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the training-optimized GPU Operator values diff --git a/recipes/overlays/eks.yaml b/recipes/overlays/eks.yaml index 3b9459d06..195186014 100644 --- a/recipes/overlays/eks.yaml +++ b/recipes/overlays/eks.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # EKS-specific components componentRefs: diff --git a/recipes/overlays/gke-cos-inference.yaml b/recipes/overlays/gke-cos-inference.yaml index d659c931f..1240b9304 100644 --- a/recipes/overlays/gke-cos-inference.yaml +++ b/recipes/overlays/gke-cos-inference.yaml @@ -33,6 +33,6 @@ spec: # Inference specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/gke-cos-training.yaml b/recipes/overlays/gke-cos-training.yaml index 3cea0c4bd..e0e1fb19d 100644 --- a/recipes/overlays/gke-cos-training.yaml +++ b/recipes/overlays/gke-cos-training.yaml @@ -29,7 +29,7 @@ spec: # Training specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the GKE-COS training-optimized GPU Operator values diff --git a/recipes/overlays/gke-cos.yaml b/recipes/overlays/gke-cos.yaml index 29e42499c..2dab8f638 100644 --- a/recipes/overlays/gke-cos.yaml +++ b/recipes/overlays/gke-cos.yaml @@ -28,7 +28,7 @@ spec: # GKE-specific constraints constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # The GPU stack shape is a configuration profile (ADR-015, issue # #1761): one qualified GKE cluster state per value — how the driver is diff --git a/recipes/overlays/kind-inference.yaml b/recipes/overlays/kind-inference.yaml index 0ebbf84b1..3467d5b17 100644 --- a/recipes/overlays/kind-inference.yaml +++ b/recipes/overlays/kind-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/kind.yaml b/recipes/overlays/kind.yaml index e8a20d5a7..08afd5c5b 100644 --- a/recipes/overlays/kind.yaml +++ b/recipes/overlays/kind.yaml @@ -27,7 +27,7 @@ spec: # Kind-specific constraints constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" # Kind-specific component overrides componentRefs: diff --git a/recipes/overlays/l40s-oke-inference.yaml b/recipes/overlays/l40s-oke-inference.yaml index dc9830e97..710394771 100644 --- a/recipes/overlays/l40s-oke-inference.yaml +++ b/recipes/overlays/l40s-oke-inference.yaml @@ -29,7 +29,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/l40s-oke-training.yaml b/recipes/overlays/l40s-oke-training.yaml index 6314819d2..957e43b5a 100644 --- a/recipes/overlays/l40s-oke-training.yaml +++ b/recipes/overlays/l40s-oke-training.yaml @@ -32,7 +32,7 @@ spec: # requirement, so the recipe keeps the OKE training baseline K8s floor. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # L40S-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/lke-inference.yaml b/recipes/overlays/lke-inference.yaml index d4b34a107..54d8b0e97 100644 --- a/recipes/overlays/lke-inference.yaml +++ b/recipes/overlays/lke-inference.yaml @@ -32,6 +32,6 @@ spec: # Inference specific constraints for LKE workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke-training.yaml b/recipes/overlays/lke-training.yaml index 7e4812c8c..0e7ee9f45 100644 --- a/recipes/overlays/lke-training.yaml +++ b/recipes/overlays/lke-training.yaml @@ -29,6 +29,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke.yaml b/recipes/overlays/lke.yaml index da811a074..795883ace 100644 --- a/recipes/overlays/lke.yaml +++ b/recipes/overlays/lke.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" # LKE-specific components componentRefs: diff --git a/recipes/overlays/oke-ol-inference.yaml b/recipes/overlays/oke-ol-inference.yaml index 24b212de5..c46c6295c 100644 --- a/recipes/overlays/oke-ol-inference.yaml +++ b/recipes/overlays/oke-ol-inference.yaml @@ -30,6 +30,6 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/oke-ol-training.yaml b/recipes/overlays/oke-ol-training.yaml index 0d26f3dd5..097819f05 100644 --- a/recipes/overlays/oke-ol-training.yaml +++ b/recipes/overlays/oke-ol-training.yaml @@ -27,7 +27,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/oke-ol.yaml b/recipes/overlays/oke-ol.yaml index 57f4687d9..0944a5bdd 100644 --- a/recipes/overlays/oke-ol.yaml +++ b/recipes/overlays/oke-ol.yaml @@ -26,7 +26,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # OKE-specific components componentRefs: diff --git a/recipes/overlays/rtx-pro-6000-lke-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-inference.yaml index bdeb89112..2fba32e06 100644 --- a/recipes/overlays/rtx-pro-6000-lke-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-inference.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-training.yaml b/recipes/overlays/rtx-pro-6000-lke-training.yaml index e397e2b3f..94fb6e0fa 100644 --- a/recipes/overlays/rtx-pro-6000-lke-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-training.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml index 5b8d518a2..f74fbe601 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for inference workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml index d90aeba8a..1554736d2 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for training workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: []