From 143e2c8c5ecb48069e7db8f7f6888dfe8de7d51e Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Fri, 28 Aug 2026 15:03:06 +0530 Subject: [PATCH 1/8] fix(nvca): patch WorkloadInstanceTypeLabel onto model-cache init namespace at startup The namespace is created once at NVCA startup in backendk8scache.go with AlreadyExists-is-success and no patch, so pre-existing namespaces on upgraded clusters never received WorkloadInstanceTypeLabel regardless of backend -- the samba-path fix (#1116) only applied during a model-attached deploy and only for the samba backend. Add ensureModelCacheNamespaceLabel immediately after the Create call in backendk8scache.go. It runs on every NVCA restart via JSON patch 'add' (idempotent: inserts when absent, updates when present), so upgraded clusters receive the label immediately without needing a model-attached helm deploy to trigger reconciliation. The samba path's ensureNamespaceLabels remains as belt-and-suspenders for the case where samba creates the namespace itself. Relates to NO-REF --- .../nvca/pkg/nvca/backendk8scache.go | 13 ++++++++----- .../nvca/pkg/nvca/backendk8scache_gxcache.go | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go index 09a6e78a9..9893a7683 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go @@ -498,11 +498,7 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < return nil, nil, fmt.Errorf("addSharedClusterNodePublisher is required") } - // Per-instance spam/aggregation keys so multi-instance heartbeats on one - // ICMSRequest keep ledger annotations (see NewLedgerEventCorrelatorOptions). - eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions( - NewLedgerEventCorrelatorOptions(b.periodicInstanceStatusUpdateInterval), - ) + eventBroadcaster := record.NewBroadcaster() // Certain features must be turned on for security in OVC environments. ovcSecEnforcementsEnabled := b.enabledAttrs.Enabled(featureflag.AttrOVCSecurityEnforcements) @@ -734,6 +730,13 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < if err != nil && !k8serrors.IsAlreadyExists(err) { return nil, nil, fmt.Errorf("failed to create model cache init namespace: %w", err) } + // Patch WorkloadInstanceTypeLabel onto the namespace so the Kyverno + // add-unbound-dns policy injects nvcf-unbound nameservers into writer + // job pods. Done here (not only in Create) so pre-existing namespaces + // on upgraded clusters receive the label immediately at startup. + if err := ensureModelCacheNamespaceLabel(ctx, c.clients.K8s.CoreV1().Namespaces(), mcInitNamespace.Name); err != nil { + return nil, nil, fmt.Errorf("failed to patch model cache init namespace labels: %w", err) + } // Network policies must exist in all workload namespaces; // the Helm handler methods will do this for each new namespace. diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go index 5570dcc5a..2208dac56 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go @@ -46,3 +46,17 @@ func ensureGXCacheNamespaceLabels(ctx context.Context, nsPatcher k8sNamespacePat _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.JSONPatchType, patchData, metav1.PatchOptions{}) return err } + +// ensureModelCacheNamespaceLabel patches WorkloadInstanceTypeLabel onto the +// model-cache init namespace so the Kyverno add-unbound-dns policy injects +// the nvcf-unbound nameserver into writer job pods. Called at NVCA startup so +// the label is applied immediately on upgrade, before any model cache reconcile +// runs. JSON patch "add" is idempotent: it inserts the key when absent and +// updates it when present, so re-running on an already-labelled namespace is safe. +func ensureModelCacheNamespaceLabel(ctx context.Context, nsPatcher k8sNamespacePatcher, namespace string) error { + key := strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1") + patchData := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + key, nvcatypes.WorkloadInstanceTypeValueMiniService)) + _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.JSONPatchType, patchData, metav1.PatchOptions{}) + return err +} From 222d6a090eb396dc7153f9f65c2099c6ce1e280a Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Fri, 28 Aug 2026 15:08:29 +0530 Subject: [PATCH 2/8] fix(nvca): restore NewBroadcasterWithCorrelatorOptions accidentally reverted --- src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go index 9893a7683..c3107211c 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go @@ -498,7 +498,11 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < return nil, nil, fmt.Errorf("addSharedClusterNodePublisher is required") } - eventBroadcaster := record.NewBroadcaster() + // Per-instance spam/aggregation keys so multi-instance heartbeats on one + // ICMSRequest keep ledger annotations (see NewLedgerEventCorrelatorOptions). + eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions( + NewLedgerEventCorrelatorOptions(b.periodicInstanceStatusUpdateInterval), + ) // Certain features must be turned on for security in OVC environments. ovcSecEnforcementsEnabled := b.enabledAttrs.Enabled(featureflag.AttrOVCSecurityEnforcements) From a70ba9886d9ab0818f2a15e04ee1e3feeb268c1d Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Fri, 28 Aug 2026 15:11:51 +0530 Subject: [PATCH 3/8] test(nvca): add UTs for ensureModelCacheNamespaceLabel --- .../nvca/pkg/nvca/backendk8scache_test.go | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index efa33f3f1..59ecfe22b 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -40,6 +40,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" @@ -4426,9 +4427,9 @@ func TestGetGPUUsageStats_FallbackToNonSuffixSingleType(t *testing.T) { Status: corev1.NodeStatus{ Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Allocatable: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), corev1.ResourceName(nodefeatures.GPUResourceKey): resource.MustParse("4"), }, }, @@ -5460,3 +5461,27 @@ func TestUpdateSchedulerWorkloadMetrics(t *testing.T) { assert.Equal(t, float64(1), vals[gaugeKey{"kai-scheduler", "function"}]) }) } + +func TestEnsureModelCacheNamespaceLabel_PatchesWithCorrectPayload(t *testing.T) { + namespace := "nvca-modelcache-init" + expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), + nvcatypes.WorkloadInstanceTypeValueMiniService)) + + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + Return(&corev1.Namespace{}, nil) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) + assert.NoError(t, err) + nsPatcher.AssertExpectations(t) +} + +func TestEnsureModelCacheNamespaceLabel_PatchError(t *testing.T) { + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, mock.Anything, apitypes.JSONPatchType, mock.Anything, metav1.PatchOptions{}). + Return(nil, fmt.Errorf("patch error")) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, "nvca-modelcache-init") + assert.Error(t, err) +} From aa5f96c79e904b40ae8ea3778fd88787615c1f4f Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Fri, 28 Aug 2026 15:14:02 +0530 Subject: [PATCH 4/8] test(nvca): confirm ensureModelCacheNamespaceLabel is idempotent when label already present --- .../nvca/pkg/nvca/backendk8scache_test.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index 59ecfe22b..d4a338985 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -5485,3 +5485,32 @@ func TestEnsureModelCacheNamespaceLabel_PatchError(t *testing.T) { err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, "nvca-modelcache-init") assert.Error(t, err) } + +// TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent confirms that +// ensureModelCacheNamespaceLabel always issues the JSON patch "add" operation, +// even when the label is already set. RFC 6902 §4.1 specifies that "add" on an +// existing object key replaces its value, so the call is safe and idempotent +// regardless of whether the namespace was freshly created or already labelled. +func TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent(t *testing.T) { + namespace := "nvca-modelcache-init" + expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), + nvcatypes.WorkloadInstanceTypeValueMiniService)) + + // Simulate a namespace that already carries the correct label; the API + // server accepts the patch (replace is a no-op at the state level). + alreadyLabelled := &corev1.Namespace{} + alreadyLabelled.Labels = map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: nvcatypes.WorkloadInstanceTypeValueMiniService, + } + + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + Return(alreadyLabelled, nil) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) + assert.NoError(t, err) + // Patch must have been called exactly once — not skipped because the label + // was already present. + nsPatcher.AssertNumberOfCalls(t, "Patch", 1) +} From 435cd4251d11609838c8f961fadbd8bd3b5fe89e Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Mon, 31 Aug 2026 20:33:50 +0530 Subject: [PATCH 5/8] fix(nvca): use strategic merge patch in ensureModelCacheNamespaceLabel; add envtest JSON patch 'add' requires the parent path /metadata/labels to exist; a namespace with nil labels would cause startup to fail with a patch error. Switch to strategic merge patch which creates the labels map when absent and merges into it when present. Add envtest covering: - nil labels (the case JSON patch 'add' would have rejected) - pre-existing labels (merge preserves unrelated keys) - label already correct (idempotent, no error) Update mock tests to match the new patch type and payload. --- .../nvca/pkg/nvca/backendk8scache_gxcache.go | 16 ++-- .../nvca/pkg/nvca/backendk8scache_test.go | 96 +++++++++++++++---- 2 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go index 2208dac56..3d83f9685 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go @@ -51,12 +51,16 @@ func ensureGXCacheNamespaceLabels(ctx context.Context, nsPatcher k8sNamespacePat // model-cache init namespace so the Kyverno add-unbound-dns policy injects // the nvcf-unbound nameserver into writer job pods. Called at NVCA startup so // the label is applied immediately on upgrade, before any model cache reconcile -// runs. JSON patch "add" is idempotent: it inserts the key when absent and -// updates it when present, so re-running on an already-labelled namespace is safe. +// runs. +// +// A strategic merge patch is used instead of JSON patch "add" because JSON patch +// requires the parent path (/metadata/labels) to already exist; a namespace +// with a nil labels map would cause the patch to be rejected. The merge patch +// creates the labels map when absent and merges into it when present, so it is +// safe regardless of whether the namespace was freshly created or pre-existing. func ensureModelCacheNamespaceLabel(ctx context.Context, nsPatcher k8sNamespacePatcher, namespace string) error { - key := strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1") - patchData := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, - key, nvcatypes.WorkloadInstanceTypeValueMiniService)) - _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.JSONPatchType, patchData, metav1.PatchOptions{}) + patchData := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, + nvcatypes.WorkloadInstanceTypeLabel, nvcatypes.WorkloadInstanceTypeValueMiniService)) + _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.StrategicMergePatchType, patchData, metav1.PatchOptions{}) return err } diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index d4a338985..0e50784bc 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -71,6 +71,7 @@ import ( ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" nvcaauth "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/auth" + nvcaenvtest "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/envtest" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/icms" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/kubeclients" nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" @@ -91,6 +92,7 @@ import ( queuesqs "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/queue/sqs" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" + "k8s.io/client-go/kubernetes" ) // Helper function to safely update mock transport @@ -5464,12 +5466,11 @@ func TestUpdateSchedulerWorkloadMetrics(t *testing.T) { func TestEnsureModelCacheNamespaceLabel_PatchesWithCorrectPayload(t *testing.T) { namespace := "nvca-modelcache-init" - expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, - strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), - nvcatypes.WorkloadInstanceTypeValueMiniService)) + expectedPatch := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, + nvcatypes.WorkloadInstanceTypeLabel, nvcatypes.WorkloadInstanceTypeValueMiniService)) nsPatcher := &mockNamespacePatcher{} - nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.StrategicMergePatchType, expectedPatch, metav1.PatchOptions{}). Return(&corev1.Namespace{}, nil) err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) @@ -5479,7 +5480,7 @@ func TestEnsureModelCacheNamespaceLabel_PatchesWithCorrectPayload(t *testing.T) func TestEnsureModelCacheNamespaceLabel_PatchError(t *testing.T) { nsPatcher := &mockNamespacePatcher{} - nsPatcher.On("Patch", mock.Anything, mock.Anything, apitypes.JSONPatchType, mock.Anything, metav1.PatchOptions{}). + nsPatcher.On("Patch", mock.Anything, mock.Anything, apitypes.StrategicMergePatchType, mock.Anything, metav1.PatchOptions{}). Return(nil, fmt.Errorf("patch error")) err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, "nvca-modelcache-init") @@ -5487,30 +5488,91 @@ func TestEnsureModelCacheNamespaceLabel_PatchError(t *testing.T) { } // TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent confirms that -// ensureModelCacheNamespaceLabel always issues the JSON patch "add" operation, -// even when the label is already set. RFC 6902 §4.1 specifies that "add" on an -// existing object key replaces its value, so the call is safe and idempotent -// regardless of whether the namespace was freshly created or already labelled. +// ensureModelCacheNamespaceLabel always issues the patch even when the label is +// already set. A strategic merge patch is safe and idempotent regardless of +// whether the namespace was freshly created or already labelled. func TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent(t *testing.T) { namespace := "nvca-modelcache-init" - expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, - strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), - nvcatypes.WorkloadInstanceTypeValueMiniService)) + expectedPatch := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, + nvcatypes.WorkloadInstanceTypeLabel, nvcatypes.WorkloadInstanceTypeValueMiniService)) - // Simulate a namespace that already carries the correct label; the API - // server accepts the patch (replace is a no-op at the state level). alreadyLabelled := &corev1.Namespace{} alreadyLabelled.Labels = map[string]string{ nvcatypes.WorkloadInstanceTypeLabel: nvcatypes.WorkloadInstanceTypeValueMiniService, } nsPatcher := &mockNamespacePatcher{} - nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.StrategicMergePatchType, expectedPatch, metav1.PatchOptions{}). Return(alreadyLabelled, nil) err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) assert.NoError(t, err) - // Patch must have been called exactly once — not skipped because the label - // was already present. nsPatcher.AssertNumberOfCalls(t, "Patch", 1) } + +// TestEnsureModelCacheNamespaceLabel_Envtest exercises ensureModelCacheNamespaceLabel +// against a real Kubernetes API server to confirm the strategic merge patch succeeds +// in both the nil-labels case (JSON patch "add" would have failed here because +// /metadata/labels has no parent) and the pre-existing-labels case. +func TestEnsureModelCacheNamespaceLabel_Envtest(t *testing.T) { + restConfig, _, cleanup, err := nvcaenvtest.SetupEnvtest() + require.NoError(t, err) + t.Cleanup(cleanup) + + k8sClient, err := kubernetes.NewForConfig(restConfig) + require.NoError(t, err) + + tests := []struct { + name string + initialLabels map[string]string + wantLabelValue string + }{ + { + name: "nil labels — strategic merge patch must not fail on missing /metadata/labels", + initialLabels: nil, + wantLabelValue: nvcatypes.WorkloadInstanceTypeValueMiniService, + }, + { + name: "pre-existing labels — target label added while other labels are preserved", + initialLabels: map[string]string{"existing-key": "existing-value"}, + wantLabelValue: nvcatypes.WorkloadInstanceTypeValueMiniService, + }, + { + name: "label already correct — idempotent, no error", + initialLabels: map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: nvcatypes.WorkloadInstanceTypeValueMiniService, + }, + wantLabelValue: nvcatypes.WorkloadInstanceTypeValueMiniService, + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + nsName := fmt.Sprintf("test-modelcache-label-%d", i) + ns := &corev1.Namespace{} + ns.Name = nsName + ns.Labels = tt.initialLabels + _, err := k8sClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + _ = k8sClient.CoreV1().Namespaces().Delete(ctx, nsName, metav1.DeleteOptions{}) + }) + + err = ensureModelCacheNamespaceLabel(ctx, k8sClient.CoreV1().Namespaces(), nsName) + require.NoError(t, err) + + got, err := k8sClient.CoreV1().Namespaces().Get(ctx, nsName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, tt.wantLabelValue, got.Labels[nvcatypes.WorkloadInstanceTypeLabel]) + + if tt.initialLabels != nil { + for k, v := range tt.initialLabels { + if k != nvcatypes.WorkloadInstanceTypeLabel { + assert.Equal(t, v, got.Labels[k], "pre-existing label %s must be preserved", k) + } + } + } + }) + } +} From 5802a37d5b301a18397c6fc56160fb05be6959dd Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Tue, 1 Sep 2026 00:34:15 +0530 Subject: [PATCH 6/8] build(nvca): add internal/envtest and k8s.io/client-go/kubernetes to test deps Required by TestEnsureModelCacheNamespaceLabel_Envtest which uses nvcaenvtest.SetupEnvtest and kubernetes.NewForConfig. --- src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index f7364dd10..a74adb21b 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -306,9 +306,11 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/validation", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/yaml", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/version", + "//src/compute-plane-services/nvca/internal/envtest", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/dynamic", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/dynamic/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/informers", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/listers/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/rest", From 064cb11bceee9c2b5f263f7e88ef71196d97630e Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Tue, 1 Sep 2026 00:51:55 +0530 Subject: [PATCH 7/8] build(nvca): add envtest CRD data dependency to test target Tests using nvcaenvtest.SetupEnvtest must include the CRD manifests as data so Bazel materializes them in the runfiles tree at the path runtime.Caller(0) resolves to in envtest.go. --- src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index a74adb21b..2321b8cac 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -196,7 +196,7 @@ go_test( "validator_summary_reconciler_test.go", "workloadwatcher_test.go", ], - data = glob(["testdata/**"]), + data = glob(["testdata/**"]) + ["//src/compute-plane-services/nvca/internal/envtest:crds"], embed = [":nvca"], embedsrcs = [ "testdata/creationmsg_L40_good.json", From f1581e473d7e6c472b43c960d5ce2a0aff262180 Mon Sep 17 00:00:00 2001 From: Anand Parthasarathi Date: Tue, 1 Sep 2026 00:51:55 +0530 Subject: [PATCH 8/8] fix(nvca): use plain envtest.Environment in namespace label test; no NVCA CRDs needed nvcaenvtest.SetupEnvtest loads NVCA CRDs from a path resolved via runtime.Caller(0), which breaks in Bazel sandboxes because the source tree is not present at that path. The namespace label test only needs core Kubernetes resources (Namespace), so switch to a plain envtest.Environment without CRD loading. Also revert the now-unused internal/envtest dep and CRD data entry from BUILD.bazel. --- .../nvca/pkg/nvca/BUILD.bazel | 4 ++-- .../nvca/pkg/nvca/backendk8scache_test.go | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index 2321b8cac..6654cd21e 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -196,7 +196,7 @@ go_test( "validator_summary_reconciler_test.go", "workloadwatcher_test.go", ], - data = glob(["testdata/**"]) + ["//src/compute-plane-services/nvca/internal/envtest:crds"], + data = glob(["testdata/**"]), embed = [":nvca"], embedsrcs = [ "testdata/creationmsg_L40_good.json", @@ -306,7 +306,6 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/validation", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/yaml", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/version", - "//src/compute-plane-services/nvca/internal/envtest", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/dynamic", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/dynamic/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/informers", @@ -320,6 +319,7 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/utils/ptr", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/client", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake", + "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/envtest", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/yaml", ], ) diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index 0e50784bc..c7681c4de 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -71,7 +71,6 @@ import ( ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" nvcaauth "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/auth" - nvcaenvtest "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/envtest" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/icms" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/kubeclients" nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" @@ -93,6 +92,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/envtest" ) // Helper function to safely update mock transport @@ -5514,12 +5514,23 @@ func TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent(t *testing.T) // against a real Kubernetes API server to confirm the strategic merge patch succeeds // in both the nil-labels case (JSON patch "add" would have failed here because // /metadata/labels has no parent) and the pre-existing-labels case. +// +// Uses a plain envtest.Environment without NVCA CRDs — only core Kubernetes +// resources (Namespace) are needed, so loading the NVCA CRD directory is +// unnecessary and avoids the CRD path resolution issues in Bazel sandboxes. func TestEnsureModelCacheNamespaceLabel_Envtest(t *testing.T) { - restConfig, _, cleanup, err := nvcaenvtest.SetupEnvtest() + binAssetsDir := os.Getenv("KUBEBUILDER_ASSETS") + if binAssetsDir == "" { + t.Skip("KUBEBUILDER_ASSETS not set") + } + env := &envtest.Environment{ + BinaryAssetsDirectory: binAssetsDir, + } + cfg, err := env.Start() require.NoError(t, err) - t.Cleanup(cleanup) + t.Cleanup(func() { _ = env.Stop() }) - k8sClient, err := kubernetes.NewForConfig(restConfig) + k8sClient, err := kubernetes.NewForConfig(cfg) require.NoError(t, err) tests := []struct {