From 833cec0e16326ae9c591bd9798f2f10b1e5d2e81 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 01:20:23 -0700
Subject: [PATCH 01/17] test(bdd): cover secure multi-cluster registration

---
 ...ster-helmfile-llm-registration-tls.feature | 179 ++++++++++++++++++
 tests/bdd/godog_test.go                       | 131 +++++++++++++
 2 files changed, 310 insertions(+)
 create mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
new file mode 100644
index 000000000..726c34053
--- /dev/null
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -0,0 +1,179 @@
+@ncp-local @multi-cluster @helmfile @pki @llm-registration
+Feature: Register an LLM worker securely with every router in a local split-cluster stack
+  As a self-managed NVCF operator,
+  I want Pylon registration to use the stack-issued TLS identity across clusters,
+  so that plaintext or untrusted registration cannot silently enter the routing plane.
+
+  Rule: Secure registration is observable from the operator boundary
+
+    Background:
+      Given these environment variables are set:
+        | name            |
+        | NGC_API_KEY     |
+        | NVCF_CLI        |
+        | REPO_ROOT       |
+        | SAMPLE_NGC_ORG  |
+        | SAMPLE_NGC_TEAM |
+      And I prepare Helmfile environment "local-bdd-registration-tls" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
+        | global.imagePullSecrets[0].name                          | nvcr-pull-secret                                                            |
+        | global.helm.sources.repository                           | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
+        | global.image.repository                                  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
+        | global.workerEndpoints.llmRequestRouterAddress           | https://llm-request-router.nvcf.svc.cluster.local:50071                       |
+        | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE            | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0          |
+        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
+        | observability.profile                                    | disabled                                                                     |
+      And I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values:
+        | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
+        | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+        | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+        | observability.profile           | disabled                             |
+      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+      # Conflict precheck: the single-cluster topology owns the same host
+      # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local
+      # before retrying. k3d v5 exits 1 when the cluster is absent.
+      When I run command "k3d cluster get ncp-local"
+      Then the command exit code should be 1
+      And multi-cluster ncp-local compute clusters are running:
+        | ncp-local-compute-1 |
+      And command has succeeded:
+        """
+        kubectl config use-context k3d-ncp-local-cp
+        """
+      And the "nvcr-pull-secret" image pull secret exists in namespaces:
+        | cassandra-system |
+        | nats-system      |
+        | nvcf             |
+        | api-keys         |
+        | ess              |
+        | sis              |
+        | vault-system     |
+        | nvca-operator    |
+        | cert-manager     |
+
+    @llm-registration-tls-install
+    Scenario: Operator installs a TLS-only registration endpoint with three concrete routers
+      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls"
+      Then the command exit code should be 0
+      And the rendered manifests in "deploy/stacks/self-managed/out" should contain:
+        | text                                                                                 |
+        | https://llm-request-router.nvcf.svc.cluster.local:50071                              |
+        | --grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071 |
+
+      When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls"
+      Then the command exit code should be 0
+
+      When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
+      Then the command exit code should be 0
+
+      When I run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml"
+      Then the command exit code should be 0
+      And the command output should contain "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071"
+
+      # openssl verifies the externally reachable listener against the same
+      # stack-issued CA and DNS identity that a compute-plane Pylon uses.
+      When I run command:
+        """
+        /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
+        """
+      Then the command exit code should be 0
+      And the command output should contain "Verify return code: 0 (ok)"
+      And the command output should contain "ALPN protocol: h2"
+
+      When I run command:
+        """
+        grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates
+        """
+      Then the command exit code should be 1
+
+      # WatchStargates is a long-lived stream. grpcurl exits on its explicit
+      # deadline after printing the initial snapshot, so exit 1 is expected.
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        """
+      Then the command exit code should be 1
+      And the command output should contain "llm-request-router-0"
+      And the command output should contain "llm-request-router-1"
+      And the command output should contain "llm-request-router-2"
+      And the command output should contain "https://llm-request-router.nvcf.svc.cluster.local:50071"
+
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-tls --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp
+        """
+      Then the command exit code should be 0
+      And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist
+      And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys:
+        | key                                 |
+        | managementTls.caBundlePem           |
+        | transportTls.trustBundleFingerprint |
+        | transportTls.trustBundlePem         |
+
+      And command has succeeded:
+        """
+        /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null'
+        """
+      When I run command "kubectl config use-context k3d-ncp-local-compute-1"
+      Then the command exit code should be 0
+      When I run command:
+        """
+        make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
+        """
+      Then the command exit code should be 0
+      And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-compute-1-register-values.yaml" should exist
+      And the "nvcr-pull-secret" image pull secret exists in namespaces:
+        | nvca-operator |
+      When I run command:
+        """
+        make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-tls COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI}
+        """
+      Then the command exit code should be 0
+      Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m"
+
+    @llm-registration-tls-runtime
+    Scenario: Pylon registers with every router and serves an authenticated LLM request
+      Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml"
+      When I successfully create function "bdd-registration-tls" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options:
+        | option           | value                                                                                               |
+        | --function-type  | LLM                                                                                                 |
+        | --inference-url  | /v1/chat/completions                                                                                |
+        | --inference-port | 8000                                                                                                |
+        | --health-uri     | /health                                                                                             |
+        | --health-port    | 8000                                                                                                |
+        | --health-timeout | PT30S                                                                                               |
+        | --llm-model      | name=openai-compatible-sample,uris=/v1/chat/completions\|/v1/embeddings,routingMethod=round_robin |
+      And I successfully deploy the function selected by NVCF CLI with options:
+        | option          | value               |
+        | --gpu           | H100                |
+        | --instance-type | NCP.GPU.H100_1x     |
+        | --backend       | ncp-local-compute-1 |
+        | --regions       | us-west-1           |
+        | --min-instances | 1                   |
+        | --max-instances | 1                   |
+        | --timeout       | 900                 |
+      And I successfully generate a function API key with CLI options:
+        | option        | value                                                               |
+        | --description | bdd-registration-tls                                                |
+        | --scopes      | invoke_function,list_functions,queue_details,list_functions_details |
+
+      # Discover the workload pod by its public sidecar name, then poll the
+      # Pylon metrics endpoint until all three router streams and reverse
+      # tunnels are connected.
+      When I run command:
+        """
+        /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); tunnels=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; fi; sleep 5; done; exit 1'
+        """
+      Then the command exit code should be 0
+      And the command output should contain "registration=3 reverse=3"
+
+      When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds:
+        """
+        {"messages":[{"role":"user","content":"bdd-registration-tls"}]}
+        """
+      Then the command output should contain "chat.completion"
+      And the command output should contain "fixed 128-byte response"
+      And I successfully undeploy the function selected by NVCF CLI
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index a71788ab9..4e05ce911 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1350,6 +1350,127 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t
 	}
 }
 
+// TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps runs the
+// focused secure registration feature against a fake runner. The canned
+// external observations cover the TLS listener, WatchStargates snapshot,
+// Pylon metrics, and authenticated invocation.
+func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testing.T) {
+	t.Setenv("NGC_API_KEY", "test-key")
+	t.Setenv("SAMPLE_NGC_ORG", "test-org")
+	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
+	t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli")
+	t.Setenv("REPO_ROOT", "/repo-root-placeholder")
+
+	const (
+		tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
+			`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
+			`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
+			`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
+		plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates"
+		tlsWatchCommand       = `/bin/bash -c 'grpcurl -max-time 3 ` +
+			`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
+			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
+			`-authority llm-request-router.nvcf.svc.cluster.local ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+		pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` +
+			`row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` +
+			`jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` +
+			`[.metadata.namespace,.metadata.name] | @tsv] | first // empty"); ` +
+			`if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); ` +
+			`pod=$(printf "%s" "$row" | cut -f2); ` +
+			`metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw ` +
+			`"/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); ` +
+			`registration=$(printf "%s\n" "$metrics" | ` +
+			`grep -c "^pylon_registration_stream_connected.* 1$" || true); ` +
+			`tunnels=$(printf "%s\n" "$metrics" | ` +
+			`grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); ` +
+			`if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then ` +
+			`printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; ` +
+			`fi; sleep 5; done; exit 1'`
+		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
+			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
+			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120"
+	)
+
+	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
+		"k3d cluster get ncp-local": {ExitCode: 1},
+		"kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml": {
+			ExitCode: 0,
+			Stdout:   "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071\n",
+		},
+		tlsHandshakeCommand: {
+			ExitCode: 0,
+			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
+		},
+		plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"},
+		tlsWatchCommand: {
+			ExitCode: 1,
+			Stdout: `{
+  "stargates": [
+    {"stargateId": "llm-request-router-0", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"},
+    {"stargateId": "llm-request-router-1", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"},
+    {"stargateId": "llm-request-router-2", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}
+  ]
+}`,
+			Stderr: "ERROR: DeadlineExceeded",
+		},
+		pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=3 reverse=3\n"},
+		invokeCommand: {
+			ExitCode: 0,
+			Stdout: "Function invocation completed!\n\nResponse:\n" +
+				`{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` +
+				"\n",
+		},
+	}))
+	seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot)
+	seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot)
+	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
+	writeProfileHandoffArtifact(t, suite.Config.RepoRoot)
+	writeMulticlusterComputeRegisterValues(t, suite.Config.RepoRoot, "nvcf-compute-plane", "ncp-local-compute-1")
+	writeArtifact(
+		t,
+		suite.Config.RepoRoot,
+		"self-managed",
+		"registration-tls-rendered.yaml",
+		"https://llm-request-router.nvcf.svc.cluster.local:50071\n"+
+			"--grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071\n",
+	)
+
+	sc := steps.NewScenarioContext(suite)
+	featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls.feature")
+	var out strings.Builder
+	status := godog.TestSuite{
+		Name: "multi-cluster-helmfile-llm-registration-tls-wiring",
+		ScenarioInitializer: func(ctx *godog.ScenarioContext) {
+			steps.RegisterAll(ctx, sc)
+		},
+		Options: &godog.Options{
+			Format: "pretty",
+			Paths:  []string{featurePath},
+			Strict: true,
+			Output: &out,
+		},
+	}.Run()
+	if status != 0 {
+		t.Fatalf("godog suite status = %d\n%s", status, out.String())
+	}
+	if !commandRanThatContainsAll(
+		suite.Runner.(*fakeRunner).runs,
+		"function create --name bdd-registration-tls",
+		"--function-type LLM",
+		"--llm-model",
+	) {
+		t.Fatal("secure registration sample was not created as an LLM function")
+	}
+	if !commandRanExactly(suite.Runner.(*fakeRunner).runs, pylonMetricsCommand) {
+		t.Fatal("Pylon registration and reverse-tunnel metrics were not observed")
+	}
+	if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) {
+		t.Fatal("WatchStargates was not observed over the trusted TLS listener")
+	}
+}
+
 // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the
 // focused upstream-image feature against a fake runner. The seeded global
 // template contains the exact documentation blocks so the ledger-backed
@@ -2125,6 +2246,16 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregion(t *testing.T) {
 	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-multiregion.feature")
 }
 
+// TestMultiClusterHelmfileLLMRegistrationTLS is the live entry point for the
+// focused secure Pylon registration feature on local split-cluster k3d.
+// Skipped under -short.
+func TestMultiClusterHelmfileLLMRegistrationTLS(t *testing.T) {
+	if testing.Short() {
+		t.Skip("live run skipped under -short")
+	}
+	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls.feature")
+}
+
 // TestSingleClusterEKSHelmfile is the live entry point for the
 // single-cluster EKS Helmfile feature. Skipped under -short.
 func TestSingleClusterEKSHelmfile(t *testing.T) {

From b687a722722f32cd71c71e97c4a0a133eaf5cf3e Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 01:26:27 -0700
Subject: [PATCH 02/17] test(bdd): pin router workload for TLS registration

---
 .../features/multi-cluster-helmfile-llm-registration-tls.feature | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 726c34053..58187c296 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -20,6 +20,7 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         | global.image.repository                                  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
         | global.workerEndpoints.llmRequestRouterAddress           | https://llm-request-router.nvcf.svc.cluster.local:50071                       |
         | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE            | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0          |
+        | addons.llm.requestRouter.workload.kind                    | StatefulSet                                                                 |
         | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
         | observability.profile                                    | disabled                                                                     |
       And I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values:

From 6d458d62bf844249644533654b0df53d29ff73bc Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 01:45:13 -0700
Subject: [PATCH 03/17] test(bdd): accept grpcurl stream timeout codes

---
 .../multi-cluster-helmfile-llm-registration-tls.feature      | 4 ++--
 tests/bdd/godog_test.go                                      | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 58187c296..68a24d7d0 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -94,9 +94,9 @@ Feature: Register an LLM worker securely with every router in a local split-clus
       # deadline after printing the initial snapshot, so exit 1 is expected.
       When I run command:
         """
-        /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]'
         """
-      Then the command exit code should be 1
+      Then the command exit code should be 0
       And the command output should contain "llm-request-router-0"
       And the command output should contain "llm-request-router-1"
       And the command output should contain "llm-request-router-2"
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 4e05ce911..d9d4a152f 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1372,7 +1372,8 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
 			`-authority llm-request-router.nvcf.svc.cluster.local ` +
 			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; ` +
+			`rc=$?; [ "$rc" -ne 0 ]'`
 		pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` +
 			`row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` +
 			`jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` +
@@ -1405,7 +1406,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 		},
 		plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"},
 		tlsWatchCommand: {
-			ExitCode: 1,
+			ExitCode: 0,
 			Stdout: `{
   "stargates": [
     {"stargateId": "llm-request-router-0", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"},

From e33b0bf2a6bfedec7754e2779b19f5594a4a0911 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 02:34:56 -0700
Subject: [PATCH 04/17] test(bdd): use stack Pylon artifact configuration

---
 .../features/multi-cluster-helmfile-llm-registration-tls.feature | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 68a24d7d0..6f18a784b 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -19,7 +19,6 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         | global.helm.sources.repository                           | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
         | global.image.repository                                  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
         | global.workerEndpoints.llmRequestRouterAddress           | https://llm-request-router.nvcf.svc.cluster.local:50071                       |
-        | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE            | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0          |
         | addons.llm.requestRouter.workload.kind                    | StatefulSet                                                                 |
         | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
         | observability.profile                                    | disabled                                                                     |

From 3d1c7654832c04457e8dd1d83cfbce2b3a26b454 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 03:10:35 -0700
Subject: [PATCH 05/17] test(bdd): clarify streaming Watch assertion

---
 .../multi-cluster-helmfile-llm-registration-tls.feature       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 6f18a784b..2262904fc 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -89,8 +89,8 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         """
       Then the command exit code should be 1
 
-      # WatchStargates is a long-lived stream. grpcurl exits on its explicit
-      # deadline after printing the initial snapshot, so exit 1 is expected.
+      # WatchStargates is a long-lived stream. Normalize grpcurl's deadline
+      # exit after it prints the initial snapshot.
       When I run command:
         """
         /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]'

From 6a12a4e2e3784ca4f1a65aa397b70e3daf0a5fed Mon Sep 17 00:00:00 2001
From: Mike Camp <mcamp@nvidia.com>
Date: Fri, 28 Aug 2026 16:16:09 -0400
Subject: [PATCH 06/17] test(bdd): assert plaintext TLS rejection (#1323)

Signed-off-by: Mike Camp <mcamp@nvidia.com>
---
 ...ster-helmfile-llm-registration-tls.feature |  8 +-
 tests/bdd/godog_test.go                       | 14 +++-
 .../plaintext_tls_rejection_script_test.go    | 81 +++++++++++++++++++
 .../assert-grpcurl-plaintext-tls-rejection.sh | 19 +++++
 4 files changed, 117 insertions(+), 5 deletions(-)
 create mode 100644 tests/bdd/plaintext_tls_rejection_script_test.go
 create mode 100755 tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 2262904fc..1c890de8f 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -83,11 +83,15 @@ Feature: Register an LLM worker securely with every router in a local split-clus
       And the command output should contain "Verify return code: 0 (ok)"
       And the command output should contain "ALPN protocol: h2"
 
+      # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is
+      # sent to this verified TLS listener. The trusted Watch below proves
+      # that the same endpoint remains healthy.
       When I run command:
         """
-        grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates
+        /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'
         """
-      Then the command exit code should be 1
+      Then the command exit code should be 0
+      And the command output should contain "plaintext-watch-rejected=tls-listener-timeout"
 
       # WatchStargates is a long-lived stream. Normalize grpcurl's deadline
       # exit after it prints the initial snapshot.
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index d9d4a152f..85fa31b52 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1366,8 +1366,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 			`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
 			`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
 			`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
-		plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates"
-		tlsWatchCommand       = `/bin/bash -c 'grpcurl -max-time 3 ` +
+		plaintextWatchCommand = `/bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; ` +
+			`if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` +
+			`exit 1; fi; printf "%s\n" "$output" | ` +
+			`bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'`
+		tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` +
 			`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
 			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
 			`-authority llm-request-router.nvcf.svc.cluster.local ` +
@@ -1404,7 +1409,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 			ExitCode: 0,
 			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
 		},
-		plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"},
+		plaintextWatchCommand: {ExitCode: 0, Stdout: "plaintext-watch-rejected=tls-listener-timeout\n"},
 		tlsWatchCommand: {
 			ExitCode: 0,
 			Stdout: `{
@@ -1470,6 +1475,9 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 	if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) {
 		t.Fatal("WatchStargates was not observed over the trusted TLS listener")
 	}
+	if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) {
+		t.Fatal("plaintext WatchStargates rejection was not exercised")
+	}
 }
 
 // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the
diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go
new file mode 100644
index 000000000..fbb75ea79
--- /dev/null
+++ b/tests/bdd/plaintext_tls_rejection_script_test.go
@@ -0,0 +1,81 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bdd_tmp
+
+import (
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) {
+	cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh")
+	cmd.Stdin = strings.NewReader(
+		`Failed to dial target host "127.0.0.1:50071": context deadline exceeded`,
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("classify grpcurl deadline: %v\n%s", err, output)
+	}
+	if got, want := strings.TrimSpace(string(output)), "plaintext-watch-rejected=tls-listener-timeout"; got != want {
+		t.Fatalf("normalized output = %q, want %q", got, want)
+	}
+}
+
+func TestPlaintextTLSRejectionRejectsSnapshotThenRPCDeadline(t *testing.T) {
+	cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh")
+	cmd.Stdin = strings.NewReader(`{
+  "stargates": []
+}
+ERROR:
+  Code: DeadlineExceeded
+  Message: context deadline exceeded`)
+
+	output, err := cmd.CombinedOutput()
+	if err == nil {
+		t.Fatalf("successful plaintext snapshot followed by RPC deadline was accepted: %s", output)
+	}
+	if strings.Contains(string(output), "plaintext-watch-rejected=") {
+		t.Fatalf("successful plaintext snapshot emitted success marker: %s", output)
+	}
+}
+
+func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) {
+	for name, diagnostic := range map[string]string{
+		"binary missing":            "bash: grpcurl: command not found",
+		"connection refused":        "Failed to dial target host 127.0.0.1:50071: connection refused",
+		"dial timeout plus output":  "Failed to dial target host \"127.0.0.1:50071\": context deadline exceeded\n{}",
+		"proto import":              "Failed to process proto source files.: missing.proto does not reside in any import path",
+		"usage":                     "flag provided but not defined: -bad-flag",
+		"wrong target dial timeout": "Failed to dial target host \"127.0.0.1:50443\": context deadline exceeded",
+	} {
+		t.Run(name, func(t *testing.T) {
+			cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh")
+			cmd.Stdin = strings.NewReader(diagnostic)
+
+			output, err := cmd.CombinedOutput()
+			if err == nil {
+				t.Fatalf("unrelated failure was accepted: %s", output)
+			}
+			if strings.Contains(string(output), "plaintext-watch-rejected=") {
+				t.Fatalf("unrelated failure emitted success marker: %s", output)
+			}
+		})
+	}
+}
diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh
new file mode 100755
index 000000000..453cdbc12
--- /dev/null
+++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Normalize only the complete grpcurl 1.9.3 dial-stage diagnostic observed when
+# plaintext HTTP/2 is sent to the verified TLS listener. RPC stream deadlines,
+# additional output, and unrelated command or endpoint failures remain errors.
+
+set -euo pipefail
+
+diagnostic="$(cat)"
+expected_diagnostic='Failed to dial target host "127.0.0.1:50071": context deadline exceeded'
+if [[ "${diagnostic}" == "${expected_diagnostic}" ]]; then
+  printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout'
+  exit 0
+fi
+
+printf '%s\n' 'plaintext Watch failed without the expected TLS-listener dial timeout' >&2
+exit 1

From ca590fea645839c7cbc59723fa6c70c1758b52c2 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Sun, 30 Aug 2026 00:03:36 -0700
Subject: [PATCH 07/17] test(bdd): use shared registration observability steps

Reuse the shared WatchStargates and Pylon metrics assertions while keeping the endpoint, trust, topology, and expected counts visible in Gherkin.

Refs #1305
Refs #1358

Signed-off-by: Stephanie Baum <sbaum@nvidia.com>
---
 ...ster-helmfile-llm-registration-tls.feature | 37 ++++++++-----------
 tests/bdd/godog_test.go                       | 35 ++++++------------
 2 files changed, 27 insertions(+), 45 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 1c890de8f..f2e5cc931 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -93,17 +93,13 @@ Feature: Register an LLM worker securely with every router in a local split-clus
       Then the command exit code should be 0
       And the command output should contain "plaintext-watch-rejected=tls-listener-timeout"
 
-      # WatchStargates is a long-lived stream. Normalize grpcurl's deadline
-      # exit after it prints the initial snapshot.
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]'
-        """
-      Then the command exit code should be 0
-      And the command output should contain "llm-request-router-0"
-      And the command output should contain "llm-request-router-1"
-      And the command output should contain "llm-request-router-2"
-      And the command output should contain "https://llm-request-router.nvcf.svc.cluster.local:50071"
+      When I successfully observe WatchStargates at "127.0.0.1:50071" with TLS authority "llm-request-router.nvcf.svc.cluster.local" using CA secret "stargate-quic-tls" in namespace "nvcf" and context "k3d-ncp-local-cp" for "3" seconds
+      Then the command output should contain all:
+        | text                                                                    |
+        | llm-request-router-0                                                    |
+        | llm-request-router-1                                                    |
+        | llm-request-router-2                                                    |
+        | https://llm-request-router.nvcf.svc.cluster.local:50071                |
 
       When I run command:
         """
@@ -164,20 +160,17 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         | --description | bdd-registration-tls                                                |
         | --scopes      | invoke_function,list_functions,queue_details,list_functions_details |
 
-      # Discover the workload pod by its public sidecar name, then poll the
-      # Pylon metrics endpoint until all three router streams and reverse
-      # tunnels are connected.
-      When I run command:
-        """
-        /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); tunnels=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; fi; sleep 5; done; exit 1'
-        """
-      Then the command exit code should be 0
-      And the command output should contain "registration=3 reverse=3"
+      Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m":
+        | metric                               | comparison | count |
+        | pylon_registration_stream_connected | exactly    | 3     |
+        | pylon_reverse_tunnel_connected       | exactly    | 3     |
 
       When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds:
         """
         {"messages":[{"role":"user","content":"bdd-registration-tls"}]}
         """
-      Then the command output should contain "chat.completion"
-      And the command output should contain "fixed 128-byte response"
+      Then the command output should contain all:
+        | text                    |
+        | chat.completion         |
+        | fixed 128-byte response |
       And I successfully undeploy the function selected by NVCF CLI
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 85fa31b52..19457b7fe 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1372,28 +1372,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 			`if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` +
 			`exit 1; fi; printf "%s\n" "$output" | ` +
 			`bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'`
-		tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` +
-			`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
-			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
-			`-authority llm-request-router.nvcf.svc.cluster.local ` +
-			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; ` +
-			`rc=$?; [ "$rc" -ne 0 ]'`
-		pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` +
-			`row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` +
-			`jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` +
-			`[.metadata.namespace,.metadata.name] | @tsv] | first // empty"); ` +
-			`if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); ` +
-			`pod=$(printf "%s" "$row" | cut -f2); ` +
-			`metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw ` +
-			`"/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); ` +
-			`registration=$(printf "%s\n" "$metrics" | ` +
-			`grep -c "^pylon_registration_stream_connected.* 1$" || true); ` +
-			`tunnels=$(printf "%s\n" "$metrics" | ` +
-			`grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); ` +
-			`if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then ` +
-			`printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; ` +
-			`fi; sleep 5; done; exit 1'`
+		tlsWatchCommand = "bash tests/bdd/scripts/observe-watch-stargates.sh" +
+			" 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local" +
+			" stargate-quic-tls nvcf k3d-ncp-local-cp 3"
+		pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" +
+			" llm-worker k3d-ncp-local-compute-1 10m" +
+			" pylon_registration_stream_connected exactly 3" +
+			" pylon_reverse_tunnel_connected exactly 3"
 		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
 			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
 			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120"
@@ -1421,7 +1406,11 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 }`,
 			Stderr: "ERROR: DeadlineExceeded",
 		},
-		pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=3 reverse=3\n"},
+		pylonMetricsCommand: {
+			ExitCode: 0,
+			Stdout: "pylon_registration_stream_connected=3\n" +
+				"pylon_reverse_tunnel_connected=3\n",
+		},
 		invokeCommand: {
 			ExitCode: 0,
 			Stdout: "Function invocation completed!\n\nResponse:\n" +

From 1d4f3e55d34ad9e81cc56c1f8ede2d71089149ca Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Sun, 30 Aug 2026 00:41:56 -0700
Subject: [PATCH 08/17] test(bdd): target registration Pylons by function

Refs #1292

Signed-off-by: Stephanie Baum <sbaum@nvidia.com>
---
 .../multi-cluster-helmfile-llm-registration-tls.feature         | 2 +-
 tests/bdd/godog_test.go                                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index f2e5cc931..4979e59c2 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -160,7 +160,7 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         | --description | bdd-registration-tls                                                |
         | --scopes      | invoke_function,list_functions,queue_details,list_functions_details |
 
-      Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m":
+      Then every Pylon for function "bdd-registration-tls" using container "llm-worker" and context "k3d-ncp-local-compute-1" should report metrics within "10m":
         | metric                               | comparison | count |
         | pylon_registration_stream_connected | exactly    | 3     |
         | pylon_reverse_tunnel_connected       | exactly    | 3     |
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 19457b7fe..a4943aa48 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1376,7 +1376,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 			" 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local" +
 			" stargate-quic-tls nvcf k3d-ncp-local-cp 3"
 		pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" +
-			" llm-worker k3d-ncp-local-compute-1 10m" +
+			" bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m" +
 			" pylon_registration_stream_connected exactly 3" +
 			" pylon_reverse_tunnel_connected exactly 3"
 		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +

From a8c564e2ecbf85ab9598a5a643879d921d282c2d Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 02:39:52 -0700
Subject: [PATCH 09/17] test(bdd): cover fail-closed Pylon TLS registration

---
 ...e-llm-registration-tls-fail-closed.feature | 102 ++++++++++++++++++
 tests/bdd/godog_test.go                       |  86 +++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
new file mode 100644
index 000000000..a092c182d
--- /dev/null
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
@@ -0,0 +1,102 @@
+@ncp-local @multi-cluster @helmfile @pki @llm-registration @negative
+Feature: Reject insecure or invalid LLM worker registration
+  As a self-managed NVCF operator,
+  I want the Pylon registration endpoint to fail closed,
+  so that invalid trust or authority cannot silently enter the routing plane.
+
+  Rule: Every rejected path is observable at the operator boundary
+
+    Background:
+      Given these environment variables are set:
+        | name            |
+        | NGC_API_KEY     |
+        | SAMPLE_NGC_ORG  |
+        | SAMPLE_NGC_TEAM |
+      And I prepare Helmfile environment "local-bdd-registration-tls-fail-closed" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
+        | global.imagePullSecrets[0].name                            | nvcr-pull-secret                                                     |
+        | global.helm.sources.repository                             | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.image.repository                                    | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.workerEndpoints.llmRequestRouterAddress             | https://llm-request-router.nvcf.svc.cluster.local:50071              |
+        | addons.llm.requestRouter.workload.kind                      | StatefulSet                                                          |
+        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071              |
+        | observability.profile                                      | disabled                                                             |
+      And I prepare Helmfile environment "local-bdd-registration-tls-invalid-authority" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
+        | global.imagePullSecrets[0].name                            | nvcr-pull-secret                                                     |
+        | global.helm.sources.repository                             | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.image.repository                                    | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.workerEndpoints.llmRequestRouterAddress             | https://llm_request_router.nvcf.svc.cluster.local:50071              |
+        | addons.llm.requestRouter.workload.kind                      | StatefulSet                                                          |
+        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm_request_router.nvcf.svc.cluster.local:50071              |
+        | observability.profile                                      | disabled                                                             |
+      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-fail-closed-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+      When I run command "k3d cluster get ncp-local"
+      Then the command exit code should be 1
+      And multi-cluster ncp-local compute clusters are running:
+        | ncp-local-compute-1 |
+      And command has succeeded:
+        """
+        kubectl config use-context k3d-ncp-local-cp
+        """
+      And the "nvcr-pull-secret" image pull secret exists in namespaces:
+        | cassandra-system |
+        | nats-system      |
+        | nvcf             |
+        | api-keys         |
+        | ess              |
+        | sis              |
+        | vault-system     |
+        | nvca-operator    |
+        | cert-manager     |
+
+    Scenario: Registration rejects every untrusted client path
+      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-fail-closed"
+      Then the command exit code should be 0
+      When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls-fail-closed"
+      Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
+      Then the command exit code should be 0
+
+      # Establish that the TLS listener is reachable with its issued root,
+      # expected DNS identity, and HTTP/2 application protocol.
+      When I run command:
+        """
+        /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
+        """
+      Then the command exit code should be 0
+      And the command output should contain "Verify return code: 0 (ok)"
+      And the command output should contain "ALPN protocol: h2"
+
+      When I run command:
+        """
+        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-root-rejected\n"'
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-host-rejected\n"'
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "missing-trust-rejected\n"'
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "plaintext-rejected\n"'
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        /bin/sh -c 'make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "invalid-authority-rejected\n"'
+        """
+      Then the command exit code should be 0
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index a4943aa48..60fee85a2 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1469,6 +1469,83 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 	}
 }
 
+// TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
+// runs the negative TLS registration matrix against a fake runner.
+func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps(t *testing.T) {
+	t.Setenv("NGC_API_KEY", "test-key")
+	t.Setenv("SAMPLE_NGC_ORG", "test-org")
+	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
+
+	const tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
+		`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
+		`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
+		`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
+	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
+		"k3d cluster get ncp-local": {ExitCode: 1},
+		tlsHandshakeCommand: {
+			ExitCode: 0,
+			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
+		},
+	}))
+	seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot)
+	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
+
+	sc := steps.NewScenarioContext(suite)
+	featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature")
+	var out strings.Builder
+	status := godog.TestSuite{
+		Name: "multi-cluster-helmfile-llm-registration-tls-fail-closed-wiring",
+		ScenarioInitializer: func(ctx *godog.ScenarioContext) {
+			steps.RegisterAll(ctx, sc)
+		},
+		Options: &godog.Options{
+			Format: "pretty",
+			Paths:  []string{featurePath},
+			Strict: true,
+			Output: &out,
+		},
+	}.Run()
+	if status != 0 {
+		t.Fatalf("godog suite status = %d\n%s", status, out.String())
+	}
+	for _, marker := range []string{
+		"wrong-root-rejected",
+		"wrong-host-rejected",
+		"missing-trust-rejected",
+		"plaintext-rejected",
+		"invalid-authority-rejected",
+	} {
+		if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, marker) {
+			t.Fatalf("negative registration command containing %q was not invoked", marker)
+		}
+	}
+	validEnvironment := filepath.Join(
+		suite.Config.RepoRoot,
+		"deploy", "stacks", "self-managed", "environments",
+		"local-bdd-registration-tls-fail-closed.yaml",
+	)
+	invalidEnvironment := filepath.Join(
+		suite.Config.RepoRoot,
+		"deploy", "stacks", "self-managed", "environments",
+		"local-bdd-registration-tls-invalid-authority.yaml",
+	)
+	for _, assertion := range []struct {
+		path string
+		want string
+	}{
+		{path: validEnvironment, want: "https://llm-request-router.nvcf.svc.cluster.local:50071"},
+		{path: invalidEnvironment, want: "https://llm_request_router.nvcf.svc.cluster.local:50071"},
+	} {
+		got, found, err := dsl.ReadYAMLKey(assertion.path, "global.workerEndpoints.llmRequestRouterAddress")
+		if err != nil {
+			t.Fatalf("read worker endpoint override: %v", err)
+		}
+		if !found || got != assertion.want {
+			t.Fatalf("worker endpoint = %q, found = %t; want %q", got, found, assertion.want)
+		}
+	}
+}
+
 // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the
 // focused upstream-image feature against a fake runner. The seeded global
 // template contains the exact documentation blocks so the ledger-backed
@@ -2254,6 +2331,15 @@ func TestMultiClusterHelmfileLLMRegistrationTLS(t *testing.T) {
 	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls.feature")
 }
 
+// TestMultiClusterHelmfileLLMRegistrationTLSFailClosed is the live entry
+// point for the negative TLS registration matrix. Skipped under -short.
+func TestMultiClusterHelmfileLLMRegistrationTLSFailClosed(t *testing.T) {
+	if testing.Short() {
+		t.Skip("live run skipped under -short")
+	}
+	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature")
+}
+
 // TestSingleClusterEKSHelmfile is the live entry point for the
 // single-cluster EKS Helmfile feature. Skipped under -short.
 func TestSingleClusterEKSHelmfile(t *testing.T) {

From e25d1029b61ea4fb31d2b41144a7ace03e7512a7 Mon Sep 17 00:00:00 2001
From: Mike Camp <mcamp@nvidia.com>
Date: Fri, 28 Aug 2026 16:18:47 -0400
Subject: [PATCH 10/17] test(bdd): require TLS rejection diagnostics (#1324)

Signed-off-by: Mike Camp <mcamp@nvidia.com>
---
 ...e-llm-registration-tls-fail-closed.feature | 14 ++++---
 tests/bdd/godog_test.go                       | 37 ++++++++++++++-----
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
index a092c182d..af778e57b 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
@@ -30,6 +30,8 @@ Feature: Reject insecure or invalid LLM worker registration
         | observability.profile                                      | disabled                                                             |
       And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-fail-closed-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
       And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+      When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
+      Then the command exit code should be 0
       When I run command "k3d cluster get ncp-local"
       Then the command exit code should be 1
       And multi-cluster ncp-local compute clusters are running:
@@ -65,7 +67,7 @@ Feature: Reject insecure or invalid LLM worker registration
       # expected DNS identity, and HTTP/2 application protocol.
       When I run command:
         """
-        /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
+        /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
         """
       Then the command exit code should be 0
       And the command output should contain "Verify return code: 0 (ok)"
@@ -73,30 +75,30 @@ Feature: Reject insecure or invalid LLM worker registration
 
       When I run command:
         """
-        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-root-rejected\n"'
+        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
       When I run command:
         """
-        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-host-rejected\n"'
+        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong hostname was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"not wrong-host.nvcf.svc.cluster.local"*) printf "wrong-host-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
       When I run command:
         """
-        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "missing-trust-rejected\n"'
+        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
       When I run command:
         """
-        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "plaintext-rejected\n"'
+        /bin/bash -c 'if diagnostic=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "plaintext was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"context deadline exceeded"*|*"error reading server preface"*) printf "plaintext-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
       When I run command:
         """
-        /bin/sh -c 'make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "invalid-authority-rejected\n"'
+        /bin/sh -c 'if diagnostic=$(make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority 2>&1); then printf "invalid authority was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535"*) printf "invalid-authority-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 60fee85a2..c5bc2b20b 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1477,11 +1477,14 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
 
 	const tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
-		`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
+		`-servername llm-request-router.nvcf.svc.cluster.local ` +
+		`-verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
 		`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
 		`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
+	const grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'`
 	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
 		"k3d cluster get ncp-local": {ExitCode: 1},
+		grpcurlPreflightCommand:     {ExitCode: 0},
 		tlsHandshakeCommand: {
 			ExitCode: 0,
 			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
@@ -1508,15 +1511,31 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 	if status != 0 {
 		t.Fatalf("godog suite status = %d\n%s", status, out.String())
 	}
-	for _, marker := range []string{
-		"wrong-root-rejected",
-		"wrong-host-rejected",
-		"missing-trust-rejected",
-		"plaintext-rejected",
-		"invalid-authority-rejected",
+	runs := suite.Runner.(*fakeRunner).runs
+	if !commandRanExactly(runs, grpcurlPreflightCommand) {
+		t.Fatal("grpcurl availability was not checked before the live probes")
+	}
+	for _, assertion := range []struct {
+		marker     string
+		diagnostic string
+	}{
+		{marker: "wrong-root-rejected", diagnostic: "certificate signed by unknown authority"},
+		{marker: "wrong-host-rejected", diagnostic: "not wrong-host.nvcf.svc.cluster.local"},
+		{marker: "missing-trust-rejected", diagnostic: "certificate signed by unknown authority"},
+		{marker: "plaintext-rejected", diagnostic: "context deadline exceeded"},
+		{
+			marker: "invalid-authority-rejected",
+			diagnostic: "global.workerEndpoints.llmRequestRouterAddress must use " +
+				"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
+				"with port 1-65535",
+		},
 	} {
-		if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, marker) {
-			t.Fatalf("negative registration command containing %q was not invoked", marker)
+		if !commandRanThatContainsAll(runs, assertion.marker, assertion.diagnostic) {
+			t.Fatalf(
+				"negative registration command containing %q did not require diagnostic %q",
+				assertion.marker,
+				assertion.diagnostic,
+			)
 		}
 	}
 	validEnvironment := filepath.Join(

From d4b88a2f0f94e2fb24af689982815f3f96c4db20 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Fri, 28 Aug 2026 23:24:30 -0700
Subject: [PATCH 11/17] test(bdd): accept platform TLS trust diagnostics

---
 ...-cluster-helmfile-llm-registration-tls-fail-closed.feature | 4 ++--
 tests/bdd/godog_test.go                                       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
index af778e57b..a170b357b 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
@@ -75,7 +75,7 @@ Feature: Reject insecure or invalid LLM worker registration
 
       When I run command:
         """
-        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
@@ -87,7 +87,7 @@ Feature: Reject insecure or invalid LLM worker registration
 
       When I run command:
         """
-        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
         """
       Then the command exit code should be 0
 
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index c5bc2b20b..819be2f9b 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1521,7 +1521,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 	}{
 		{marker: "wrong-root-rejected", diagnostic: "certificate signed by unknown authority"},
 		{marker: "wrong-host-rejected", diagnostic: "not wrong-host.nvcf.svc.cluster.local"},
-		{marker: "missing-trust-rejected", diagnostic: "certificate signed by unknown authority"},
+		{marker: "missing-trust-rejected", diagnostic: "certificate is not trusted"},
 		{marker: "plaintext-rejected", diagnostic: "context deadline exceeded"},
 		{
 			marker: "invalid-authority-rejected",

From 11a22548a1477465a526c415942e1a882edb1a23 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Sun, 30 Aug 2026 00:03:36 -0700
Subject: [PATCH 12/17] test(bdd): expose negative registration command
 failures

Run each invalid registration probe as the user would and assert both its nonzero result and its accepted diagnostic through the shared command DSL.

Refs #1306
Refs #1358

Signed-off-by: Stephanie Baum <sbaum@nvidia.com>
---
 ...e-llm-registration-tls-fail-closed.feature | 41 +++++++----
 tests/bdd/godog_test.go                       | 72 +++++++++++++------
 2 files changed, 80 insertions(+), 33 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
index a170b357b..59075b3b0 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
@@ -75,30 +75,45 @@ Feature: Reject insecure or invalid LLM worker registration
 
       When I run command:
         """
-        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
         """
-      Then the command exit code should be 0
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                    |
+        | certificate signed by unknown authority |
+        | certificate is not trusted              |
 
       When I run command:
         """
-        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong hostname was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"not wrong-host.nvcf.svc.cluster.local"*) printf "wrong-host-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
         """
-      Then the command exit code should be 0
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                         |
+        | not wrong-host.nvcf.svc.cluster.local        |
 
       When I run command:
         """
-        /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
         """
-      Then the command exit code should be 0
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                    |
+        | certificate signed by unknown authority |
+        | certificate is not trusted              |
 
       When I run command:
         """
-        /bin/bash -c 'if diagnostic=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "plaintext was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"context deadline exceeded"*|*"error reading server preface"*) printf "plaintext-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
+        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
         """
-      Then the command exit code should be 0
+      Then the command should fail
+      And the command output should contain one of:
+        | text                         |
+        | context deadline exceeded    |
+        | error reading server preface |
 
-      When I run command:
-        """
-        /bin/sh -c 'if diagnostic=$(make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority 2>&1); then printf "invalid authority was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535"*) printf "invalid-authority-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
-        """
-      Then the command exit code should be 0
+      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                                                                                                                                    |
+        | global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535 |
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 819be2f9b..f63acd298 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1482,6 +1482,28 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 		`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
 		`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
 	const grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'`
+	const wrongRootCommand = `/bin/bash -c 'set -u; cert_dir=$(mktemp -d); ` +
+		`trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes ` +
+		`-subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 ` +
+		`>/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" ` +
+		`-authority llm-request-router.nvcf.svc.cluster.local ` +
+		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+	const wrongHostCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
+		`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
+		`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
+		`-authority wrong-host.nvcf.svc.cluster.local ` +
+		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+	const missingTrustCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
+		`-authority llm-request-router.nvcf.svc.cluster.local ` +
+		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+	const plaintextCommand = `/bin/bash -c 'grpcurl -plaintext -max-time 5 ` +
+		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+	const invalidAuthorityCommand = "make -C deploy/stacks/self-managed template " +
+		"HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
 	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
 		"k3d cluster get ncp-local": {ExitCode: 1},
 		grpcurlPreflightCommand:     {ExitCode: 0},
@@ -1489,6 +1511,28 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 			ExitCode: 0,
 			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
 		},
+		wrongRootCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate signed by unknown authority\n",
+		},
+		wrongHostCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate is valid for another name, not wrong-host.nvcf.svc.cluster.local\n",
+		},
+		missingTrustCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate is not trusted\n",
+		},
+		plaintextCommand: {
+			ExitCode: 1,
+			Stderr:   "context deadline exceeded\n",
+		},
+		invalidAuthorityCommand: {
+			ExitCode: 1,
+			Stderr: "global.workerEndpoints.llmRequestRouterAddress must use " +
+				"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
+				"with port 1-65535\n",
+		},
 	}))
 	seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot)
 	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
@@ -1515,27 +1559,15 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 	if !commandRanExactly(runs, grpcurlPreflightCommand) {
 		t.Fatal("grpcurl availability was not checked before the live probes")
 	}
-	for _, assertion := range []struct {
-		marker     string
-		diagnostic string
-	}{
-		{marker: "wrong-root-rejected", diagnostic: "certificate signed by unknown authority"},
-		{marker: "wrong-host-rejected", diagnostic: "not wrong-host.nvcf.svc.cluster.local"},
-		{marker: "missing-trust-rejected", diagnostic: "certificate is not trusted"},
-		{marker: "plaintext-rejected", diagnostic: "context deadline exceeded"},
-		{
-			marker: "invalid-authority-rejected",
-			diagnostic: "global.workerEndpoints.llmRequestRouterAddress must use " +
-				"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
-				"with port 1-65535",
-		},
+	for name, command := range map[string]string{
+		"wrong root":        wrongRootCommand,
+		"wrong host":        wrongHostCommand,
+		"missing trust":     missingTrustCommand,
+		"plaintext":         plaintextCommand,
+		"invalid authority": invalidAuthorityCommand,
 	} {
-		if !commandRanThatContainsAll(runs, assertion.marker, assertion.diagnostic) {
-			t.Fatalf(
-				"negative registration command containing %q did not require diagnostic %q",
-				assertion.marker,
-				assertion.diagnostic,
-			)
+		if !commandRanExactly(runs, command) {
+			t.Fatalf("%s negative registration command was not invoked", name)
 		}
 	}
 	validEnvironment := filepath.Join(

From af37f7bbee5085da288ac4cf60a7a81ed7a021bc Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Sun, 30 Aug 2026 09:52:17 -0700
Subject: [PATCH 13/17] test(bdd): combine secure registration coverage

Keep the trusted and fail-closed registration paths in one feature while preserving independent, explicit setup for each negative case.

Refs #1305
Refs #1306
Refs #1292

Signed-off-by: Stephanie Baum <sbaum@nvidia.com>
---
 ...e-llm-registration-tls-fail-closed.feature | 119 -----------
 ...ster-helmfile-llm-registration-tls.feature | 194 +++++++++++++-----
 tests/bdd/godog_test.go                       | 165 +++++----------
 3 files changed, 193 insertions(+), 285 deletions(-)
 delete mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
deleted file mode 100644
index 59075b3b0..000000000
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature
+++ /dev/null
@@ -1,119 +0,0 @@
-@ncp-local @multi-cluster @helmfile @pki @llm-registration @negative
-Feature: Reject insecure or invalid LLM worker registration
-  As a self-managed NVCF operator,
-  I want the Pylon registration endpoint to fail closed,
-  so that invalid trust or authority cannot silently enter the routing plane.
-
-  Rule: Every rejected path is observable at the operator boundary
-
-    Background:
-      Given these environment variables are set:
-        | name            |
-        | NGC_API_KEY     |
-        | SAMPLE_NGC_ORG  |
-        | SAMPLE_NGC_TEAM |
-      And I prepare Helmfile environment "local-bdd-registration-tls-fail-closed" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
-        | global.imagePullSecrets[0].name                            | nvcr-pull-secret                                                     |
-        | global.helm.sources.repository                             | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
-        | global.image.repository                                    | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
-        | global.workerEndpoints.llmRequestRouterAddress             | https://llm-request-router.nvcf.svc.cluster.local:50071              |
-        | addons.llm.requestRouter.workload.kind                      | StatefulSet                                                          |
-        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071              |
-        | observability.profile                                      | disabled                                                             |
-      And I prepare Helmfile environment "local-bdd-registration-tls-invalid-authority" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
-        | global.imagePullSecrets[0].name                            | nvcr-pull-secret                                                     |
-        | global.helm.sources.repository                             | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
-        | global.image.repository                                    | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
-        | global.workerEndpoints.llmRequestRouterAddress             | https://llm_request_router.nvcf.svc.cluster.local:50071              |
-        | addons.llm.requestRouter.workload.kind                      | StatefulSet                                                          |
-        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm_request_router.nvcf.svc.cluster.local:50071              |
-        | observability.profile                                      | disabled                                                             |
-      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-fail-closed-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
-      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
-      When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
-      Then the command exit code should be 0
-      When I run command "k3d cluster get ncp-local"
-      Then the command exit code should be 1
-      And multi-cluster ncp-local compute clusters are running:
-        | ncp-local-compute-1 |
-      And command has succeeded:
-        """
-        kubectl config use-context k3d-ncp-local-cp
-        """
-      And the "nvcr-pull-secret" image pull secret exists in namespaces:
-        | cassandra-system |
-        | nats-system      |
-        | nvcf             |
-        | api-keys         |
-        | ess              |
-        | sis              |
-        | vault-system     |
-        | nvca-operator    |
-        | cert-manager     |
-
-    Scenario: Registration rejects every untrusted client path
-      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-fail-closed"
-      Then the command exit code should be 0
-      When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls-fail-closed"
-      Then the command exit code should be 0
-      When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
-      Then the command exit code should be 0
-      When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
-      Then the command exit code should be 0
-      When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
-      Then the command exit code should be 0
-
-      # Establish that the TLS listener is reachable with its issued root,
-      # expected DNS identity, and HTTP/2 application protocol.
-      When I run command:
-        """
-        /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
-        """
-      Then the command exit code should be 0
-      And the command output should contain "Verify return code: 0 (ok)"
-      And the command output should contain "ALPN protocol: h2"
-
-      When I run command:
-        """
-        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                    |
-        | certificate signed by unknown authority |
-        | certificate is not trusted              |
-
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                         |
-        | not wrong-host.nvcf.svc.cluster.local        |
-
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                    |
-        | certificate signed by unknown authority |
-        | certificate is not trusted              |
-
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                         |
-        | context deadline exceeded    |
-        | error reading server preface |
-
-      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                                                                                                                                    |
-        | global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535 |
diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 4979e59c2..789fd9c63 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -1,73 +1,70 @@
 @ncp-local @multi-cluster @helmfile @pki @llm-registration
-Feature: Register an LLM worker securely with every router in a local split-cluster stack
+Feature: Register an LLM worker securely with a local split-cluster routing plane
   As a self-managed NVCF operator,
   I want Pylon registration to use the stack-issued TLS identity across clusters,
   so that plaintext or untrusted registration cannot silently enter the routing plane.
 
-  Rule: Secure registration is observable from the operator boundary
+  Background:
+    Given these environment variables are set:
+      | name            |
+      | NGC_API_KEY     |
+      | NVCF_CLI        |
+      | REPO_ROOT       |
+      | SAMPLE_NGC_ORG  |
+      | SAMPLE_NGC_TEAM |
+    And I prepare Helmfile environment "local-bdd-registration-tls" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
+      | global.imagePullSecrets[0].name                          | nvcr-pull-secret                                                            |
+      | global.helm.sources.repository                           | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
+      | global.image.repository                                  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
+      | global.workerEndpoints.llmRequestRouterAddress           | https://llm-request-router.nvcf.svc.cluster.local:50071                       |
+      | addons.llm.requestRouter.workload.kind                    | StatefulSet                                                                 |
+      | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
+      | observability.profile                                    | disabled                                                                     |
+    And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+    When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
+    Then the command exit code should be 0
+    # Conflict precheck: the single-cluster topology owns the same host
+    # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local
+    # before retrying. k3d v5 exits 1 when the cluster is absent.
+    When I run command "k3d cluster get ncp-local"
+    Then the command exit code should be 1
+    And multi-cluster ncp-local compute clusters are running:
+      | ncp-local-compute-1 |
+    And command has succeeded:
+      """
+      kubectl config use-context k3d-ncp-local-cp
+      """
+    And the "nvcr-pull-secret" image pull secret exists in namespaces:
+      | cassandra-system |
+      | nats-system      |
+      | nvcf             |
+      | api-keys         |
+      | ess              |
+      | sis              |
+      | vault-system     |
+      | nvca-operator    |
+      | cert-manager     |
 
-    Background:
-      Given these environment variables are set:
-        | name            |
-        | NGC_API_KEY     |
-        | NVCF_CLI        |
-        | REPO_ROOT       |
-        | SAMPLE_NGC_ORG  |
-        | SAMPLE_NGC_TEAM |
-      And I prepare Helmfile environment "local-bdd-registration-tls" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
-        | global.imagePullSecrets[0].name                          | nvcr-pull-secret                                                            |
-        | global.helm.sources.repository                           | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
-        | global.image.repository                                  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                       |
-        | global.workerEndpoints.llmRequestRouterAddress           | https://llm-request-router.nvcf.svc.cluster.local:50071                       |
-        | addons.llm.requestRouter.workload.kind                    | StatefulSet                                                                 |
-        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
-        | observability.profile                                    | disabled                                                                     |
-      And I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values:
+  Rule: Trusted registration is observable from the operator boundary
+
+    @llm-registration-tls-install @llm-registration-tls-runtime
+    Scenario: A trusted Pylon registers with every router and serves an authenticated request
+      Given I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values:
         | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
         | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
         | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
         | observability.profile           | disabled                             |
-      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
-      # Conflict precheck: the single-cluster topology owns the same host
-      # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local
-      # before retrying. k3d v5 exits 1 when the cluster is absent.
-      When I run command "k3d cluster get ncp-local"
-      Then the command exit code should be 1
-      And multi-cluster ncp-local compute clusters are running:
-        | ncp-local-compute-1 |
-      And command has succeeded:
-        """
-        kubectl config use-context k3d-ncp-local-cp
-        """
-      And the "nvcr-pull-secret" image pull secret exists in namespaces:
-        | cassandra-system |
-        | nats-system      |
-        | nvcf             |
-        | api-keys         |
-        | ess              |
-        | sis              |
-        | vault-system     |
-        | nvca-operator    |
-        | cert-manager     |
-
-    @llm-registration-tls-install
-    Scenario: Operator installs a TLS-only registration endpoint with three concrete routers
-      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls"
-      Then the command exit code should be 0
+      When I successfully run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls"
       And the rendered manifests in "deploy/stacks/self-managed/out" should contain:
         | text                                                                                 |
         | https://llm-request-router.nvcf.svc.cluster.local:50071                              |
         | --grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071 |
 
-      When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls"
-      Then the command exit code should be 0
+      When I successfully run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls"
 
-      When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
-      Then the command exit code should be 0
-      When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
-      Then the command exit code should be 0
-      When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
-      Then the command exit code should be 0
+      When I successfully run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
+      When I successfully run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
+      When I successfully run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
 
       When I run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml"
       Then the command exit code should be 0
@@ -134,8 +131,6 @@ Feature: Register an LLM worker securely with every router in a local split-clus
       Then the command exit code should be 0
       Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m"
 
-    @llm-registration-tls-runtime
-    Scenario: Pylon registers with every router and serves an authenticated LLM request
       Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml"
       When I successfully create function "bdd-registration-tls" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options:
         | option           | value                                                                                               |
@@ -174,3 +169,90 @@ Feature: Register an LLM worker securely with every router in a local split-clus
         | chat.completion         |
         | fixed 128-byte response |
       And I successfully undeploy the function selected by NVCF CLI
+
+  @negative
+  Rule: Registration rejects clients that do not satisfy the TLS contract
+
+    Background:
+      Given command has succeeded:
+        """
+        make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls
+        """
+      And command has succeeded:
+        """
+        make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls
+        """
+      And command has succeeded:
+        """
+        kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m
+        """
+      And command has succeeded:
+        """
+        kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m
+        """
+      And command has succeeded:
+        """
+        kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m
+        """
+
+    Scenario: Registration rejects a client with an untrusted root
+      When I run command:
+        """
+        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        """
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                    |
+        | certificate signed by unknown authority |
+        | certificate is not trusted              |
+
+    Scenario: Registration rejects a client using the wrong authority
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        """
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                  |
+        | not wrong-host.nvcf.svc.cluster.local |
+
+    Scenario: Registration rejects a client without the issued root
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        """
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                    |
+        | certificate signed by unknown authority |
+        | certificate is not trusted              |
+
+    Scenario: Registration rejects a plaintext client
+      When I run command:
+        """
+        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
+        """
+      Then the command should fail
+      And the command output should contain one of:
+        | text                      |
+        | context deadline exceeded |
+        | error reading server preface |
+
+  @negative
+  Rule: Invalid registration authorities fail before installation
+
+    Scenario: Operator cannot render an invalid worker authority
+      Given I prepare Helmfile environment "local-bdd-registration-tls-invalid-authority" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values:
+        | global.imagePullSecrets[0].name                            | nvcr-pull-secret                                                     |
+        | global.helm.sources.repository                             | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.image.repository                                    | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                                |
+        | global.workerEndpoints.llmRequestRouterAddress             | https://llm_request_router.nvcf.svc.cluster.local:50071              |
+        | addons.llm.requestRouter.workload.kind                      | StatefulSet                                                          |
+        | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm_request_router.nvcf.svc.cluster.local:50071              |
+        | observability.profile                                      | disabled                                                             |
+      And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+      When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
+      Then the command should fail
+      And the command output should contain one of:
+        | text                                                                                                                                                 |
+        | global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535 |
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index f63acd298..3ba949fcc 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1362,7 +1362,8 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 	t.Setenv("REPO_ROOT", "/repo-root-placeholder")
 
 	const (
-		tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
+		grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'`
+		tlsHandshakeCommand     = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
 			`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
 			`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
 			`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
@@ -1382,10 +1383,33 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
 			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
 			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120"
+		wrongRootCommand = `/bin/bash -c 'set -u; cert_dir=$(mktemp -d); ` +
+			`trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes ` +
+			`-subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 ` +
+			`>/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" ` +
+			`-authority llm-request-router.nvcf.svc.cluster.local ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+		wrongHostCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
+			`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
+			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
+			`-authority wrong-host.nvcf.svc.cluster.local ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+		missingTrustCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
+			`-authority llm-request-router.nvcf.svc.cluster.local ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+		plaintextCommand = `/bin/bash -c 'grpcurl -plaintext -max-time 5 ` +
+			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
+			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
+		invalidAuthorityCommand = "make -C deploy/stacks/self-managed template " +
+			"HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
 	)
 
 	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
 		"k3d cluster get ncp-local": {ExitCode: 1},
+		grpcurlPreflightCommand:     {ExitCode: 0},
 		"kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml": {
 			ExitCode: 0,
 			Stdout:   "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071\n",
@@ -1417,6 +1441,28 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 				`{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` +
 				"\n",
 		},
+		wrongRootCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate signed by unknown authority\n",
+		},
+		wrongHostCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate is valid for another name, not wrong-host.nvcf.svc.cluster.local\n",
+		},
+		missingTrustCommand: {
+			ExitCode: 1,
+			Stderr:   "certificate is not trusted\n",
+		},
+		plaintextCommand: {
+			ExitCode: 1,
+			Stderr:   "context deadline exceeded\n",
+		},
+		invalidAuthorityCommand: {
+			ExitCode: 1,
+			Stderr: "global.workerEndpoints.llmRequestRouterAddress must use " +
+				"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
+				"with port 1-65535\n",
+		},
 	}))
 	seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot)
 	seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot)
@@ -1467,94 +1513,6 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 	if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) {
 		t.Fatal("plaintext WatchStargates rejection was not exercised")
 	}
-}
-
-// TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
-// runs the negative TLS registration matrix against a fake runner.
-func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps(t *testing.T) {
-	t.Setenv("NGC_API_KEY", "test-key")
-	t.Setenv("SAMPLE_NGC_ORG", "test-org")
-	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
-
-	const tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
-		`-servername llm-request-router.nvcf.svc.cluster.local ` +
-		`-verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
-		`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
-		`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
-	const grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'`
-	const wrongRootCommand = `/bin/bash -c 'set -u; cert_dir=$(mktemp -d); ` +
-		`trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes ` +
-		`-subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 ` +
-		`>/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" ` +
-		`-authority llm-request-router.nvcf.svc.cluster.local ` +
-		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-	const wrongHostCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
-		`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
-		`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
-		`-authority wrong-host.nvcf.svc.cluster.local ` +
-		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-	const missingTrustCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
-		`-authority llm-request-router.nvcf.svc.cluster.local ` +
-		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-	const plaintextCommand = `/bin/bash -c 'grpcurl -plaintext -max-time 5 ` +
-		`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-		`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-	const invalidAuthorityCommand = "make -C deploy/stacks/self-managed template " +
-		"HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
-	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
-		"k3d cluster get ncp-local": {ExitCode: 1},
-		grpcurlPreflightCommand:     {ExitCode: 0},
-		tlsHandshakeCommand: {
-			ExitCode: 0,
-			Stdout:   "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
-		},
-		wrongRootCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate signed by unknown authority\n",
-		},
-		wrongHostCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate is valid for another name, not wrong-host.nvcf.svc.cluster.local\n",
-		},
-		missingTrustCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate is not trusted\n",
-		},
-		plaintextCommand: {
-			ExitCode: 1,
-			Stderr:   "context deadline exceeded\n",
-		},
-		invalidAuthorityCommand: {
-			ExitCode: 1,
-			Stderr: "global.workerEndpoints.llmRequestRouterAddress must use " +
-				"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
-				"with port 1-65535\n",
-		},
-	}))
-	seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot)
-	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
-
-	sc := steps.NewScenarioContext(suite)
-	featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature")
-	var out strings.Builder
-	status := godog.TestSuite{
-		Name: "multi-cluster-helmfile-llm-registration-tls-fail-closed-wiring",
-		ScenarioInitializer: func(ctx *godog.ScenarioContext) {
-			steps.RegisterAll(ctx, sc)
-		},
-		Options: &godog.Options{
-			Format: "pretty",
-			Paths:  []string{featurePath},
-			Strict: true,
-			Output: &out,
-		},
-	}.Run()
-	if status != 0 {
-		t.Fatalf("godog suite status = %d\n%s", status, out.String())
-	}
 	runs := suite.Runner.(*fakeRunner).runs
 	if !commandRanExactly(runs, grpcurlPreflightCommand) {
 		t.Fatal("grpcurl availability was not checked before the live probes")
@@ -1570,22 +1528,18 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
 			t.Fatalf("%s negative registration command was not invoked", name)
 		}
 	}
-	validEnvironment := filepath.Join(
-		suite.Config.RepoRoot,
-		"deploy", "stacks", "self-managed", "environments",
-		"local-bdd-registration-tls-fail-closed.yaml",
-	)
-	invalidEnvironment := filepath.Join(
-		suite.Config.RepoRoot,
-		"deploy", "stacks", "self-managed", "environments",
-		"local-bdd-registration-tls-invalid-authority.yaml",
-	)
 	for _, assertion := range []struct {
 		path string
 		want string
 	}{
-		{path: validEnvironment, want: "https://llm-request-router.nvcf.svc.cluster.local:50071"},
-		{path: invalidEnvironment, want: "https://llm_request_router.nvcf.svc.cluster.local:50071"},
+		{
+			path: filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "self-managed", "environments", "local-bdd-registration-tls.yaml"),
+			want: "https://llm-request-router.nvcf.svc.cluster.local:50071",
+		},
+		{
+			path: filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "self-managed", "environments", "local-bdd-registration-tls-invalid-authority.yaml"),
+			want: "https://llm_request_router.nvcf.svc.cluster.local:50071",
+		},
 	} {
 		got, found, err := dsl.ReadYAMLKey(assertion.path, "global.workerEndpoints.llmRequestRouterAddress")
 		if err != nil {
@@ -2382,15 +2336,6 @@ func TestMultiClusterHelmfileLLMRegistrationTLS(t *testing.T) {
 	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls.feature")
 }
 
-// TestMultiClusterHelmfileLLMRegistrationTLSFailClosed is the live entry
-// point for the negative TLS registration matrix. Skipped under -short.
-func TestMultiClusterHelmfileLLMRegistrationTLSFailClosed(t *testing.T) {
-	if testing.Short() {
-		t.Skip("live run skipped under -short")
-	}
-	runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature")
-}
-
 // TestSingleClusterEKSHelmfile is the live entry point for the
 // single-cluster EKS Helmfile feature. Skipped under -short.
 func TestSingleClusterEKSHelmfile(t *testing.T) {

From 0769f296cb6110864496ad02ae81fc7ed0811649 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Mon, 31 Aug 2026 00:40:33 -0700
Subject: [PATCH 14/17] test(bdd): simplify secure registration coverage

---
 ...ster-helmfile-llm-registration-tls.feature | 83 ++-----------------
 tests/bdd/godog_test.go                       | 48 +----------
 2 files changed, 9 insertions(+), 122 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 789fd9c63..19b57483f 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -55,20 +55,19 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
         | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
         | observability.profile           | disabled                             |
       When I successfully run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls"
-      And the rendered manifests in "deploy/stacks/self-managed/out" should contain:
+      Then the rendered manifests in "deploy/stacks/self-managed/out" should contain:
         | text                                                                                 |
         | https://llm-request-router.nvcf.svc.cluster.local:50071                              |
         | --grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071 |
 
       When I successfully run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls"
 
-      When I successfully run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
-      When I successfully run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
-      When I successfully run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
+      And I successfully run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
+      And I successfully run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
+      And I successfully run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m"
 
-      When I run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml"
-      Then the command exit code should be 0
-      And the command output should contain "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071"
+      And I successfully run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml"
+      Then the command output should contain "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071"
 
       # openssl verifies the externally reachable listener against the same
       # stack-issued CA and DNS identity that a compute-plane Pylon uses.
@@ -129,7 +128,7 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
         make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-tls COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI}
         """
       Then the command exit code should be 0
-      Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m"
+      And NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m"
 
       Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml"
       When I successfully create function "bdd-registration-tls" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options:
@@ -170,74 +169,6 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
         | fixed 128-byte response |
       And I successfully undeploy the function selected by NVCF CLI
 
-  @negative
-  Rule: Registration rejects clients that do not satisfy the TLS contract
-
-    Background:
-      Given command has succeeded:
-        """
-        make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls
-        """
-      And command has succeeded:
-        """
-        make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls
-        """
-      And command has succeeded:
-        """
-        kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m
-        """
-      And command has succeeded:
-        """
-        kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m
-        """
-      And command has succeeded:
-        """
-        kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m
-        """
-
-    Scenario: Registration rejects a client with an untrusted root
-      When I run command:
-        """
-        /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                    |
-        | certificate signed by unknown authority |
-        | certificate is not trusted              |
-
-    Scenario: Registration rejects a client using the wrong authority
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                  |
-        | not wrong-host.nvcf.svc.cluster.local |
-
-    Scenario: Registration rejects a client without the issued root
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                                    |
-        | certificate signed by unknown authority |
-        | certificate is not trusted              |
-
-    Scenario: Registration rejects a plaintext client
-      When I run command:
-        """
-        /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'
-        """
-      Then the command should fail
-      And the command output should contain one of:
-        | text                      |
-        | context deadline exceeded |
-        | error reading server preface |
-
   @negative
   Rule: Invalid registration authorities fail before installation
 
diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go
index 3ba949fcc..ae054b555 100644
--- a/tests/bdd/godog_test.go
+++ b/tests/bdd/godog_test.go
@@ -1383,26 +1383,6 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
 			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
 			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120"
-		wrongRootCommand = `/bin/bash -c 'set -u; cert_dir=$(mktemp -d); ` +
-			`trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes ` +
-			`-subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 ` +
-			`>/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" ` +
-			`-authority llm-request-router.nvcf.svc.cluster.local ` +
-			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-		wrongHostCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
-			`-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
-			`-o jsonpath="{.data.ca\.crt}" | base64 -d) ` +
-			`-authority wrong-host.nvcf.svc.cluster.local ` +
-			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-		missingTrustCommand = `/bin/bash -c 'grpcurl -max-time 5 ` +
-			`-authority llm-request-router.nvcf.svc.cluster.local ` +
-			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
-		plaintextCommand = `/bin/bash -c 'grpcurl -plaintext -max-time 5 ` +
-			`-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` +
-			`127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'`
 		invalidAuthorityCommand = "make -C deploy/stacks/self-managed template " +
 			"HELMFILE_ENV=local-bdd-registration-tls-invalid-authority"
 	)
@@ -1441,22 +1421,6 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 				`{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` +
 				"\n",
 		},
-		wrongRootCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate signed by unknown authority\n",
-		},
-		wrongHostCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate is valid for another name, not wrong-host.nvcf.svc.cluster.local\n",
-		},
-		missingTrustCommand: {
-			ExitCode: 1,
-			Stderr:   "certificate is not trusted\n",
-		},
-		plaintextCommand: {
-			ExitCode: 1,
-			Stderr:   "context deadline exceeded\n",
-		},
 		invalidAuthorityCommand: {
 			ExitCode: 1,
 			Stderr: "global.workerEndpoints.llmRequestRouterAddress must use " +
@@ -1517,16 +1481,8 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin
 	if !commandRanExactly(runs, grpcurlPreflightCommand) {
 		t.Fatal("grpcurl availability was not checked before the live probes")
 	}
-	for name, command := range map[string]string{
-		"wrong root":        wrongRootCommand,
-		"wrong host":        wrongHostCommand,
-		"missing trust":     missingTrustCommand,
-		"plaintext":         plaintextCommand,
-		"invalid authority": invalidAuthorityCommand,
-	} {
-		if !commandRanExactly(runs, command) {
-			t.Fatalf("%s negative registration command was not invoked", name)
-		}
+	if !commandRanExactly(runs, invalidAuthorityCommand) {
+		t.Fatal("invalid registration authority was not rejected before installation")
 	}
 	for _, assertion := range []struct {
 		path string

From b6ca09a4d4f6b93ba150b041d2e45513022d25be Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Mon, 31 Aug 2026 09:54:48 -0700
Subject: [PATCH 15/17] test(bdd): track grpcurl preflight cleanup

Refs #1411
---
 .../features/multi-cluster-helmfile-llm-registration-tls.feature | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 19b57483f..9450ec19d 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -21,6 +21,7 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
       | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071                     |
       | observability.profile                                    | disabled                                                                     |
     And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
+    # Explore a shared BDD preflight so feature files need not repeat this check: https://github.com/NVIDIA/nvcf/issues/1411
     When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
     Then the command exit code should be 0
     # Conflict precheck: the single-cluster topology owns the same host

From 040cc237c6195155359f9b78e1477e3ee0250548 Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Mon, 31 Aug 2026 09:56:28 -0700
Subject: [PATCH 16/17] test(bdd): track TLS probe DSL cleanup

Refs #1412
---
 .../multi-cluster-helmfile-llm-registration-tls.feature         | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index 9450ec19d..ec656ed08 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -72,6 +72,7 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
 
       # openssl verifies the externally reachable listener against the same
       # stack-issued CA and DNS identity that a compute-plane Pylon uses.
+      # Explore a readable shared BDD DSL for this TLS listener probe: https://github.com/NVIDIA/nvcf/issues/1412
       When I run command:
         """
         /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
@@ -83,6 +84,7 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
       # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is
       # sent to this verified TLS listener. The trusted Watch below proves
       # that the same endpoint remains healthy.
+      # Explore a readable shared BDD DSL for this plaintext rejection probe: https://github.com/NVIDIA/nvcf/issues/1412
       When I run command:
         """
         /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'

From 61f92e78772412f019c6446c3940333a34ee9a9d Mon Sep 17 00:00:00 2001
From: Stephanie Baum <sbaum@nvidia.com>
Date: Mon, 31 Aug 2026 11:54:28 -0700
Subject: [PATCH 17/17] test(bdd): require successful registration commands

Refs #1292
---
 ...ster-helmfile-llm-registration-tls.feature | 31 +++++++------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
index ec656ed08..f0bdd1075 100644
--- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
+++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature
@@ -22,8 +22,7 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
       | observability.profile                                    | disabled                                                                     |
     And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
     # Explore a shared BDD preflight so feature files need not repeat this check: https://github.com/NVIDIA/nvcf/issues/1411
-    When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
-    Then the command exit code should be 0
+    When I successfully run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
     # Conflict precheck: the single-cluster topology owns the same host
     # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local
     # before retrying. k3d v5 exits 1 when the cluster is absent.
@@ -73,24 +72,22 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
       # openssl verifies the externally reachable listener against the same
       # stack-issued CA and DNS identity that a compute-plane Pylon uses.
       # Explore a readable shared BDD DSL for this TLS listener probe: https://github.com/NVIDIA/nvcf/issues/1412
-      When I run command:
+      When I successfully run command:
         """
         /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
         """
-      Then the command exit code should be 0
-      And the command output should contain "Verify return code: 0 (ok)"
+      Then the command output should contain "Verify return code: 0 (ok)"
       And the command output should contain "ALPN protocol: h2"
 
       # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is
       # sent to this verified TLS listener. The trusted Watch below proves
       # that the same endpoint remains healthy.
       # Explore a readable shared BDD DSL for this plaintext rejection probe: https://github.com/NVIDIA/nvcf/issues/1412
-      When I run command:
+      When I successfully run command:
         """
         /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'
         """
-      Then the command exit code should be 0
-      And the command output should contain "plaintext-watch-rejected=tls-listener-timeout"
+      Then the command output should contain "plaintext-watch-rejected=tls-listener-timeout"
 
       When I successfully observe WatchStargates at "127.0.0.1:50071" with TLS authority "llm-request-router.nvcf.svc.cluster.local" using CA secret "stargate-quic-tls" in namespace "nvcf" and context "k3d-ncp-local-cp" for "3" seconds
       Then the command output should contain all:
@@ -100,37 +97,33 @@ Feature: Register an LLM worker securely with a local split-cluster routing plan
         | llm-request-router-2                                                    |
         | https://llm-request-router.nvcf.svc.cluster.local:50071                |
 
-      When I run command:
+      When I successfully run command:
         """
         ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-tls --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp
         """
-      Then the command exit code should be 0
-      And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist
+      Then file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist
       And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys:
         | key                                 |
         | managementTls.caBundlePem           |
         | transportTls.trustBundleFingerprint |
         | transportTls.trustBundlePem         |
 
-      And command has succeeded:
+      When command has succeeded:
         """
         /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null'
         """
-      When I run command "kubectl config use-context k3d-ncp-local-compute-1"
-      Then the command exit code should be 0
-      When I run command:
+      And I successfully run command "kubectl config use-context k3d-ncp-local-compute-1"
+      And I successfully run command:
         """
         make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
         """
-      Then the command exit code should be 0
-      And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-compute-1-register-values.yaml" should exist
+      Then file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-compute-1-register-values.yaml" should exist
       And the "nvcr-pull-secret" image pull secret exists in namespaces:
         | nvca-operator |
-      When I run command:
+      When I successfully run command:
         """
         make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-tls COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI}
         """
-      Then the command exit code should be 0
       And NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m"
 
       Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml"