chore(stack): update secure LLM component pins

ai-tooling/user/skills/nvcf-self-managed-installation/references/helmfile-structure.md

@@ -208,10 +208,17 @@ repository, set the generated worker sidecar image explicitly:
 
 ```yaml
 api:
-  env:
-    NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE: <registry>/<repository>/pylon:0.2.1
+  remoteConfig:
+    configData:
+      nvcf:
+        sidecars:
+          llm-router-client-image: <registry>/<repository>/pylon:0.14.1
 ```
 
+The legacy `api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE` path is deprecated.
+The stack translates it for one compatibility window, but new configurations
+must use the remote-config path. Conflicting values fail rendering.
+
 Render and apply the updated control-plane environment, then refresh every
 registered compute plane using the same handoff used for its installation so
 NVCA receives `agentConfig.mergeConfig`:
@@ -226,9 +233,11 @@ Use the complete compute-plane install command from
 CLI-profile installation must remain profile-driven; a Helmfile installation
 must remain values-driven.
 
-Existing LLM function pods keep their existing sidecar arguments. Recreate or
-redeploy those functions after the compute-plane refresh. Verify the control
-plane, route, and worker sidecar:
+Existing LLM function versions retain the worker-sidecar image metadata
+captured when the version is created. Replacing pods or redeploying the same
+version does not apply a new Pylon image. After the control-plane update,
+create and deploy a new function version. Verify the control plane, route, and
+worker sidecar:
 
 ```bash
 kubectl get deploy -n nvcf llm-api-gateway

deploy/helm/cloud-functions/nvcf-api/values.yaml

@@ -245,7 +245,7 @@ api:
           ess-agent-container: ${nvcf.sidecars.hostname}/${nvcf.sidecars.repository}/ess-agent:1.4.0
           otel-collector-container: dummy
           llm-credential-manager-image: ${nvcf.sidecars.hostname}/${nvcf.sidecars.repository}/nvcf-worker-llm-credentials-oss:1.1.1
-          llm-router-client-image: ${nvcf.sidecars.hostname}/${nvcf.sidecars.repository}/pylon:0.10.0
+          llm-router-client-image: ${nvcf.sidecars.hostname}/${nvcf.sidecars.repository}/pylon:0.14.1
         notary:
           turn:
             enabled: true

deploy/helm/cloud-functions/tests/sidecar_release_artifacts_test.sh

@@ -53,7 +53,7 @@ expected_annotations=(
   "release-artifact-niclls-container-image: \"${hostname}/${repository}/nvcf_worker_niclls:2.109.4\""
   "release-artifact-ess-agent-container-image: \"${hostname}/${repository}/ess-agent:1.4.0\""
   "release-artifact-llm-credential-manager-image: \"${hostname}/${repository}/nvcf-worker-llm-credentials-oss:1.1.1\""
-  "release-artifact-llm-router-client-image: \"${hostname}/${repository}/pylon:0.10.0\""
+  "release-artifact-llm-router-client-image: \"${hostname}/${repository}/pylon:0.14.1\""
 )
 
 for annotation in "${expected_annotations[@]}"; do

deploy/helm/llm-request-router/llm-request-router/Chart.yaml

@@ -18,4 +18,4 @@ name: helm-nvcf-llm-request-router
 description: Helm chart for the NVCF LLM Request Router backed by Stargate
 type: application
 version: 0.0.0 # managed by semantic-release
-appVersion: "0.11.1" # match the versioned stargate image tag
+appVersion: "0.14.1" # match the versioned stargate image tag

deploy/helm/llm-request-router/llm-request-router/values.yaml

@@ -43,7 +43,7 @@ llmRequestRouter:
   image:
     registry: ""
     repository: ""
-    tag: "0.11.1"
+    tag: "0.14.1"
     pullPolicy: IfNotPresent
 
   imagePullSecrets: []

deploy/helm/llm-request-router/scripts/check-backend-router-render.sh

@@ -303,9 +303,9 @@ if ! grep -Fq -- "serviceAccountName: external-backend-router" "$external_servic
 fi
 assert_backend_router_role_binding_subject "$external_service_account" "external-backend-router"
 
-assert_zero_config_contains "image: registry.example.invalid/nvcf/stargate:0.11.1" \
+assert_zero_config_contains "image: registry.example.invalid/nvcf/stargate:0.14.1" \
   "backend router must inherit the released Stargate image with no router image configuration"
-assert_zero_config_contains 'app.kubernetes.io/version: "0.11.1"' \
+assert_zero_config_contains 'app.kubernetes.io/version: "0.14.1"' \
   "backend router labels must identify the inherited Stargate image version"
 
 helm template llm-request-router "$chart_dir" \

deploy/stacks/self-managed/Makefile

@@ -1,7 +1,7 @@
 # Public developer entry point for the self-managed stack.
 include Makefile.dist
 
-.PHONY: test
+.PHONY: test test-published-charts
 test:
 	@tests/llm-router-worker-address.sh
 	@tests/llm-router-split-cluster.sh
@@ -18,3 +18,11 @@ test:
 	@tests/invocation-tracing-baggage.sh
 	@tests/image-override-wiring.sh
 	@tests/sidecar-registry-wiring.sh
+	@tests/api-env-wiring.sh
+
+test-published-charts:
+	@: "$${NVCF_PUBLISHED_CHART_REGISTRY:?NVCF_PUBLISHED_CHART_REGISTRY is required}"
+	@: "$${NVCF_PUBLISHED_CHART_REPOSITORY:?NVCF_PUBLISHED_CHART_REPOSITORY is required}"
+	@tests/llm-router-published-chart.sh
+	@tests/gateway-routes-published-chart.sh
+	@tests/openbao-published-chart.sh

deploy/stacks/self-managed/environments/base.yaml

@@ -171,12 +171,20 @@ functionAutoscaler:
     # environment label.
     ignoreEnv: true
 
+# API environment entries are merged over the stack defaults. Remote config is
+# passed to the API chart and overrides its packaged application profile. When
+# the LLM addon is enabled, the stack defaults the Pylon image from
+# global.sidecars (or global.image) at version 0.14.1.
+api:
+  env: {}
+  remoteConfig:
+    configData: {}
+
 # Image overrides for the API account-bootstrap job. alpine-k8s is not
 # republished under the public nvidia/nvcf catalog, so it defaults to its
 # upstream source, docker.io/alpine/k8s:1.36.1. Leave these unset unless you
 # have mirrored the image into your own registry, in which case set registry
 # and repository together:
-# api:
 #   accountBootstrap:
 #     image:
 #       registry: <your-registry>
@@ -306,6 +314,11 @@ addons:
     enabled: false
     requestRouter:
       replicaCount: 3
+      # Pin the Stargate release independently of the request-router chart.
+      # Published chart defaults can lag the image release selected by this
+      # stack; backendRouter.image.tag inherits this value when omitted.
+      image:
+        tag: "0.14.1"
       # Set only for source-tree validation. Empty uses the released OCI chart.
       chartPath: ""
       workload:
@@ -339,6 +352,9 @@ addons:
       backendRouter:
         # enabled: defaults to addons.llm.enabled
         replicaCount: 2
+        image:
+          # Empty inherits addons.llm.requestRouter.image.tag.
+          tag: ""
         # Addresses workers dial to reach the router. Both default to the
         # backend-router Service in-cluster, which is correct when workers run
         # alongside the control plane. Set both to externally reachable

deploy/stacks/self-managed/global.yaml.gotmpl

@@ -376,6 +376,81 @@ natsAuthCalloutService:
 {{- end }}
 {{- $nvcfApiGrpcRouteHostnames := dig "ingress" "gatewayApi" "routes" "nvcfApi" "grpc" "hostnames" (list) .Values | default (list (printf "api-grpc.%s" .Values.global.domain)) }}
 {{- $nvctApiGrpcRouteHostnames := dig "ingress" "gatewayApi" "routes" "nvctApi" "grpc" "hostnames" (list) .Values | default (list (printf "tasks-grpc.%s" .Values.global.domain)) }}
+{{- $configuredApiEnv := dig "api" "env" dict .Values }}
+{{- if not (kindIs "map" $configuredApiEnv) }}
+{{- fail "api.env must be a map" }}
+{{- end }}
+{{- $renderedApiEnv := deepCopy $configuredApiEnv }}
+{{- $legacyPylonEnvKey := "NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE" }}
+{{- $legacyPylonImageConfigured := hasKey $renderedApiEnv $legacyPylonEnvKey }}
+{{- $legacyPylonImage := "" }}
+{{- if $legacyPylonImageConfigured }}
+{{- $legacyPylonImage = index $renderedApiEnv $legacyPylonEnvKey }}
+{{- if not (kindIs "string" $legacyPylonImage) }}
+{{- fail "api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE must be a string" }}
+{{- end }}
+{{- if eq (trim $legacyPylonImage) "" }}
+{{- fail "api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE must be a non-empty string" }}
+{{- end }}
+{{- $legacyPylonImage = trim $legacyPylonImage }}
+{{- $_ := unset $renderedApiEnv $legacyPylonEnvKey }}
+{{- end }}
+{{- $configuredApiRemoteConfigData := dig "api" "remoteConfig" "configData" dict .Values }}
+{{- if not (kindIs "map" $configuredApiRemoteConfigData) }}
+{{- fail "api.remoteConfig.configData must be a map" }}
+{{- end }}
+{{- $apiRemoteConfigData := deepCopy $configuredApiRemoteConfigData }}
+{{- $configuredNvcfRemoteConfig := dict }}
+{{- if hasKey $apiRemoteConfigData "nvcf" }}
+{{- $configuredNvcfRemoteConfig = index $apiRemoteConfigData "nvcf" }}
+{{- if not (kindIs "map" $configuredNvcfRemoteConfig) }}
+{{- fail "api.remoteConfig.configData.nvcf must be a map" }}
+{{- end }}
+{{- end }}
+{{- $configuredSidecarsRemoteConfig := dict }}
+{{- if hasKey $configuredNvcfRemoteConfig "sidecars" }}
+{{- $configuredSidecarsRemoteConfig = index $configuredNvcfRemoteConfig "sidecars" }}
+{{- if not (kindIs "map" $configuredSidecarsRemoteConfig) }}
+{{- fail "api.remoteConfig.configData.nvcf.sidecars must be a map" }}
+{{- end }}
+{{- end }}
+{{- $pylonRemoteConfigKey := "llm-router-client-image" }}
+{{- $configuredPylonImagePresent := hasKey $configuredSidecarsRemoteConfig $pylonRemoteConfigKey }}
+{{- $configuredPylonImage := "" }}
+{{- if $configuredPylonImagePresent }}
+{{- $configuredPylonImage = index $configuredSidecarsRemoteConfig $pylonRemoteConfigKey }}
+{{- if not (kindIs "string" $configuredPylonImage) }}
+{{- fail "api.remoteConfig.configData.nvcf.sidecars.llm-router-client-image must be a string" }}
+{{- end }}
+{{- if eq (trim $configuredPylonImage) "" }}
+{{- fail "api.remoteConfig.configData.nvcf.sidecars.llm-router-client-image must be a non-empty string" }}
+{{- end }}
+{{- $configuredPylonImage = trim $configuredPylonImage }}
+{{- $_ := set $configuredSidecarsRemoteConfig $pylonRemoteConfigKey $configuredPylonImage }}
+{{- end }}
+{{- if and $legacyPylonImageConfigured $configuredPylonImagePresent (ne $legacyPylonImage $configuredPylonImage) }}
+{{- fail "api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE is deprecated and conflicts with api.remoteConfig.configData.nvcf.sidecars.llm-router-client-image" }}
+{{- end }}
+{{- $effectivePylonImage := "" }}
+{{- if $configuredPylonImagePresent }}
+{{- $effectivePylonImage = $configuredPylonImage }}
+{{- else if $legacyPylonImageConfigured }}
+{{- $effectivePylonImage = $legacyPylonImage }}
+{{- else if $llmEnabled }}
+{{- $effectivePylonImage = printf "%s/%s/pylon:0.14.1" $sidecarsHost $sidecarsRepo }}
+{{- end }}
+{{- $stackApiRemoteConfigData := dict }}
+{{- $stackNvcfRemoteConfig := dict }}
+{{- if or $llmEnabled $legacyPylonImageConfigured }}
+{{- $_ := set $stackNvcfRemoteConfig "sidecars" (dict $pylonRemoteConfigKey $effectivePylonImage) }}
+{{- end }}
+{{- if $llmEnabled }}
+{{- $_ := set $stackNvcfRemoteConfig "llm-request-router" (dict "worker-address" $llmRequestRouterWorkerAddress) }}
+{{- end }}
+{{- if $stackNvcfRemoteConfig }}
+{{- $_ := set $stackApiRemoteConfigData "nvcf" $stackNvcfRemoteConfig }}
+{{- end }}
+{{- $apiRemoteConfigData = mergeOverwrite $apiRemoteConfigData $stackApiRemoteConfigData }}
 api:
   fullnameOverride: nvcf-api
   {{- if .Values.global.imagePullSecrets }}
@@ -396,12 +471,10 @@ api:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- if $llmEnabled }}
+  {{- if $apiRemoteConfigData }}
   remoteConfig:
     configData:
-      nvcf:
-        llm-request-router:
-          worker-address: {{ $llmRequestRouterWorkerAddress | quote }}
+      {{- toYaml $apiRemoteConfigData | nindent 6 }}
   {{- end }}
   # alpine-k8s is not republished under the public nvidia/nvcf catalog, so it
   # defaults to its upstream Docker Hub source and a public-catalog install
@@ -442,38 +515,44 @@ api:
       {{- end }}
       maxRegistryCreds: {{ $maxRegistryCreds }}
 
+  {{- /* Build the fixed API environment first, then merge configured api.env
+         last so operators can override a default without emitting duplicate
+         YAML keys. toYaml provides deterministic key ordering and treats
+         configured values as data rather than evaluating them as templates. */}}
+  {{- $apiEnv := dict
+        "NVCF_NATS_REGION_PLACEMENT_TAG" "dc"
+        "NVCF_SIDECARS_HOSTNAME" $sidecarsHost
+        "NVCF_SIDECARS_REPOSITORY" $sidecarsRepo
+        "MANAGEMENT_TRACING_ENABLED" (printf "%v" .Values.global.observability.tracing.enabled) -}}
+  {{- with $nvcfWorkerServiceURL }}
+  {{- $_ := set $apiEnv "NVCF_FQDN" . }}
+  {{- end }}
+  {{- with $nvcfWorkerGrpcServiceURL }}
+  {{- $_ := set $apiEnv "NVCF_GLOBAL_FQDN_GRPC" . }}
+  {{- end }}
+  {{- with $nvcfNatsWorkerServiceURL }}
+  {{- $_ := set $apiEnv "NVCF_NATS_WORKER_URL" . }}
+  {{- end }}
+  {{- with $essWorkerBaseURL }}
+  {{- /* Keep the legacy worker alias until all API builds consume nvcf.ess.base-url directly. */}}
+  {{- $_ := set $apiEnv "NVCF_ESS_WORKER_BASE_URL" . }}
+  {{- $_ := set $apiEnv "NVCF_ESS_BASE_URL" . }}
+  {{- end }}
+  {{- with dig "api" "icmsBaseUrl" nil .Values }}
+  {{- $_ := set $apiEnv "NVCF_ICMS_BASE_URL" . }}
+  {{- end }}
+  {{- with dig "api" "jwt" "issuerUri" nil .Values }}
+  {{- $_ := set $apiEnv "SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI" . }}
+  {{- end }}
+  {{- with dig "api" "jwt" "jwkSetUri" nil .Values }}
+  {{- $_ := set $apiEnv "SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI" . }}
+  {{- end }}
+  {{- if .Values.global.observability.tracing.enabled }}
+  {{- $_ := set $apiEnv "MANAGEMENT_OTLP_TRACING_ENDPOINT" (printf "%s://%s:%v/v1/traces" .Values.global.observability.tracing.collectorProtocol .Values.global.observability.tracing.collectorEndpoint .Values.global.observability.tracing.collectorPort) }}
+  {{- end }}
+  {{- $apiEnv = mergeOverwrite $apiEnv $renderedApiEnv }}
   env:
-    NVCF_NATS_REGION_PLACEMENT_TAG: "dc"
-    NVCF_SIDECARS_HOSTNAME: {{ $sidecarsHost }}
-    NVCF_SIDECARS_REPOSITORY: {{ $sidecarsRepo }}
-    {{- with $nvcfWorkerServiceURL }}
-    NVCF_FQDN: {{ . | quote }}
-    {{- end }}
-    {{- with $nvcfWorkerGrpcServiceURL }}
-    NVCF_GLOBAL_FQDN_GRPC: {{ . | quote }}
-    {{- end }}
-    {{- with $nvcfNatsWorkerServiceURL }}
-    NVCF_NATS_WORKER_URL: {{ . | quote }}
-    {{- end }}
-    {{- with $essWorkerBaseURL }}
-    # Keep the legacy worker alias until all API builds consume nvcf.ess.base-url directly.
-    NVCF_ESS_WORKER_BASE_URL: {{ . | quote }}
-    NVCF_ESS_BASE_URL: {{ . | quote }}
-    {{- end }}
-    {{- with dig "api" "icmsBaseUrl" nil .Values }}
-    NVCF_ICMS_BASE_URL: {{ . | quote }}
-    {{- end }}
-    {{- with dig "api" "jwt" "issuerUri" nil .Values }}
-    SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI: {{ . | quote }}
-    {{- end }}
-    {{- with dig "api" "jwt" "jwkSetUri" nil .Values }}
-    SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI: {{ . | quote }}
-    {{- end }}
-    # Observability
-    MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }}
-    {{- if .Values.global.observability.tracing.enabled }}
-    MANAGEMENT_OTLP_TRACING_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}/v1/traces"
-    {{- end }}
+    {{- toYaml $apiEnv | nindent 4 }}
 
 invocation:
   fullnameOverride: invocation-service
@@ -874,6 +953,8 @@ llmApiGateway:
 {{- $pylonGrpcDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonGrpcDialAddress" "" .Values | default "" | toString | trim }}
 {{- $pylonReverseTunnelDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonReverseTunnelDialAddress" "" .Values | default "" | toString | trim }}
 {{- $backendRouterEnabled := dig "addons" "llm" "requestRouter" "backendRouter" "enabled" $llmEnabled .Values }}
+{{- $llmRequestRouterImageTag := dig "addons" "llm" "requestRouter" "image" "tag" "0.14.1" .Values | default "0.14.1" | toString }}
+{{- $backendRouterImageTag := dig "addons" "llm" "requestRouter" "backendRouter" "image" "tag" "" .Values | default $llmRequestRouterImageTag | toString }}
 {{- $llmWorkerRouteEnabled := dig "ingress" "gatewayApi" "routes" "llmWorker" "enabled" false .Values }}
 {{- $grpcTlsEnabled := dig "addons" "llm" "requestRouter" "grpcTls" "enabled" false .Values }}
 {{- $grpcTlsAllowInsecureHttp := dig "addons" "llm" "requestRouter" "grpcTls" "allowInsecureHttp" false .Values }}
@@ -918,6 +999,7 @@ llmRequestRouter:
   image:
     registry: {{ .Values.global.image.registry }}
     repository: {{ .Values.global.image.repository }}/stargate
+    tag: {{ $llmRequestRouterImageTag | quote }}
   {{- /*
   Backend routing follows the LLM addon rather than being separately opted into.
   A worker holds one registration stream and one reverse tunnel per replica, and
@@ -937,6 +1019,7 @@ llmRequestRouter:
     image:
       registry: {{ .Values.global.image.registry }}
       repository: {{ .Values.global.image.repository }}/stargate
+      tag: {{ $backendRouterImageTag | quote }}
     {{- with $pylonGrpcDialAddress }}
     pylonGrpcDialAddress: {{ . | quote }}
     {{- end }}

deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl

@@ -120,7 +120,7 @@ releases:
     <<: *dependency
 
   - name: openbao-server # this name MUST not change
-    version: 0.30.25
+    version: 0.32.1
     condition: openbao.enabled # From defaults.yaml or env overrides
     namespace: vault-system
     <<: *dependency # Inherits base values from the dependency template

deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl

@@ -168,7 +168,7 @@ releases:
     {{- $llmRequestRouterChartPath := dig "addons" "llm" "requestRouter" "chartPath" "" .Values }}
     chart: {{ $llmRequestRouterChartPath | default "nvcf/helm-nvcf-llm-request-router" | quote }}
     {{- if not $llmRequestRouterChartPath }}
-    version: 1.10.0
+    version: 1.12.0
     {{- end }}
     namespace: nvcf
     condition: addons.llm.enabled
@@ -217,7 +217,7 @@ releases:
     {{- $gatewayRoutesChartPath := dig "ingress" "gatewayApi" "chartPath" "" .Values }}
     chart: {{ $gatewayRoutesChartPath | default "nvcf/nvcf-gateway-routes" | quote }}
     {{- if not $gatewayRoutesChartPath }}
-    version: 1.16.0
+    version: 1.17.0
     {{- end }}
     needs:
       - nvcf/notary-service