From db7734d7d33a4c168c3f01570935bfb1a954639e Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 15:15:13 +0100 Subject: [PATCH 01/29] feat(helm): add generic gateway TOML serializer Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/_toml.tpl | 90 +++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 deploy/helm/openshell/templates/_toml.tpl diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl new file mode 100644 index 0000000000..d81ef65ace --- /dev/null +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -0,0 +1,90 @@ +{{/* +Render gatewayConfig as TOML. + +The chart deliberately treats the top-level keys as TOML table names. Nested +maps are TOML inline tables, and maps in arrays are inline-table array items. +This keeps the YAML-to-TOML boundary generic: adding a non-secret gateway +field must not require a Helm template change. +*/}} + +{{/* Quote a TOML key. Quoted keys safely support every YAML map key. */}} +{{- define "openshell.toml.key" -}} +{{- . | toString | quote -}} +{{- end -}} + +{{/* Render a scalar. Strings alone are Helm-templated. */}} +{{- define "openshell.toml.scalar" -}} +{{- $root := index . 0 -}} +{{- $value := index . 1 -}} +{{- if kindIs "string" $value -}} +{{- tpl $value $root | quote -}} +{{- else if or (kindIs "bool" $value) (kindIs "int" $value) (kindIs "int64" $value) (kindIs "float64" $value) -}} +{{- $value | toJson -}} +{{- else -}} +{{- fail (printf "gatewayConfig values must be strings, booleans, numbers, maps, or arrays; got %s" (kindOf $value)) -}} +{{- end -}} +{{- end -}} + +{{/* Render a TOML inline table, omitting YAML null values. */}} +{{- define "openshell.toml.inlineTable" -}} +{{- $root := index . 0 -}} +{{- $table := index . 1 -}} +{{- $entries := list -}} +{{- range $key := keys $table | sortAlpha -}} +{{- $value := get $table $key -}} +{{- if ne $value nil -}} +{{- $entry := printf "%s = %s" (include "openshell.toml.key" $key) (include "openshell.toml.value" (list $root $value)) -}} +{{- $entries = append $entries $entry -}} +{{- end -}} +{{- end -}} +{{- printf "{ %s }" (join ", " $entries) -}} +{{- end -}} + +{{/* Render an array. Maps become TOML inline-table entries. */}} +{{- define "openshell.toml.array" -}} +{{- $root := index . 0 -}} +{{- $array := index . 1 -}} +{{- $entries := list -}} +{{- range $value := $array -}} +{{- if eq $value nil -}} +{{- fail "gatewayConfig arrays cannot contain null values" -}} +{{- end -}} +{{- $entries = append $entries (include "openshell.toml.value" (list $root $value)) -}} +{{- end -}} +{{- printf "[%s]" (join ", " $entries) -}} +{{- end -}} + +{{/* Render any supported YAML value as TOML. */}} +{{- define "openshell.toml.value" -}} +{{- $root := index . 0 -}} +{{- $value := index . 1 -}} +{{- if kindIs "map" $value -}} +{{- include "openshell.toml.inlineTable" (list $root $value) -}} +{{- else if kindIs "slice" $value -}} +{{- include "openshell.toml.array" (list $root $value) -}} +{{- else -}} +{{- include "openshell.toml.scalar" (list $root $value) -}} +{{- end -}} +{{- end -}} + +{{/* Render the top-level gatewayConfig map as deterministic TOML tables. */}} +{{- define "openshell.gatewayConfigToml" -}} +{{- $root := . -}} +{{- $config := .Values.gatewayConfig | default dict -}} +{{- range $tableName := keys $config | sortAlpha -}} +{{- $fields := get $config $tableName -}} +{{- if ne $fields nil -}} +{{- if not (kindIs "map" $fields) -}} +{{- fail (printf "gatewayConfig table %q must be a map, got %s" $tableName (kindOf $fields)) -}} +{{- end -}} +[{{ include "openshell.toml.key" $tableName }}] +{{- range $fieldName := keys $fields | sortAlpha }} +{{- $value := get $fields $fieldName -}} +{{- if ne $value nil }} +{{ include "openshell.toml.key" $fieldName }} = {{ include "openshell.toml.value" (list $root $value) }} +{{- end }} +{{- end }} + +{{- end -}} +{{- end -}} +{{- end -}} From f7d8a09e9f5e9297de3eabb7863aa0a9acad2d78 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 15:44:42 +0100 Subject: [PATCH 02/29] docs(architecture): define Helm gateway configuration boundary Signed-off-by: Gaizka Menendez Hernandez --- docs/how-it-works/gateways/configuration.mdx | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 1308871cea..81d5164d4d 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -16,6 +16,27 @@ The OpenShell gateway reads its configuration from a TOML file when `--config` o Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in default ``` +## Helm Configuration Boundary + +The Kubernetes Helm chart exposes `gatewayConfig` as the non-secret gateway +application-configuration boundary. Each top-level map key names a TOML table, +and the chart serializes its fields into the mounted `gateway.toml` ConfigMap. +New non-secret gateway options therefore do not require a chart-template +change. + +Secret material never belongs in `gatewayConfig` or the ConfigMap. Database +URLs, credentials, private keys, and equivalent values use Kubernetes Secrets +through the chart's supported environment-variable, file, or volume wiring. +The normal precedence above still applies to the resulting mounted TOML file. + +The serializer has a deterministic YAML-to-TOML contract: YAML `null` fields +are omitted, `null` array members are rejected, strings are the only values +evaluated as Helm templates, and equivalent input produces the same ConfigMap +checksum. Helm retains ownership of Services, workloads, probes, Secrets, +certificate resources, Routes, RBAC, NetworkPolicies, and mounts. When one of +those inputs also determines a gateway runtime value, the chart derives it +from its resource owner instead of exposing two independent settings. + `database_url` is env-only. The loader rejects it when it appears in the file. When `OPENSHELL_DB_URL` is unset, the gateway stores its SQLite database under `$XDG_STATE_HOME/openshell/gateway/openshell.db`. On-disk SQLite databases run in WAL mode with `synchronous=FULL`: the gateway keeps `openshell.db-wal` and `openshell.db-shm` next to the database file, the file must live on a local filesystem, and a backup must use `sqlite3 openshell.db ".backup "` rather than copying `openshell.db` on its own. Every acknowledged write, including SSH session revocations, survives a power loss. The one exception is SSH session issuance, which skips the per-commit sync; a power loss can drop a just-issued session token, and the client must request a new one. From cd4f48024af309df1d5509229ab18591dbfb3d3a Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 15:45:26 +0100 Subject: [PATCH 03/29] feat(helm): define default gateway configuration map Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/values.yaml | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 4bf21b88a8..b79e322f27 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -256,6 +256,51 @@ tolerations: [] # -- Affinity rules for the gateway pod. affinity: {} +# -- Non-secret gateway application configuration. Top-level keys name TOML +# tables and are rendered into the mounted gateway.toml file. Kubernetes +# resource inputs remain outside this map; template expressions derive the +# corresponding runtime values from their resource owner. +gatewayConfig: + openshell: + version: 2 + openshell.gateway: + name: '{{ include "openshell.fullname" . }}' + bind_address: '0.0.0.0:{{ .Values.service.port }}' + health_bind_address: '0.0.0.0:{{ .Values.service.healthPort }}' + metrics_bind_address: '0.0.0.0:{{ .Values.service.metricsPort }}' + compute_driver: kubernetes + enable_loopback_service_http: true + openshell.gateway.gateway_jwt: + signing_key_path: /etc/openshell-jwt/signing.pem + public_key_path: /etc/openshell-jwt/public.pem + kid_path: /etc/openshell-jwt/kid + gateway_id: '{{ include "openshell.fullname" . }}' + openshell.gateway.tls: + cert_path: /etc/openshell-tls/server/tls.crt + key_path: /etc/openshell-tls/server/tls.key + client_ca_path: /etc/openshell-tls/client-ca/ca.crt + openshell.drivers.kubernetes: + namespace: '{{ include "openshell.sandboxNamespace" . }}' + default_image: ghcr.io/nvidia/openshell-community/sandboxes/base:latest + client_tls_secret_name: '{{ .Values.server.tls.clientTlsSecretName }}' + workspace_mode: shared + gateway_id: '{{ include "openshell.fullname" . }}' + grpc_endpoint: '{{ include "openshell.grpcEndpoint" . }}' + service_account_name: '{{ include "openshell.sandboxServiceAccountName" . }}' + supervisor_sideload_method: '{{ include "openshell.supervisorSideloadMethod" . }}' + topology: '{{ .Values.supervisor.topology }}' + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: true + gateway_namespace: '{{ .Release.Namespace }}' + gateway_pod_selector: + app.kubernetes.io/name: '{{ include "openshell.name" . }}' + app.kubernetes.io/instance: '{{ .Release.Name }}' + openshell.drivers.kubernetes.sidecar: + proxy_uid: 1337 + process_binary_aware_network_policy: true + openshell.gateway.credential_storage: + key_encryption_key_env: '{{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }}' + # Server configuration server: # -- Operator-facing gateway name. Defaults to the chart fullname so all From 7f308463b7ac7593d1401f5cc8973b202e8593f2 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 15:59:18 +0100 Subject: [PATCH 04/29] refactor(helm): render gateway config from values map Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/_toml.tpl | 22 +- .../openshell/templates/gateway-config.yaml | 297 +----------------- 2 files changed, 21 insertions(+), 298 deletions(-) diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index d81ef65ace..b9a3b93ce2 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -7,9 +7,14 @@ This keeps the YAML-to-TOML boundary generic: adding a non-secret gateway field must not require a Helm template change. */}} -{{/* Quote a TOML key. Quoted keys safely support every YAML map key. */}} +{{/* Render a TOML key. Bare keys keep ordinary output readable. */}} {{- define "openshell.toml.key" -}} -{{- . | toString | quote -}} +{{- $key := . | toString -}} +{{- if regexMatch "^[A-Za-z0-9_-]+$" $key -}} +{{- $key -}} +{{- else -}} +{{- $key | quote -}} +{{- end -}} {{- end -}} {{/* Render a scalar. Strings alone are Helm-templated. */}} @@ -77,14 +82,21 @@ field must not require a Helm template change. {{- if not (kindIs "map" $fields) -}} {{- fail (printf "gatewayConfig table %q must be a map, got %s" $tableName (kindOf $fields)) -}} {{- end -}} -[{{ include "openshell.toml.key" $tableName }}] +{{- $header := list -}} +{{- $segments := splitList "." $tableName -}} +{{- range $index, $segment := $segments -}} +{{- if eq $segment "" -}} +{{- fail (printf "gatewayConfig table %q contains an empty TOML key segment" $tableName) -}} +{{- end -}} +{{- $header = append $header (include "openshell.toml.key" $segment) -}} +{{- end -}} +{{ printf "[%s]\n" (join "." $header) }} {{- range $fieldName := keys $fields | sortAlpha }} {{- $value := get $fields $fieldName -}} {{- if ne $value nil }} -{{ include "openshell.toml.key" $fieldName }} = {{ include "openshell.toml.value" (list $root $value) }} +{{ printf "%s = %s\n" (include "openshell.toml.key" $fieldName) (include "openshell.toml.value" (list $root $value)) }} {{- end }} {{- end }} - {{- end -}} {{- end -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index a3e0210a35..bd7fefdc07 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -7,23 +7,10 @@ The gateway reads `/etc/openshell/gateway.toml` (mounted from this ConfigMap) at startup. CLI flags and OPENSHELL_* env vars on the gateway workload container still override anything in this file. -One value is intentionally NOT rendered here: - - server.dbUrl → passed via OPENSHELL_DB_URL env var (from Secret) - when server.externalDbSecret is set, otherwise - --db-url arg for SQLite +gatewayConfig is the complete non-secret application configuration boundary. +Database URLs and other credentials remain injected through Secret-backed +environment variables, files, or volumes outside this ConfigMap. */}} -{{- $credentialDrivers := list -}} -{{- $otlp := .Values.server.otlp | default dict -}} -{{- $ocsfLog := .Values.server.ocsfLog | default dict -}} -{{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} -{{- end -}} -{{- if .Values.server.credentialDrivers.vault.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "vault" -}} -{{- end -}} -{{- if and .Values.certManager.serverIssuerRef.name (not .Values.certManager.enabled) }} -{{- fail "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." }} -{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -32,280 +19,4 @@ metadata: {{- include "openshell.labels" . | nindent 4 }} data: gateway.toml: | - [openshell] - version = 2 - - [openshell.gateway] - name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} - bind_address = "0.0.0.0:{{ .Values.service.port }}" - {{- if .Values.service.healthPort }} - health_bind_address = "0.0.0.0:{{ .Values.service.healthPort }}" - {{- end }} - {{- if .Values.service.metricsPort }} - metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" - {{- end }} - log_level = {{ .Values.server.logLevel | quote }} - compute_driver = "kubernetes" - {{- if $credentialDrivers }} - credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] - {{- end }} - {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} - {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} - {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} - {{- end }} - policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - {{- if .Values.server.disableTls }} - disable_tls = true - {{- end }} - enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} - enable_websocket_tunnel = {{ .Values.server.enableWebsocketTunnel }} - {{- $sans := list -}} - {{- if and .Values.certManager.enabled .Values.certManager.serverDnsNames }} - {{- $sans = .Values.certManager.serverDnsNames }} - {{- else if and .Values.pkiInitJob.enabled .Values.pkiInitJob.serverDnsNames }} - {{- $sans = .Values.pkiInitJob.serverDnsNames }} - {{- end }} - {{- if $sans }} - server_sans = [{{- range $i, $san := $sans }}{{ if $i }}, {{ end }}{{ $san | quote }}{{- end }}] - {{- end }} - {{- $rlRequests := int .Values.server.grpcRateLimit.requests }} - {{- $rlWindowSeconds := int .Values.server.grpcRateLimit.windowSeconds }} - {{- if or (lt $rlRequests 0) (lt $rlWindowSeconds 0) }} - {{- fail "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" }} - {{- end }} - {{- if and (gt $rlRequests 0) (gt $rlWindowSeconds 0) }} - grpc_rate_limit_requests = {{ $rlRequests }} - grpc_rate_limit_window_seconds = {{ $rlWindowSeconds }} - {{- else if or (gt $rlRequests 0) (gt $rlWindowSeconds 0) }} - {{- fail "server.grpcRateLimit requires both requests and windowSeconds to be positive to enable rate limiting, or both 0/unset to disable it" }} - {{- end }} - - {{- if $otlp.endpoint }} - - [openshell.gateway.otlp] - endpoint = {{ $otlp.endpoint | quote }} - {{- if $otlp.serviceName }} - service_name = {{ $otlp.serviceName | quote }} - {{- end }} - {{- end }} - - {{- if $ocsfLog.enabled }} - {{- if not $ocsfLog.path -}} - {{- fail "server.ocsfLog.path must be set when server.ocsfLog.enabled is true" -}} - {{- end }} - - {{- $ocsfRotation := $ocsfLog.rotation | default "daily" -}} - {{- if not (has $ocsfRotation (list "daily" "never")) -}} - {{- fail "server.ocsfLog.rotation must be daily or never" -}} - {{- end }} - {{- $ocsfQueueCapacity := int (ternary 10000 $ocsfLog.queueCapacity (kindIs "invalid" $ocsfLog.queueCapacity)) -}} - {{- $ocsfQueueMaxBytes := int (ternary 16777216 $ocsfLog.queueMaxBytes (kindIs "invalid" $ocsfLog.queueMaxBytes)) -}} - {{- if or (lt $ocsfQueueCapacity 1) (lt $ocsfQueueMaxBytes 1) -}} - {{- fail "server.ocsfLog.queueCapacity and queueMaxBytes must be positive" -}} - {{- end }} - [openshell.gateway.ocsf_log] - path = {{ $ocsfLog.path | quote }} - {{- $ocsfSchemaVersion := $ocsfLog.schemaVersion | default "" -}} - {{- if not (has $ocsfSchemaVersion (list "" "1.1" "1.3")) -}} - {{- fail "server.ocsfLog.schemaVersion must be empty, 1.1, or 1.3" -}} - {{- end }} - {{- if $ocsfSchemaVersion }} - schema_version = {{ $ocsfSchemaVersion | quote }} - {{- end }} - rotation = {{ $ocsfRotation | quote }} - {{- if eq $ocsfRotation "daily" }} - {{- $ocsfMaxFiles := int (ternary 7 $ocsfLog.maxFiles (kindIs "invalid" $ocsfLog.maxFiles)) -}} - {{- if lt $ocsfMaxFiles 1 -}} - {{- fail "server.ocsfLog.maxFiles must be positive when rotation is daily" -}} - {{- end }} - max_files = {{ $ocsfMaxFiles }} - {{- end }} - queue_capacity = {{ $ocsfQueueCapacity }} - queue_max_bytes = {{ $ocsfQueueMaxBytes }} - {{- end }} - - {{- if not .Values.server.disableTls }} - - [openshell.gateway.tls] - cert_path = "/etc/openshell-tls/server/tls.crt" - key_path = "/etc/openshell-tls/server/tls.key" - {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} - client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" - {{- end }} - {{- if .Values.certManager.serverIssuerRef.name }} - external_cert_path = "/etc/openshell-tls/server-external/tls.crt" - external_key_path = "/etc/openshell-tls/server-external/tls.key" - external_server_names = [{{- range $i, $name := .Values.certManager.serverDnsNames }}{{ if $i }}, {{ end }}{{ $name | quote }}{{- end }}] - {{- end }} - {{- end }} - - {{- if .Values.server.auth.allowUnauthenticatedUsers }} - - [openshell.gateway.auth] - allow_unauthenticated_users = true - {{- end }} - - [openshell.gateway.gateway_jwt] - signing_key_path = "/etc/openshell-jwt/signing.pem" - public_key_path = "/etc/openshell-jwt/public.pem" - kid_path = "/etc/openshell-jwt/kid" - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - ttl_secs = {{ .Values.server.sandboxJwt.ttlSecs | default 3600 }} - - {{- if .Values.server.oidc.issuer }} - - [openshell.gateway.oidc] - issuer = {{ .Values.server.oidc.issuer | quote }} - dangerously_allow_insecure_http = {{ .Values.server.oidc.dangerouslyAllowInsecureHttp }} - jwks_allowed_origins = {{ .Values.server.oidc.jwksAllowedOrigins | toJson }} - audience = {{ .Values.server.oidc.audience | quote }} - jwks_ttl_secs = {{ .Values.server.oidc.jwksTtl }} - {{- if .Values.server.oidc.rolesClaim }} - roles_claim = {{ .Values.server.oidc.rolesClaim | quote }} - {{- end }} - {{- if .Values.server.oidc.adminRole }} - admin_role = {{ .Values.server.oidc.adminRole | quote }} - {{- end }} - {{- if .Values.server.oidc.userRole }} - user_role = {{ .Values.server.oidc.userRole | quote }} - {{- end }} - {{- if .Values.server.oidc.scopesClaim }} - scopes_claim = {{ .Values.server.oidc.scopesClaim | quote }} - {{- end }} - {{- end }} - - [openshell.drivers.kubernetes] - allow_driver_config = {{ .Values.server.drivers.kubernetes.allowDriverConfig }} - namespace = {{ include "openshell.sandboxNamespace" . | quote }} - default_image = {{ include "openshell.sandboxImage" . | quote }} - {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} - sandbox_runtime_image = {{ include "openshell.sandboxRuntimeImage" . | quote }} - {{- end }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} - {{- if not .Values.server.disableTls }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} - {{- end }} - workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} - service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} - {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} - operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} - {{- end }} - {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} - operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} - {{- end }} - sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} - {{- if .Values.upstreamProxy.url }} - https_proxy = {{ .Values.upstreamProxy.url | quote }} - {{- end }} - {{- if .Values.upstreamProxy.noProxy }} - no_proxy = {{ .Values.upstreamProxy.noProxy | quote }} - {{- end }} - {{- if .Values.upstreamProxy.authSecret.name }} - proxy_auth_secret_name = {{ .Values.upstreamProxy.authSecret.name | quote }} - {{- end }} - {{- if .Values.upstreamProxy.authSecret.key }} - proxy_auth_secret_key = {{ .Values.upstreamProxy.authSecret.key | quote }} - {{- end }} - {{- if and .Values.upstreamProxy.authSecret.name .Values.upstreamProxy.authSecret.key }} - proxy_auth_allow_insecure = {{ .Values.upstreamProxy.authAllowInsecure }} - {{- end }} - {{- if .Values.upstreamProxy.connectByHostname }} - proxy_connect_by_hostname = true - {{- end }} - {{- if .Values.upstreamProxy.caBundle.configMapName }} - proxy_ca_bundle = "/etc/openshell-tls/proxy-ca/ca.crt" - {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} - provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} - {{- end }} - {{- if .Values.sandbox.image.pullPolicy }} - image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandbox.image.pullPolicy | quote }} - {{- end }} - {{- $sandboxImagePullSecretNames := list -}} - {{- range .Values.server.sandboxImagePullSecrets }} - {{- if .name }} - {{- $sandboxImagePullSecretNames = append $sandboxImagePullSecretNames .name }} - {{- end }} - {{- end }} - {{- if $sandboxImagePullSecretNames }} - image_pull_secrets = [{{- range $i, $name := $sandboxImagePullSecretNames }}{{ if $i }}, {{ end }}{{ $name | quote }}{{- end }}] - {{- end }} - {{- if .Values.server.workspaceDefaultStorageSize }} - workspace_default_storage_size = {{ .Values.server.workspaceDefaultStorageSize | quote }} - {{- end }} - {{- if .Values.server.workspaceStorageClass }} - workspace_storage_class = {{ .Values.server.workspaceStorageClass | quote }} - {{- end }} - {{- if .Values.server.defaultRuntimeClassName }} - default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} - {{- end }} - {{- if (.Values.supervisor.image.pullPolicy | default .Values.global.image.pullPolicy) }} - supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" (.Values.supervisor.image.pullPolicy | default .Values.global.image.pullPolicy) | quote }} - {{- end }} - {{- if (.Values.sandboxRuntime.image.pullPolicy | default .Values.global.image.pullPolicy) }} - sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" (.Values.sandboxRuntime.image.pullPolicy | default .Values.global.image.pullPolicy) | quote }} - {{- end }} - - [openshell.drivers.kubernetes.resource_admission] - enabled = {{ .Values.server.drivers.kubernetes.resourceAdmission.enabled }} - {{- if ne .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels nil }} - [openshell.drivers.kubernetes.resource_admission.required_labels] - {{- range $key, $value := .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels }} - {{ $key | quote }} = {{ $value | quote }} - {{- end }} - {{- end }} - - [openshell.drivers.kubernetes.managed_ssh_ingress] - enabled = {{ .Values.networkPolicy.enabled }} - gateway_namespace = {{ .Release.Namespace | quote }} - gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } - - [openshell.drivers.kubernetes.sandbox_runtime] - boundary_port = {{ .Values.supervisor.sandboxRuntime.boundaryPort | default 5500 }} - - {{- if not $credentialDrivers }} - - [openshell.gateway.credential_storage] - key_encryption_key_env = {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . | quote }} - {{- end }} - - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - - [openshell.credential_drivers.kubernetes-secrets] - namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} - {{- end }} - - {{- if .Values.server.credentialDrivers.vault.enabled }} - - [openshell.credential_drivers.vault] - address = {{ .Values.server.credentialDrivers.vault.address | quote }} - {{- if .Values.server.credentialDrivers.vault.caConfigMapName }} - ca_bundle = "/etc/openshell-tls/vault-ca/ca.crt" - {{- end }} - mount = {{ .Values.server.credentialDrivers.vault.mount | quote }} - kv_version = {{ .Values.server.credentialDrivers.vault.kvVersion | quote }} - auth_method = {{ .Values.server.credentialDrivers.vault.authMethod | quote }} - {{- if .Values.server.credentialDrivers.vault.role }} - role = {{ .Values.server.credentialDrivers.vault.role | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.kubernetesAuthMount }} - kubernetes_auth_mount = {{ .Values.server.credentialDrivers.vault.kubernetesAuthMount | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.serviceAccountTokenPath }} - service_account_token_path = {{ .Values.server.credentialDrivers.vault.serviceAccountTokenPath | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.tokenPath }} - token_path = {{ .Values.server.credentialDrivers.vault.tokenPath | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.timeoutSecs }} - timeout_secs = {{ .Values.server.credentialDrivers.vault.timeoutSecs }} - {{- end }} - {{- end }} +{{ include "openshell.gatewayConfigToml" . | nindent 4 }} From 9986db64afb2401510c6f0e8e9db2461ade5acd6 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 16:06:19 +0100 Subject: [PATCH 05/29] test(helm): cover generic gateway TOML rendering Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/_toml.tpl | 15 ++- .../tests/gateway_config_serializer_test.yaml | 91 +++++++++++++++++++ docs/how-it-works/gateways/configuration.mdx | 25 +++-- 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 deploy/helm/openshell/tests/gateway_config_serializer_test.yaml diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index b9a3b93ce2..f83a30cb3f 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -23,7 +23,20 @@ field must not require a Helm template change. {{- $value := index . 1 -}} {{- if kindIs "string" $value -}} {{- tpl $value $root | quote -}} -{{- else if or (kindIs "bool" $value) (kindIs "int" $value) (kindIs "int64" $value) (kindIs "float64" $value) -}} +{{- else if or + (kindIs "bool" $value) + (kindIs "int" $value) + (kindIs "int8" $value) + (kindIs "int16" $value) + (kindIs "int32" $value) + (kindIs "int64" $value) + (kindIs "uint" $value) + (kindIs "uint8" $value) + (kindIs "uint16" $value) + (kindIs "uint32" $value) + (kindIs "uint64" $value) + (kindIs "float32" $value) + (kindIs "float64" $value) -}} {{- $value | toJson -}} {{- else -}} {{- fail (printf "gatewayConfig values must be strings, booleans, numbers, maps, or arrays; got %s" (kindOf $value)) -}} diff --git a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml new file mode 100644 index 0000000000..776b5f0075 --- /dev/null +++ b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: gatewayConfig TOML serializer +templates: + - templates/gateway-config.yaml +release: + name: serializer-test + namespace: serializer-namespace + +tests: + - it: renders all supported YAML shapes with deterministic TOML output + set: + gatewayConfig: + example.config: + string_value: plain text + boolean_value: true + integer_value: 42 + float_value: 1.5 + scalar_array: + - first + - second + nested_map: + zebra: 2 + alpha: false + map_array: + - name: first + enabled: true + - name: second + enabled: false + omitted_value: null + empty_value: "" + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)^\[example\.config\]\n.*?boolean_value = true\n.*?empty_value = ""\n.*?float_value = 1\.5\n.*?integer_value = 42\n.*?map_array = \[\{ enabled = true, name = "first" \}, \{ enabled = false, name = "second" \}\]\n.*?nested_map = \{ alpha = false, zebra = 2 \}\n.*?scalar_array = \["first", "second"\]\n.*?string_value = "plain text"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: omitted_value + + - it: evaluates templates only in string values + set: + gatewayConfig: + example: + rendered_string: '{{ .Release.Namespace }}/{{ .Release.Name }}' + boolean_value: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^rendered_string = "serializer-namespace/serializer-test"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^boolean_value = true$' + + - it: quotes non-bare TOML keys safely + set: + gatewayConfig: + example: + "field with spaces": value + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^"field with spaces" = "value"$' + + - it: rejects a top-level table that is not a map + set: + gatewayConfig: + example: invalid + asserts: + - failedTemplate: + errorMessage: 'gatewayConfig table "example" must be a map, got string' + + - it: rejects null members in arrays + set: + gatewayConfig: + example: + values: + - valid + - null + asserts: + - failedTemplate: + errorMessage: gatewayConfig arrays cannot contain null values + + - it: rejects table names with empty TOML key segments + set: + gatewayConfig: + "example..config": + value: valid + asserts: + - failedTemplate: + errorMessage: 'gatewayConfig table "example..config" contains an empty TOML key segment' diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 81d5164d4d..0fc4a27517 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -29,13 +29,24 @@ URLs, credentials, private keys, and equivalent values use Kubernetes Secrets through the chart's supported environment-variable, file, or volume wiring. The normal precedence above still applies to the resulting mounted TOML file. -The serializer has a deterministic YAML-to-TOML contract: YAML `null` fields -are omitted, `null` array members are rejected, strings are the only values -evaluated as Helm templates, and equivalent input produces the same ConfigMap -checksum. Helm retains ownership of Services, workloads, probes, Secrets, -certificate resources, Routes, RBAC, NetworkPolicies, and mounts. When one of -those inputs also determines a gateway runtime value, the chart derives it -from its resource owner instead of exposing two independent settings. +The serializer has a deterministic YAML-to-TOML contract. YAML `null` fields +are omitted; `null` array members are rejected because TOML has no equivalent. +Strings, booleans, integers, and floats preserve their types. Scalar arrays +become TOML arrays, maps become inline tables, and arrays of maps become arrays +of inline tables. Keys are ordered alphabetically, so equivalent input produces +the same ConfigMap checksum. Helm `tpl` expressions are evaluated only in +string values, never in keys or YAML structure. + +The implementation intentionally uses only long-standing Helm 3 template and +Sprig functions (`tpl`, `kindIs`, `keys`, `sortAlpha`, `splitList`, `quote`, and +`toJson`); it does not rely on a Helm-specific TOML encoder. This preserves the +chart's documented Helm 3 compatibility while making the serialization rules +explicit in the chart itself. + +Helm retains ownership of Services, workloads, probes, Secrets, certificate +resources, Routes, RBAC, NetworkPolicies, and mounts. When one of those inputs +also determines a gateway runtime value, the chart derives it from its resource +owner instead of exposing two independent settings. `database_url` is env-only. The loader rejects it when it appears in the file. When `OPENSHELL_DB_URL` is unset, the gateway stores its SQLite database under `$XDG_STATE_HOME/openshell/gateway/openshell.db`. From 31a9b9f7dffdaaa3147212bd83a662e9f0a25784 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 16:12:05 +0100 Subject: [PATCH 06/29] feat(helm): protect gateway config secret boundary Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/_helpers.tpl | 22 +++++++ deploy/helm/openshell/templates/_toml.tpl | 15 ++++- .../tests/gateway_secret_boundary_test.yaml | 63 +++++++++++++++++++ docs/how-it-works/gateways/configuration.mdx | 5 ++ 4 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index ab42458759..d736493bc1 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -379,6 +379,23 @@ never {{- end -}} {{- end }} +{{/* +Validate a non-empty, user-provided Kubernetes Secret name. Secret data never +passes through Helm values into gateway.toml; only this reference is rendered. +*/}} +{{- define "openshell.validateSecretReference" -}} +{{- $path := index . 0 -}} +{{- $name := index . 1 -}} +{{- if and (ne $name nil) (ne $name "") -}} +{{- if not (kindIs "string" $name) -}} +{{- fail (printf "%s must be a Kubernetes Secret name, got %s" $path (kindOf $name)) -}} +{{- end -}} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" $name) -}} +{{- fail (printf "%s must be a valid Kubernetes Secret name" $path) -}} +{{- end -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} @@ -401,6 +418,11 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}} {{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}} {{- end -}} +{{- include "openshell.validateSecretReference" (list "server.externalDbSecret" .Values.server.externalDbSecret) -}} +{{- include "openshell.validateSecretReference" (list "server.credentialStorage.existingSecret" .Values.server.credentialStorage.existingSecret) -}} +{{- include "openshell.validateSecretReference" (list "server.sandboxJwt.signingSecretName" .Values.server.sandboxJwt.signingSecretName) -}} +{{- include "openshell.validateSecretReference" (list "server.tls.certSecretName" .Values.server.tls.certSecretName) -}} +{{- include "openshell.validateSecretReference" (list "upstreamProxy.authSecret.name" .Values.upstreamProxy.authSecret.name) -}} {{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} {{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index f83a30cb3f..7bb84fb5ea 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -22,7 +22,14 @@ field must not require a Helm template change. {{- $root := index . 0 -}} {{- $value := index . 1 -}} {{- if kindIs "string" $value -}} -{{- tpl $value $root | quote -}} +{{- $rendered := tpl $value $root -}} +{{- if regexMatch "-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----" $rendered -}} +{{- fail "gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount" -}} +{{- end -}} +{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]+:[^/@[:space:]]+@" $rendered -}} +{{- fail "gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume" -}} +{{- end -}} +{{- $rendered | quote -}} {{- else if or (kindIs "bool" $value) (kindIs "int" $value) @@ -51,6 +58,9 @@ field must not require a Helm template change. {{- range $key := keys $table | sortAlpha -}} {{- $value := get $table $key -}} {{- if ne $value nil -}} +{{- if eq $key "database_url" -}} +{{- fail "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" -}} +{{- end -}} {{- $entry := printf "%s = %s" (include "openshell.toml.key" $key) (include "openshell.toml.value" (list $root $value)) -}} {{- $entries = append $entries $entry -}} {{- end -}} @@ -107,6 +117,9 @@ field must not require a Helm template change. {{- range $fieldName := keys $fields | sortAlpha }} {{- $value := get $fields $fieldName -}} {{- if ne $value nil }} +{{- if eq $fieldName "database_url" -}} +{{- fail "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" -}} +{{- end -}} {{ printf "%s = %s\n" (include "openshell.toml.key" $fieldName) (include "openshell.toml.value" (list $root $value)) }} {{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml new file mode 100644 index 0000000000..0b7c97f30c --- /dev/null +++ b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: gateway ConfigMap secret boundary +templates: + - templates/gateway-config.yaml + - templates/statefulset.yaml + +tests: + - it: rejects a database URL in gatewayConfig before creating the ConfigMap + template: templates/statefulset.yaml + set: + gatewayConfig.example.database_url: postgresql://gateway:password@database:5432/openshell + asserts: + - failedTemplate: + errorMessage: "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" + + - it: rejects an inline PEM private key before creating the ConfigMap + template: templates/statefulset.yaml + set: + gatewayConfig.example.private_key: "-----BEGIN PRIVATE KEY-----" + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount + + - it: rejects inline credentials embedded in a URL + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://gateway:password@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + + - it: keeps external database credentials out of gateway.toml + template: templates/gateway-config.yaml + set: + server.externalDbSecret: gateway-database + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?i)postgresql://|gateway-database|password' + + - it: references external database credentials through an environment variable + template: templates/statefulset.yaml + set: + server.externalDbSecret: gateway-database + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: gateway-database + key: uri + + - it: rejects malformed Secret references + template: templates/statefulset.yaml + set: + server.credentialStorage.existingSecret: contains spaces + asserts: + - failedTemplate: + errorMessage: server.credentialStorage.existingSecret must be a valid Kubernetes Secret name diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 0fc4a27517..f81e3d34a8 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -29,6 +29,11 @@ URLs, credentials, private keys, and equivalent values use Kubernetes Secrets through the chart's supported environment-variable, file, or volume wiring. The normal precedence above still applies to the resulting mounted TOML file. +The chart rejects the gateway's `database_url` file field and unambiguous +inline credential or PEM private-key strings before rendering a ConfigMap. It +validates Secret references as Kubernetes Secret names, but never reads or +copies referenced Secret data into `gateway.toml`. + The serializer has a deterministic YAML-to-TOML contract. YAML `null` fields are omitted; `null` array members are rejected because TOML has no equivalent. Strings, booleans, integers, and floats preserve their types. Scalar arrays From 8fb226710d5ed6b94bbb1feb0e75914b65650c60 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 16:17:43 +0100 Subject: [PATCH 07/29] docs(helm): classify legacy gateway configuration values Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b79e322f27..cb45e12c0c 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -268,13 +268,16 @@ gatewayConfig: bind_address: '0.0.0.0:{{ .Values.service.port }}' health_bind_address: '0.0.0.0:{{ .Values.service.healthPort }}' metrics_bind_address: '0.0.0.0:{{ .Values.service.metricsPort }}' + log_level: info compute_driver: kubernetes enable_loopback_service_http: true + policy_validation_failure_mode: fail_closed openshell.gateway.gateway_jwt: signing_key_path: /etc/openshell-jwt/signing.pem public_key_path: /etc/openshell-jwt/public.pem kid_path: /etc/openshell-jwt/kid gateway_id: '{{ include "openshell.fullname" . }}' + ttl_secs: 3600 openshell.gateway.tls: cert_path: /etc/openshell-tls/server/tls.crt key_path: /etc/openshell-tls/server/tls.key @@ -289,6 +292,8 @@ gatewayConfig: service_account_name: '{{ include "openshell.sandboxServiceAccountName" . }}' supervisor_sideload_method: '{{ include "openshell.supervisorSideloadMethod" . }}' topology: '{{ .Values.supervisor.topology }}' + sa_token_ttl_secs: 3600 + app_armor_profile: Unconfined openshell.drivers.kubernetes.managed_ssh_ingress: enabled: true gateway_namespace: '{{ .Release.Namespace }}' From cb27b4b416e0480a06616534f08395f7fa3f29bd Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 16:22:57 +0100 Subject: [PATCH 08/29] refactor(helm): derive dual-use resources from gateway config Signed-off-by: Gaizka Menendez Hernandez --- .../openshell/templates/_gateway-workload.tpl | 14 ++-- deploy/helm/openshell/templates/_helpers.tpl | 4 +- .../helm/openshell/templates/clusterrole.yaml | 70 ++++++++++++++++--- deploy/helm/openshell/templates/role.yaml | 10 +-- .../helm/openshell/templates/rolebinding.yaml | 4 +- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 919d3a4313..f9dd55c68a 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -5,6 +5,10 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. */}} {{- define "openshell.gatewayPodTemplate" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $gatewayRuntimeConfig := get $gatewayConfig "openshell.gateway" | default dict -}} +{{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} +{{- $kubernetesRuntimeConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -27,9 +31,9 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} serviceAccountName: {{ include "openshell.serviceAccountName" . }} - {{- if .Values.server.hostGatewayIP }} + {{- if get $kubernetesRuntimeConfig "host_gateway_ip" }} hostAliases: - - ip: {{ .Values.server.hostGatewayIP | quote }} + - ip: {{ get $kubernetesRuntimeConfig "host_gateway_ip" | quote }} hostnames: - host.docker.internal - host.openshell.internal @@ -114,7 +118,7 @@ spec: # mounted at /etc/openshell/gateway.toml. Secret-bearing settings use # env vars that the TOML references by name. Some process-level # settings consumed by libraries outside gateway code also remain here. - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} # OIDC issuer custom-CA: rustls/reqwest read SSL_CERT_FILE for # outbound TLS verification. This is a process-level env var # consumed by the TLS stack itself, not by gateway code, so it @@ -164,7 +168,7 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} - name: oidc-ca mountPath: /etc/openshell-tls/oidc-ca readOnly: true @@ -265,7 +269,7 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} - name: oidc-ca configMap: name: {{ .Values.server.oidc.caConfigMapName }} diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index d736493bc1..135e0e9212 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -423,7 +423,9 @@ Validate chart values that Helm would otherwise accept silently. {{- include "openshell.validateSecretReference" (list "server.sandboxJwt.signingSecretName" .Values.server.sandboxJwt.signingSecretName) -}} {{- include "openshell.validateSecretReference" (list "server.tls.certSecretName" .Values.server.tls.certSecretName) -}} {{- include "openshell.validateSecretReference" (list "upstreamProxy.authSecret.name" .Values.upstreamProxy.authSecret.name) -}} -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} {{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 85d10c6b60..36491ceacb 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -90,22 +92,71 @@ rules: - patch - watch {{- end }} + {{- $copiedSecretNames := list }} + {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} + {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} + {{- end }} + {{- if eq $workspaceMode "managed" }} + {{- range (get $kubernetesConfig "image_pull_secrets" | default list) }} + {{- if . }} + {{- $copiedSecretNames = append $copiedSecretNames . }} + {{- end }} + {{- end }} +{{- end }} + {{- end }} {{- if ne $workspaceMode "shared" }} - apiGroups: [""] resources: ["services"] verbs: ["create", "get"] + - apiGroups: [""] + resources: ["secrets"] + # Bootstrap Secrets are generation-scoped. Recovery lists them by label and + # deletes stale generations with UID preconditions before retrying creation. + verbs: ["create", "list", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] {{- end }} - {{- if eq $workspaceMode "managed" }} - # Operator-mode namespaces receive Secret permissions from the openshell-workspace - # chart Role instead. - - apiGroups: [""] - resources: ["secrets"] - # Bootstrap and image-pull Secrets are generation-scoped. Recovery deletes - # them by exact name. - verbs: ["create", "delete"] + {{- $copiedSecretNames = uniq $copiedSecretNames }} + {{- if $copiedSecretNames }} + # Copy only explicitly configured TLS and image-pull Secrets into workspace + # namespaces. Server-side apply authorizes these requests as patch operations. + - apiGroups: + - "" + resources: + - secrets + resourceNames: + {{- range $copiedSecretNames }} + - {{ . | quote }} + {{- end }} + verbs: + - get + - patch + # Server-side apply uses PATCH for an existing Secret, but the API server + # additionally authorizes CREATE when the named Secret does not exist yet. + # Kubernetes RBAC cannot restrict CREATE by resourceNames because create + # authorization happens before the object name is available to RBAC. Keep + # reads and mutations name-restricted above; this broader grant is required + # only to create the initial copy in a gateway-owned managed namespace. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + {{- end }} + {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # The kubernetes-secrets credential driver uses dynamic hashed names in + # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - patch + - delete {{- end }} {{- if eq $workspaceMode "managed" }} # ServiceAccount creation in managed namespaces. @@ -129,4 +180,3 @@ rules: - update {{- end }} {{- end }} -{{- end }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index d9237e743d..1bb055d796 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,4 +1,6 @@ -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) (include "openshell.rbacCreate" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -59,9 +61,9 @@ rules: verbs: ["create", "get"] - apiGroups: [""] resources: ["secrets"] - # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact - # name. - verbs: ["create", "delete"] + # Bootstrap Secrets are generation-scoped. Recovery lists them by label and + # deletes stale generations with UID preconditions before retrying creation. + verbs: ["create", "list", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index 49cc6f7fba..886dafcf2f 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,4 +1,6 @@ -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) (include "openshell.rbacCreate" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 From 37c1e2bf138a112f48f83340f105666414fa19b0 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:01:27 +0100 Subject: [PATCH 09/29] test(helm): migrate gateway config scenarios Signed-off-by: Gaizka Menendez Hernandez --- crates/openshell-server/src/config_file.rs | 11 +++++++++++ deploy/helm/openshell/templates/_helpers.tpl | 3 +++ 2 files changed, 14 insertions(+) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index de3f3df6bd..e062739553 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -764,6 +764,17 @@ mod tests { } } + #[test] + fn gateway_rate_limits_reject_negative_values_during_toml_parsing() { + for field in ["grpc_rate_limit_requests", "grpc_rate_limit_window_seconds"] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field} = -1\n")); + assert!( + matches!(load(tmp.path()), Err(ConfigFileError::Parse { .. })), + "{field} must reject negative values because gateway rate limits are unsigned" + ); + } + } + #[test] fn canonical_compute_driver_is_singular() { let file: ConfigFile = toml::from_str( diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 135e0e9212..a09b77908c 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -406,6 +406,9 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (hasKey .Values "postgres") (kindIs "map" .Values.postgres) (hasKey .Values.postgres "enabled") -}} {{- fail "postgres.enabled was removed; the OpenShell chart no longer deploys PostgreSQL. Provision PostgreSQL separately and set server.externalDbSecret to a Secret containing a PostgreSQL URI." -}} {{- end -}} +{{- if and .Values.certManager.serverIssuerRef.name (not .Values.certManager.enabled) -}} +{{- fail "certManager.serverIssuerRef.name is set but certManager.enabled is false — the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." -}} +{{- end -}} {{- if not (or (eq $workloadKind "statefulset") (eq $workloadKind "deployment")) -}} {{- fail "workload.kind must be one of: statefulset, deployment." -}} {{- end -}} From c70660fd917c346b7054380b9cb5135bd00fc049 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:05:06 +0100 Subject: [PATCH 10/29] refactor(helm): derive credential resources from gateway config Signed-off-by: Gaizka Menendez Hernandez --- .../openshell/templates/_gateway-workload.tpl | 3 ++- deploy/helm/openshell/templates/_helpers.tpl | 23 ++++++++++--------- .../helm/openshell/templates/clusterrole.yaml | 2 +- .../templates/credential-secrets-role.yaml | 4 ++-- .../credential-secrets-rolebinding.yaml | 2 +- ...ial-storage-key-encryption-key-secret.yaml | 2 +- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index f9dd55c68a..cd645abb47 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -9,6 +9,7 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. {{- $gatewayRuntimeConfig := get $gatewayConfig "openshell.gateway" | default dict -}} {{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} {{- $kubernetesRuntimeConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $hasExternalCredentialDriver := or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -100,7 +101,7 @@ spec: value: /etc/openshell-tls/peer-client/tls.key {{- end }} {{- end }} - {{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }} + {{- if not $hasExternalCredentialDriver }} - name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }} valueFrom: secretKeyRef: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index a09b77908c..944488be8f 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -265,7 +265,18 @@ shared workspace mode. Namespace where Kubernetes Secret-backed provider credentials live. */}} {{- define "openshell.credentialKubernetesSecretsNamespace" -}} -{{- .Values.server.credentialDrivers.kubernetesSecrets.namespace | default .Release.Namespace -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $config := get $gatewayConfig "openshell.credential_drivers.kubernetes-secrets" | default dict -}} +{{- get $config "namespace" | default .Release.Namespace -}} +{{- end }} + +{{/* Whether a credential driver is enabled in the generic gateway config. */}} +{{- define "openshell.credentialDriverEnabled" -}} +{{- $root := index . 0 -}} +{{- $driver := index . 1 -}} +{{- $gatewayConfig := $root.Values.gatewayConfig | default dict -}} +{{- $gateway := get $gatewayConfig "openshell.gateway" | default dict -}} +{{- if has $driver (get $gateway "credential_drivers" | default list) -}}true{{- end -}} {{- end }} {{/* @@ -432,16 +443,6 @@ Validate chart values that Helm would otherwise accept silently. {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} {{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} {{- end -}} -{{- $credentialDrivers := list -}} -{{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} -{{- end -}} -{{- if .Values.server.credentialDrivers.vault.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "vault" -}} -{{- end -}} -{{- if gt (len $credentialDrivers) 1 -}} -{{- fail "only one external server.credentialDrivers backend can be enabled at a time." -}} -{{- end -}} {{- if kindIs "invalid" .Values.server.tls.clientCaSecretName -}} {{- fail "server.tls.clientCaSecretName cannot be null; omit the key to use the chart default (openshell-server-client-ca), or set to \"\" to disable client certificate verification for HTTPS-only mode" -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 36491ceacb..8e7bf23327 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -145,7 +145,7 @@ rules: verbs: - create {{- end }} - {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + {{- if and (ne $workspaceMode "shared") (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") }} # The kubernetes-secrets credential driver uses dynamic hashed names in # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. - apiGroups: diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index f6187c9acb..1fcb97fd90 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} +{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +13,7 @@ metadata: # runtime. Kubernetes RBAC does not support label-based or prefix-based # filtering for resourceNames. To limit blast radius, deploy the gateway # with a dedicated namespace for credential Secrets -# (server.credentialDrivers.kubernetesSecrets.namespace). +# (openshell.credential_drivers.kubernetes-secrets.namespace). rules: - apiGroups: - "" diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 3a9ee0bddc..0df76a9ddc 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} +{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml b/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml index 1e53d84bfb..f57ef56307 100644 --- a/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml +++ b/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }} +{{- if not (or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true")) }} {{- if not .Values.server.credentialStorage.existingSecret }} {{- $secretName := include "openshell.credentialStorageKeyEncryptionKeySecretName" . -}} {{- $secretKey := include "openshell.credentialStorageKeyEncryptionKeySecretKey" . -}} From 3a5db92c023b7baf9f19e2d292c7fefec06c3ad1 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:07:21 +0100 Subject: [PATCH 11/29] refactor(helm): migrate gateway config overlays Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/README.md.gotmpl | 42 +++++++++++++++---- .../ci/values-corporate-proxy-e2e.yaml | 11 ++--- ...-credential-driver-kubernetes-secrets.yaml | 11 ++--- .../ci/values-credential-driver-vault.yaml | 15 +++---- .../helm/openshell/ci/values-gateway-tls.yaml | 4 +- deploy/helm/openshell/ci/values-keycloak.yaml | 12 +++--- .../openshell/ci/values-openshift-e2e.yaml | 15 +++---- deploy/helm/openshell/ci/values-skaffold.yaml | 8 ++++ deploy/helm/openshell/ci/values-spire.yaml | 8 ++-- .../ci/values-workspace-managed.yaml | 11 +++-- .../ci/values-workspace-operator.yaml | 9 ++-- deploy/helm/openshell/skaffold.yaml | 1 + 12 files changed, 91 insertions(+), 56 deletions(-) diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 1e86e0cbcf..ce94283d41 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -240,9 +240,29 @@ database. The chart creates a retained Kubernetes Secret with the shared key-encryption key and injects that key into every gateway pod, so the same default works for single-replica and external database-backed HA deployments. -Use `kubernetes-secrets` or `vault` instead when credentials should live in a -cluster or external secret backend. Enabling one external credential driver -disables the default credential-storage key-encryption key Secret and env injection. +Use `gatewayConfig` to select `kubernetes-secrets` or `vault` when credentials +should live in a cluster or external secret backend. Selecting an external +credential driver disables the default credential-storage key-encryption-key +Secret and environment injection. The map is rendered directly as gateway TOML, +so it uses the gateway's snake_case field names: + +```yaml +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: http://vault.vault.svc.cluster.local:8200 + mount: secret + kv_version: "2" + auth_method: kubernetes + role: openshell-gateway +``` + +For the Kubernetes Secret driver, use +`openshell.credential_drivers.kubernetes-secrets.namespace` in the same map. +The chart derives any required RBAC from the selected driver; use a dedicated +namespace to limit access to OpenShell-managed Secrets. #### OpenShift @@ -292,12 +312,16 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants -Set `server.providerTokenGrants.spiffe.enabled=true` to let the gateway and -sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The -chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs, -mounts the SPIFFE CSI socket into the gateway pod, exports -`OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`, and passes the socket path to -the Kubernetes driver so sandbox pods can mount the same socket. +Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` +to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +grants. The chart keeps supervisor-to-gateway authentication on gateway-minted +sandbox JWTs and passes the configured socket path to the Kubernetes driver. + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock +``` For local development, uncomment the SPIRE Helm releases in `skaffold.yaml` and add `ci/values-spire.yaml` to the OpenShell release values files. diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index 70fdec8d7b..d9803da1ca 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,8 +3,9 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -upstreamProxy: - authSecret: - name: openshell-e2e-proxy-auth - key: proxy-auth - authAllowInsecure: true +gatewayConfig: + openshell.drivers.kubernetes: + topology: sidecar + proxy_auth_secret_name: openshell-e2e-proxy-auth + proxy_auth_secret_key: proxy-auth + proxy_auth_allow_insecure: true diff --git a/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml b/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml index 096ce46e29..440a7ff2ac 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml @@ -6,8 +6,9 @@ # Use with: # skaffold run -p credential-driver-kubernetes-secrets # -server: - credentialDrivers: - kubernetesSecrets: - enabled: true - namespace: openshell +gatewayConfig: + openshell.gateway: + credential_drivers: + - kubernetes-secrets + openshell.credential_drivers.kubernetes-secrets: + namespace: openshell diff --git a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml index 5158d48d78..9380dea613 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml @@ -13,10 +13,11 @@ # configures a Kubernetes auth role named `openshell-gateway` bound to the # OpenShell gateway ServiceAccount in the `openshell` namespace. -server: - credentialDrivers: - vault: - enabled: true - address: https://openbao-0:8200 - caConfigMapName: openbao-ca - role: openshell-gateway +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: https://openbao-0:8200 + ca_bundle: /etc/openshell-tls/vault-ca/ca.crt + role: openshell-gateway diff --git a/deploy/helm/openshell/ci/values-gateway-tls.yaml b/deploy/helm/openshell/ci/values-gateway-tls.yaml index a776760141..21bde40b85 100644 --- a/deploy/helm/openshell/ci/values-gateway-tls.yaml +++ b/deploy/helm/openshell/ci/values-gateway-tls.yaml @@ -28,6 +28,8 @@ grpcRoute: server: # Envoy terminates TLS at the edge; the gateway listens plaintext behind it. disableTls: true - oidc: + +gatewayConfig: + openshell.gateway.oidc: issuer: "https://keycloak.example.com/realms/openshell" audience: "openshell-cli" diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index 6df74e3258..1e4dc45b6b 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -21,8 +21,8 @@ # CLI token acquisition: keep a port-forward running while using openshell login: # kubectl -n keycloak port-forward svc/keycloak 9090:80 -server: - oidc: +gatewayConfig: + openshell.gateway.oidc: # Must match KC_HOSTNAME set by keycloak:k8s:setup (in-cluster service hostname). issuer: "https://keycloak.keycloak.svc.cluster.local:443/realms/openshell" caConfigMapName: "openshell-keycloak-ca" @@ -30,9 +30,9 @@ server: audience: "openshell-cli" # Short TTL for dev so JWKS key rotation is picked up quickly. # Use 3600 (default) in production. - jwksTtl: 60 + jwks_ttl: 60 # Keycloak puts realm roles at realm_access.roles in the JWT. - rolesClaim: "realm_access.roles" + roles_claim: "realm_access.roles" # Leave both empty for authentication-only mode (any valid token is accepted). - adminRole: "openshell-admin" - userRole: "openshell-user" + admin_role: "openshell-admin" + user_role: "openshell-user" diff --git a/deploy/helm/openshell/ci/values-openshift-e2e.yaml b/deploy/helm/openshell/ci/values-openshift-e2e.yaml index c470f3ce84..2602de3c9d 100644 --- a/deploy/helm/openshell/ci/values-openshift-e2e.yaml +++ b/deploy/helm/openshell/ci/values-openshift-e2e.yaml @@ -25,21 +25,22 @@ # is true and mTLS is MANDATORY at the TLS handshake — a caller with only the # Route URL and no client certificate is rejected before any RPC. The passthrough # Route terminates TLS at the gateway pod, so this holds end-to-end. -# `allowUnauthenticatedUsers` only promotes the already cert-verified caller to a +# `allow_unauthenticated_users` only promotes the already cert-verified caller to a # dev principal at the app layer (mtls_auth is unsupported with the Kubernetes # driver). Both are required together; the client certificate is the access gate. gateway: image: pullPolicy: Always -supervisor: - image: - pullPolicy: Always - server: disableTls: false - auth: - allowUnauthenticatedUsers: true + +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_image_pull_policy: always + image_pull_policy: always + openshell.gateway.auth: + allow_unauthenticated_users: true openshiftRoute: enabled: true diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 4c017f669e..e34fd2a8eb 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -2,6 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). +gatewayConfig: + openshell.drivers.kubernetes: + image_pull_policy: if_not_present + openshell.gateway.otlp: + endpoint: http://openshell-collector.observability.svc.cluster.local:4317 + openshell.gateway.auth: + allow_unauthenticated_users: true + server: otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 diff --git a/deploy/helm/openshell/ci/values-spire.yaml b/deploy/helm/openshell/ci/values-spire.yaml index 201520e817..f71cea7134 100644 --- a/deploy/helm/openshell/ci/values-spire.yaml +++ b/deploy/helm/openshell/ci/values-spire.yaml @@ -2,8 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # OpenShell overlay for local SPIRE-backed provider token grants. -server: - providerTokenGrants: - spiffe: - enabled: true - workloadApiSocketPath: /spiffe-workload-api/spire-agent.sock +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml index e9f88846c4..f75c0f643e 100644 --- a/deploy/helm/openshell/ci/values-workspace-managed.yaml +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -3,9 +3,8 @@ # # E2E overlay: deploy the gateway in managed workspace mode. # Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. -server: - sandboxImagePullSecrets: - - name: e2e-regcred - drivers: - kubernetes: - workspaceMode: "managed" +gatewayConfig: + openshell.drivers.kubernetes: + image_pull_secrets: + - name: e2e-regcred + workspace_mode: managed diff --git a/deploy/helm/openshell/ci/values-workspace-operator.yaml b/deploy/helm/openshell/ci/values-workspace-operator.yaml index 8d895e4e98..a543923683 100644 --- a/deploy/helm/openshell/ci/values-workspace-operator.yaml +++ b/deploy/helm/openshell/ci/values-workspace-operator.yaml @@ -3,8 +3,7 @@ # # E2E overlay: deploy the gateway in operator workspace mode. # Namespaces must be pre-provisioned and labeled before sandbox creation. -server: - drivers: - kubernetes: - workspaceMode: "operator" - operatorNamespaceLabel: "openshell.ai/e2e-operator-workspace=true" +gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator + operator_namespace_label: "openshell.ai/e2e-operator-workspace=true" diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 2d7bf892e2..202f82a955 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -148,6 +148,7 @@ deploy: supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' sandboxRuntime.image.repository: '{{.IMAGE_REPO_openshell_sandbox}}' sandboxRuntime.image.tag: '{{.IMAGE_TAG_openshell_sandbox}}' + gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image: '{{.IMAGE_REPO_openshell_supervisor}}:{{.IMAGE_TAG_openshell_supervisor}}' profiles: # Full HA test path: installs Envoy Gateway and layers both HA replicas and # Gateway API routing values onto the OpenShell release. From 130ec4f3c9c9a29d63dad6774bf56833f503106c Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:30:43 +0100 Subject: [PATCH 12/29] refactor(helm): remove gateway configuration shadow values Signed-off-by: Gaizka Menendez Hernandez --- .../openshell/templates/_gateway-workload.tpl | 8 +- deploy/helm/openshell/templates/_helpers.tpl | 2 +- docs/how-it-works/gateways/authentication.mdx | 20 +++-- docs/kubernetes/access-control.mdx | 74 ++++++++++++------- docs/kubernetes/ingress.mdx | 20 +++-- docs/kubernetes/managing-certificates.mdx | 4 +- docs/security/best-practices.mdx | 4 +- 7 files changed, 78 insertions(+), 54 deletions(-) diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index cd645abb47..019455f46a 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -119,7 +119,7 @@ spec: # mounted at /etc/openshell/gateway.toml. Secret-bearing settings use # env vars that the TOML references by name. Some process-level # settings consumed by libraries outside gateway code also remain here. - {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} # OIDC issuer custom-CA: rustls/reqwest read SSL_CERT_FILE for # outbound TLS verification. This is a process-level env var # consumed by the TLS stack itself, not by gateway code, so it @@ -169,7 +169,7 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} - name: oidc-ca mountPath: /etc/openshell-tls/oidc-ca readOnly: true @@ -270,10 +270,10 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} - name: oidc-ca configMap: - name: {{ .Values.server.oidc.caConfigMapName }} + name: {{ .Values.oidc.caConfigMapName }} {{- end }} {{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }} - name: vault-ca diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 944488be8f..9d5180e7d4 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -441,7 +441,7 @@ Validate chart values that Helm would otherwise accept silently. {{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} -{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} +{{- fail "gatewayConfig.openshell.drivers.kubernetes.workspace_mode must be one of: shared, managed, operator." -}} {{- end -}} {{- if kindIs "invalid" .Values.server.tls.clientCaSecretName -}} {{- fail "server.tls.clientCaSecretName cannot be null; omit the key to use the chart default (openshell-server-client-ca), or set to \"\" to disable client certificate verification for HTTPS-only mode" -}} diff --git a/docs/how-it-works/gateways/authentication.mdx b/docs/how-it-works/gateways/authentication.mdx index 3dbf6e524f..3594a3e82f 100644 --- a/docs/how-it-works/gateways/authentication.mdx +++ b/docs/how-it-works/gateways/authentication.mdx @@ -100,20 +100,18 @@ The same settings are available through environment variables: | `OPENSHELL_OIDC_USER_ROLE` | Role required for standard user operations. | `openshell-user` | | `OPENSHELL_OIDC_SCOPES_CLAIM` | Dot-separated claim path containing scopes. Empty disables scope enforcement. | Empty | -For Helm deployments, set the same values under `server.oidc`: +For Helm deployments, set the same values in `gatewayConfig` under the +`openshell.gateway.oidc` TOML table: ```yaml -server: - oidc: +gatewayConfig: + openshell.gateway.oidc: issuer: https://idp.example.com/realms/openshell audience: openshell-cli - # Only needed when discovery returns a deliberately separate JWKS origin. - jwksAllowedOrigins: - - https://keys.example.com - rolesClaim: realm_access.roles - adminRole: openshell-admin - userRole: openshell-user - scopesClaim: "" + roles_claim: realm_access.roles + admin_role: openshell-admin + user_role: openshell-user + scopes_claim: "" ``` The gateway requires HTTPS for OIDC discovery and JWKS retrieval, rejects @@ -271,7 +269,7 @@ This is transparent to the user. All CLI commands work the same regardless of wh ### Plaintext -When a gateway is deployed with `server.disableTls=true`, TLS is disabled entirely. The CLI connects over plain HTTP/2. This mode is intended for local port-forwarding or gateways behind a trusted reverse proxy or tunnel that handles TLS termination externally. +When a gateway is deployed without a TLS table in `gatewayConfig`, TLS is disabled entirely. The CLI connects over plain HTTP/2. This mode is intended for local port-forwarding or gateways behind a trusted reverse proxy or tunnel that handles TLS termination externally. Register a plaintext gateway with an explicit `http://` endpoint: diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 5b295e3211..99e49d912d 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -23,7 +23,7 @@ For how the CLI resolves gateways and stores credentials, refer to [Gateway Auth Kubernetes sandbox supervisors authenticate back to the gateway as sandbox workloads. By default, the Kubernetes compute driver validates each projected ServiceAccount token and returns the authenticated sandbox ID plus a stable runtime identity to the gateway. The gateway requires that identity to match the namespace, immutable Sandbox resource UID, and supervisor Pod UID recorded during provisioning before it mints its own sandbox JWT. -Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `server.providerTokenGrants.spiffe.enabled=true` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping the projected ServiceAccount token bootstrap and gateway-minted sandbox JWT path. +Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping the projected ServiceAccount token bootstrap and gateway-minted sandbox JWT path. Provider token grants require a SPIFFE implementation such as SPIRE and identities for the gateway and sandbox pods. The repository's local SPIRE overlay assigns sandbox IDs from the pod's `openshell.ai/sandbox-id` annotation, but the gateway validation path only requires the supervisor SVID to be valid and in the same SPIFFE trust domain as the gateway SVID. Provider profiles with `token_grant` metadata cause the sandbox supervisor to request JWT-SVIDs and exchange them for upstream OAuth2 access tokens. Token-exchange profiles also require a gateway SPIFFE identity because the gateway brokers the intermediate token exchange with its own JWT-SVID. @@ -31,15 +31,23 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIF ## OIDC User Authentication -Set `server.oidc.issuer` to enable OIDC. The gateway validates the `Authorization: Bearer ` header on every request against the issuer's JWKS endpoint. It accepts JWTs signed with RS256, RS384, RS512, PS256, PS384, PS512, ES256, ES384, or EdDSA (Ed25519) keys published in the issuer JWKS. Okta tenants that sign access tokens with ES256 work without additional gateway configuration. +Set `gatewayConfig.openshell.gateway.oidc.issuer` to enable OIDC. The gateway validates the `Authorization: Bearer ` header on every request against the issuer's JWKS endpoint. It accepts JWTs signed with RS256, RS384, RS512, PS256, PS384, PS512, ES256, ES384, or EdDSA (Ed25519) keys published in the issuer JWKS. Okta tenants that sign access tokens with ES256 work without additional gateway configuration. + +Create `oidc-values.yaml`: + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://your-idp.example.com/realms/openshell + audience: openshell-cli +``` ```shell helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.oidc.issuer=https://your-idp.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set server.tls.clientCaSecretName="" ``` @@ -51,16 +59,13 @@ The `audience` value must match the client ID configured in your identity provid | Value | Default | Purpose | |---|---|---| -| `server.oidc.issuer` | `""` | OIDC issuer URL. Empty disables OIDC. | -| `server.oidc.dangerouslyAllowInsecureHttp` | `false` | Development-only acknowledgement for numeric-loopback HTTP. It does not allow cluster-service or remote HTTP issuers. | -| `server.oidc.jwksAllowedOrigins` | `[]` | Additional trusted HTTPS origins allowed to serve JWKS. | -| `server.oidc.caConfigMapName` | `""` | ConfigMap containing the private issuer CA in `ca.crt`. | -| `server.oidc.audience` | `openshell-cli` | Expected `aud` claim in the JWT. | -| `server.oidc.jwksTtl` | `3600` | JWKS key cache TTL in seconds. Must be greater than zero. | -| `server.oidc.rolesClaim` | `""` | Dot-separated path to the roles array in JWT claims. | -| `server.oidc.adminRole` | `""` | Role name that grants admin access. | -| `server.oidc.userRole` | `""` | Role name that grants standard user access. | -| `server.oidc.scopesClaim` | `""` | Dot-separated path to the scopes array in JWT claims. | +| `gatewayConfig.openshell.gateway.oidc.issuer` | `""` | OIDC issuer URL. Empty disables OIDC. | +| `gatewayConfig.openshell.gateway.oidc.audience` | `openshell-cli` | Expected `aud` claim in the JWT. | +| `gatewayConfig.openshell.gateway.oidc.jwks_ttl_secs` | `3600` | JWKS key cache TTL in seconds. Must be greater than zero. | +| `gatewayConfig.openshell.gateway.oidc.roles_claim` | `""` | Dot-separated path to the roles array in JWT claims. | +| `gatewayConfig.openshell.gateway.oidc.admin_role` | `""` | Role name that grants admin access. | +| `gatewayConfig.openshell.gateway.oidc.user_role` | `""` | Role name that grants standard user access. | +| `gatewayConfig.openshell.gateway.oidc.scopes_claim` | `""` | Dot-separated path to the scopes array in JWT claims. | The issuer must use HTTPS. The gateway rejects discovery and JWKS redirects, limits response sizes, requires a JSON media type, and rejects a `jwks_uri` on @@ -69,7 +74,7 @@ a different origin unless that origin appears in `jwksAllowedOrigins`. Use ### Auth-only mode vs. RBAC mode -Leave both `adminRole` and `userRole` empty to use auth-only mode: any request with a valid JWT from the configured issuer is accepted, but no role distinction is enforced. +Leave both `admin_role` and `user_role` empty to use auth-only mode: any request with a valid JWT from the configured issuer is accepted, but no role distinction is enforced. Set both values to enable RBAC mode, where the gateway checks the role claim and enforces access based on the assigned role: @@ -78,15 +83,23 @@ helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.oidc.issuer=https://your-idp.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ - --set server.tls.clientCaSecretName="" \ - --set server.oidc.rolesClaim=realm_access.roles \ - --set server.oidc.adminRole=openshell-admin \ - --set server.oidc.userRole=openshell-user + --values oidc-rbac-values.yaml \ + --set server.tls.clientCaSecretName="" ``` -Both `adminRole` and `userRole` must be set, or both must be empty. Setting only one is not supported. +Create `oidc-rbac-values.yaml` with the OIDC values above plus: + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://your-idp.example.com/realms/openshell + audience: openshell-cli + roles_claim: realm_access.roles + admin_role: openshell-admin + user_role: openshell-user +``` + +Both `admin_role` and `user_role` must be set, or both must be empty. Setting only one is not supported. OIDC RBAC is method-level authorization. It controls which API operations a caller can perform, but provider and sandbox records are not owned by individual OIDC subjects. In shared clusters, treat provider credentials as gateway-wide resources and use separate gateways or external tenancy controls when users must not see or attach each other's providers and sandboxes. @@ -107,7 +120,15 @@ helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.auth.allowUnauthenticatedUsers=true + --values reverse-proxy-values.yaml +``` + +Create `reverse-proxy-values.yaml`: + +```yaml +gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: true ``` The gateway still serves TLS and sandbox supervisors still authenticate with gateway-minted sandbox JWTs. User-facing CLI/API calls without OIDC or mTLS credentials are accepted as an unauthenticated local developer principal. The proxy is responsible for authenticating callers and forwarding only authorized traffic. @@ -116,9 +137,10 @@ When the gateway terminates TLS directly and callers connect without client cert To also disable TLS entirely (when the proxy terminates TLS before the request reaches the gateway): -```shell - --set server.disableTls=true \ - --set server.auth.allowUnauthenticatedUsers=true +```yaml +gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: true ``` diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 6261ffb041..192a3cfb50 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -106,7 +106,14 @@ The Secret may also be issued by cert-manager, or you can reference the chart's ### Install with HTTPS termination -Enable an HTTPS listener, point it at the Secret, disable gateway-pod TLS so Envoy forwards plaintext, and configure an OIDC issuer for client identity: +Enable an HTTPS listener, point it at the Secret, configure the gateway for a plaintext backend, and provide OIDC settings in `oidc-values.yaml`: + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://keycloak.example.com/realms/openshell + audience: openshell-cli +``` ```shell helm upgrade --install openshell \ @@ -119,9 +126,7 @@ helm upgrade --install openshell \ --set grpcRoute.gateway.listener.protocol=HTTPS \ --set grpcRoute.gateway.listener.port=443 \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ - --set server.disableTls=true \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` @@ -147,7 +152,7 @@ As an alternative to the plaintext backend path above, the chart can create a `B client → HTTPS → Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod ``` -This keeps TLS on the gateway pod rather than disabling it with `server.disableTls=true`. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. +This keeps TLS on the gateway pod. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. BackendTLSPolicy is a standard Gateway API resource. It is supported on OpenShift 4.22+ (via the OpenShift gateway controller) and on other platforms where the Gateway API implementation supports it (check your controller's documentation). @@ -168,12 +173,11 @@ helm upgrade --install openshell \ --set grpcRoute.gateway.listener.port=443 \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ --set grpcRoute.backendTLSPolicy.enabled=true \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` -Note that `server.disableTls` is **not** set — the gateway pod continues to serve TLS — but `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The chart will fail the install if you try to enable both `grpcRoute.backendTLSPolicy.enabled=true` and `server.tls.enableMtls=true` simultaneously. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured via `server.oidc.issuer`). +The gateway pod continues to serve TLS, while `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The chart will fail the install if you try to enable both `grpcRoute.backendTLSPolicy.enabled=true` and `server.tls.enableMtls=true` simultaneously. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured in `gatewayConfig.openshell.gateway.oidc`). The example above uses the default `pkiInitJob` for TLS, which creates the backend CA ConfigMap immediately. If using cert-manager instead (`--set certManager.enabled=true`), the Certificate resources are regular release objects, and a separate post-install/post-upgrade Job (`-certgen-backend-ca`) polls for up to 120 seconds waiting for cert-manager to issue the server certificate, then creates the backend CA ConfigMap. This means a single `helm install` is sufficient in most cases. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index 322815cbf1..71348f7735 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -103,9 +103,9 @@ validates this at install time and fails with an actionable error if `certManager.serverDnsNames` contains internal-only entries while `serverIssuerRef` is set. -You do **not** need to set `server.grpcEndpoint` to the external hostname. +You do **not** need to set `gatewayConfig.openshell.drivers.kubernetes.grpc_endpoint` to the external hostname. Supervisors connect via the internal service name automatically. Setting -`server.grpcEndpoint` to an external hostname would cause supervisors to +`grpc_endpoint` to an external hostname would cause supervisors to receive the ACME certificate (via SNI) which they cannot verify against the chart CA. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index a310900b98..805545e9a7 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | +| Default | Disabled. Set `gatewayConfig.openshell.drivers.kubernetes.enable_user_namespaces: true` in Helm values to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | @@ -265,7 +265,7 @@ Gateway transport uses TLS, with client certificate checks available where the d | Aspect | Detail | |---|---| | Default | Local TLS bundles enable mTLS user authentication for single-user local gateways. Helm deployments generate mTLS certificates for transport, while sandbox supervisors authenticate API calls with gateway-minted sandbox JWTs. TLS-enabled loopback gateways also accept plaintext HTTP for sandbox service hostnames by default. | -| What you can change | Configure OIDC or a trusted access proxy for multi-user gateways, set `OPENSHELL_ENABLE_MTLS_AUTH=true` for local single-user gateways, enable `server.auth.allowUnauthenticatedUsers=true` only for trusted local Kubernetes development or a fully trusted proxy, disable TLS only for trusted reverse-proxy setups, or disable loopback service HTTP with `--enable-loopback-service-http=false`. | +| What you can change | Configure OIDC or a trusted access proxy for multi-user gateways, set `OPENSHELL_ENABLE_MTLS_AUTH=true` for local single-user gateways, enable `gatewayConfig.openshell.gateway.auth.allow_unauthenticated_users=true` only for trusted local Kubernetes development or a fully trusted proxy, disable TLS only for trusted reverse-proxy setups, or disable loopback service HTTP with `--enable-loopback-service-http=false`. | | Risk if relaxed | Disabling TLS removes transport-level protection entirely. Allowing unauthenticated users removes the gateway user-auth boundary and must not be exposed to shared or public networks. Treating transport certificates as shared user identity in Kubernetes would collapse user and sandbox trust boundaries. Loopback service HTTP is local-only and rejects cross-origin browser requests, but any local process can still reach exposed service URLs directly. | | Recommendation | Use local mTLS user authentication only for single-user Docker, Podman, and VM gateways. Use OIDC or a trusted access proxy for Kubernetes and shared deployments. | From 1dc68945c590ab43a9c64422809c7c13cd5a263e Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:41:07 +0100 Subject: [PATCH 13/29] refactor(helm): align runtime config with resources Signed-off-by: Gaizka Menendez Hernandez --- .../helm/openshell/ci/values-workspace-managed.yaml | 2 +- deploy/helm/openshell/templates/_toml.tpl | 11 ++++++++++- deploy/helm/openshell/templates/clusterrole.yaml | 7 ++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml index f75c0f643e..3c3ef72526 100644 --- a/deploy/helm/openshell/ci/values-workspace-managed.yaml +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -6,5 +6,5 @@ gatewayConfig: openshell.drivers.kubernetes: image_pull_secrets: - - name: e2e-regcred + - e2e-regcred workspace_mode: managed diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 7bb84fb5ea..ec1154563e 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -98,7 +98,16 @@ field must not require a Helm template change. {{/* Render the top-level gatewayConfig map as deterministic TOML tables. */}} {{- define "openshell.gatewayConfigToml" -}} {{- $root := . -}} -{{- $config := .Values.gatewayConfig | default dict -}} +{{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} +{{- if .Values.server.disableTls -}} +{{- $gateway := get $config "openshell.gateway" | default dict -}} +{{- $_ := set $gateway "disable_tls" true -}} +{{- $_ := set $config "openshell.gateway" $gateway -}} +{{- $_ := unset $config "openshell.gateway.tls" -}} +{{- $kubernetes := get $config "openshell.drivers.kubernetes" | default dict -}} +{{- $_ := unset $kubernetes "client_tls_secret_name" -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} +{{- end -}} {{- range $tableName := keys $config | sortAlpha -}} {{- $fields := get $config $tableName -}} {{- if ne $fields nil -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 8e7bf23327..eeb01c626a 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -5,6 +5,11 @@ {{- $gatewayConfig := .Values.gatewayConfig | default dict -}} {{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" }} +{{- $managedSshIngress := get $gatewayConfig "openshell.drivers.kubernetes.managed_ssh_ingress" | default dict -}} +{{- $managedSshIngressEnabled := true -}} +{{- if hasKey $managedSshIngress "enabled" -}} +{{- $managedSshIngressEnabled = get $managedSshIngress "enabled" -}} +{{- end -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -167,7 +172,7 @@ rules: verbs: - create - get - {{- if .Values.networkPolicy.enabled }} + {{- if $managedSshIngressEnabled }} # Apply gateway-only SSH ingress isolation in managed namespaces. - apiGroups: - networking.k8s.io From f0c4a8c8b7350d897244042d0e5e4f44a3ffb8ee Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:47:03 +0100 Subject: [PATCH 14/29] docs(helm): add gateway config migration guide Signed-off-by: Gaizka Menendez Hernandez --- docs/kubernetes/migrate-gateway-config.mdx | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/kubernetes/migrate-gateway-config.mdx diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx new file mode 100644 index 0000000000..db6b303ea3 --- /dev/null +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -0,0 +1,65 @@ +# Migrate Helm gateway configuration to `gatewayConfig` + +Chart schema v2 replaces application-specific Helm values with one non-secret +`gatewayConfig` map. Its top-level keys are TOML tables; nested maps are TOML +inline tables. Kubernetes resource references (Secrets, certificates, Services, +and mounts) remain chart values. + +## Breaking upgrade + +Remove every application-only `server.*`, `supervisor.*`, and `upstreamProxy.*` +setting before upgrading. There is no compatibility translation. Kubernetes pull +policy spellings such as `Always` are not aliases: use `always`, +`if_not_present`, or `never`. Do not put passwords, private keys, or +`database_url` in `gatewayConfig`; use Secret-backed environment configuration. + +Keep `server.disableTls`, TLS Secret names, `server.externalDbSecret`, +`server.sandboxNamespace`, `oidc.caConfigMapName`, certificate settings and +Service settings: these own Kubernetes resources. The chart derives their +runtime counterparts. + +## Mapping + +| Removed value | New location | +| --- | --- | +| `server.name`, `logLevel`, `enableLoopbackServiceHttp`, `policyValidationFailureMode` | `openshell.gateway.{name,log_level,enable_loopback_service_http,policy_validation_failure_mode}` | +| `server.grpcRateLimit.{requests,windowSeconds}` | `openshell.gateway.{grpc_rate_limit_requests,grpc_rate_limit_window_seconds}` | +| `server.otlp.{endpoint,serviceName}` | `openshell.gateway.otlp.{endpoint,service_name}` | +| `server.auth.allowUnauthenticatedUsers` | `openshell.gateway.auth.allow_unauthenticated_users` | +| `server.oidc.*` | `openshell.gateway.oidc` with snake_case keys | +| `server.sandboxImage*`, workspace storage/runtime/AppArmor/user namespace settings | `openshell.drivers.kubernetes` | +| `server.drivers.kubernetes.*` | `openshell.drivers.kubernetes` | +| `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.gateway.gateway_jwt` or `openshell.drivers.kubernetes.sa_token_ttl_secs` | +| `supervisor.*` | `openshell.drivers.kubernetes` and `.sidecar` | +| `upstreamProxy.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}` | +| credential-driver runtime settings | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | + +## Examples + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://idp.example/realms/openshell + audience: openshell-cli + jwks_ttl_secs: 3600 + openshell.gateway.otlp: + endpoint: http://otel-collector:4317 + service_name: production-gateway + openshell.drivers.kubernetes: + default_image: registry.example/sandbox:latest + image_pull_policy: if_not_present + workspace_mode: managed + grpc_rate_limit_requests: null + openshell.gateway: + grpc_rate_limit_requests: 120 + grpc_rate_limit_window_seconds: 60 + openshell.credential_drivers.kubernetes-secrets: + namespace: provider-secrets +``` + +For Vault, set `openshell.gateway.credential_drivers: [vault]` and configure +`openshell.credential_drivers.vault`. For a proxy, set `https_proxy` and the +`proxy_auth_secret_name`/`proxy_auth_secret_key` references in the Kubernetes +driver table. TLS remains Secret-backed; set `server.disableTls: true` only for +trusted external termination. A null map field omits it; null array elements +are invalid. Strings evaluate Helm `tpl`; other values do not. From 1fa411adc5225c8ed916fdfb7976fcbb773be288 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 17:57:22 +0100 Subject: [PATCH 15/29] test(helm): validate rendered gateway config Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/ci/values-keycloak.yaml | 2 +- .../fixtures/parser-validation-values.yaml | 34 +++++ deploy/helm/test-gateway-config-parser.sh | 121 ++++++++++++++++++ tasks/helm.toml | 1 + 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml create mode 100755 deploy/helm/test-gateway-config-parser.sh diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index 1e4dc45b6b..4c76a8f841 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -30,7 +30,7 @@ gatewayConfig: audience: "openshell-cli" # Short TTL for dev so JWKS key rotation is picked up quickly. # Use 3600 (default) in production. - jwks_ttl: 60 + jwks_ttl_secs: 60 # Keycloak puts realm roles at realm_access.roles in the JWT. roles_claim: "realm_access.roles" # Leave both empty for authentication-only mode (any valid token is accepted). diff --git a/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml new file mode 100644 index 0000000000..12a00174b3 --- /dev/null +++ b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Inputs used by deploy/helm/test-gateway-config-parser.sh. The driver table is +# intentionally opaque to the gateway loader, which makes it a safe way to +# exercise every TOML shape supported by the chart serializer. +gatewayConfig: + openshell.gateway: + name: '{{ .Release.Namespace }}/{{ .Release.Name }} "quoted" \\ path' + server_sans: + - localhost + - gateway.example.test + provider_profile_sources: + - type: builtin + - type: user + openshell.drivers.parser-validation: + boolean_value: true + integer_value: 42 + float_value: 1.5 + empty_value: "" + scalar_array: + - first + - second + inline_map: + alpha: false + zebra: 2 + map_array: + - name: first + enabled: true + - name: second + enabled: false + escaped_string: 'quotes " and backslash \\ and a newline + are preserved' + omitted_value: null diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh new file mode 100755 index 0000000000..538db9cc46 --- /dev/null +++ b/deploy/helm/test-gateway-config-parser.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Render the chart exactly as an operator would, then validate gateway.toml +# with the gateway binary. Helm unit tests cover template-level assertions; +# this test makes the Rust loader the compatibility authority. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +chart="${repo_root}/deploy/helm/openshell" +fixture="${chart}/tests/fixtures/parser-validation-values.yaml" +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT + +cargo build --quiet --package openshell-gateway +gateway_bin="${repo_root}/target/debug/openshell-gateway" + +render() { + local output="$1" + shift + helm template parser-validation "${chart}" \ + --namespace parser-namespace \ + --set agentSandbox.preflight.enabled=false \ + "$@" >"${output}" +} + +extract_toml() { + local manifest="$1" + local toml="$2" + yq ea -e -r \ + 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .data."gateway.toml"' \ + "${manifest}" >"${toml}" +} + +preflight() { + local toml="$1" + "${gateway_bin}" config preflight --path "${toml}" +} + +render "${work_dir}/default.yaml" +extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" +preflight "${work_dir}/default.toml" + +# Exercise all serializer shapes through a raw driver table and prove that +# string-only tpl expansion, TOML escaping, and null omission survive parsing. +render "${work_dir}/shapes.yaml" --values "${fixture}" +extract_toml "${work_dir}/shapes.yaml" "${work_dir}/shapes.toml" +preflight "${work_dir}/shapes.toml" +grep -F 'name = "parser-namespace/parser-validation' "${work_dir}/shapes.toml" >/dev/null +grep -F 'empty_value = ""' "${work_dir}/shapes.toml" >/dev/null +if grep -Fq 'omitted_value' "${work_dir}/shapes.toml"; then + echo "null gatewayConfig values must be omitted from gateway.toml" >&2 + exit 1 +fi + +# The loader gives absent and empty credential-driver selection deliberately +# different meanings: absent retains encrypted storage, while an empty list is +# an invalid and ambiguous external-driver selection. +render "${work_dir}/credential-drivers-absent.yaml" \ + --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=null' +extract_toml "${work_dir}/credential-drivers-absent.yaml" "${work_dir}/credential-drivers-absent.toml" +preflight "${work_dir}/credential-drivers-absent.toml" +if grep -Fq 'credential_drivers' "${work_dir}/credential-drivers-absent.toml"; then + echo "null credential_drivers must be absent from gateway.toml" >&2 + exit 1 +fi +render "${work_dir}/credential-drivers-empty.yaml" \ + --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=[]' +extract_toml "${work_dir}/credential-drivers-empty.yaml" "${work_dir}/credential-drivers-empty.toml" +if preflight "${work_dir}/credential-drivers-empty.toml" >"${work_dir}/credential-drivers-empty.err" 2>&1; then + echo "the gateway loader accepted empty credential_drivers" >&2 + exit 1 +fi + +# Unknown non-secret values are intentionally serializable by Helm, but must +# fail at the Rust schema boundary rather than being silently ignored. +render "${work_dir}/unknown.yaml" \ + --set-string 'gatewayConfig.openshell\.gateway.unknown_non_secret=accepted-by-helm' +extract_toml "${work_dir}/unknown.yaml" "${work_dir}/unknown.toml" +if preflight "${work_dir}/unknown.toml" >"${work_dir}/unknown.err" 2>&1; then + echo "the gateway loader accepted an unknown non-secret field" >&2 + exit 1 +fi + +# Secrets remain outside the ConfigMap even when their Secret reference is +# rendered into the workload environment. +render "${work_dir}/secret-boundary.yaml" --set server.externalDbSecret=parser-database +extract_toml "${work_dir}/secret-boundary.yaml" "${work_dir}/secret-boundary.toml" +if grep -Eqi 'parser-database|postgresql:|password' "${work_dir}/secret-boundary.toml"; then + echo "gateway.toml contains Secret-backed database material" >&2 + exit 1 +fi + +# A ConfigMap-only mutation must change the StatefulSet checksum and trigger a +# rollout. Check this against full Helm output, not just a template fragment. +default_checksum="$(yq ea -e -r 'select(.kind == "StatefulSet") | .spec.template.metadata.annotations."checksum/gateway-config"' "${work_dir}/default.yaml")" +render "${work_dir}/checksum.yaml" \ + --set-string 'gatewayConfig.openshell\.gateway.log_level=debug' +changed_checksum="$(yq ea -e -r 'select(.kind == "StatefulSet") | .spec.template.metadata.annotations."checksum/gateway-config"' "${work_dir}/checksum.yaml")" +if [[ -z "${default_checksum}" || "${default_checksum}" == "${changed_checksum}" ]]; then + echo "gateway ConfigMap changes must update the StatefulSet checksum" >&2 + exit 1 +fi + +# All maintained CI/dev overlays must render a loader-valid configuration. +for values in "${chart}"/ci/values-*.yaml; do + name="$(basename "${values}" .yaml)" + if [[ "${name}" == "values-corporate-proxy-e2e" ]]; then + # The e2e wrapper supplies this generated URL alongside the overlay. + render "${work_dir}/${name}.yaml" \ + --values "${values}" \ + --set-string 'gatewayConfig.openshell\.drivers\.kubernetes.https_proxy=http://proxy.corp.example:8080' + else + render "${work_dir}/${name}.yaml" --values "${values}" + fi + extract_toml "${work_dir}/${name}.yaml" "${work_dir}/${name}.toml" + preflight "${work_dir}/${name}.toml" +done + +echo "rendered gateway TOML passed Rust loader validation" diff --git a/tasks/helm.toml b/tasks/helm.toml index e2a4e001ce..c95247597f 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -65,6 +65,7 @@ run = """ helm unittest deploy/helm/openshell helm unittest deploy/helm/openshell-workspace deploy/helm/test-split-ownership.sh + deploy/helm/test-gateway-config-parser.sh """ run_windows = "echo Skipping helm:test: Helm validation is not part of the native Windows lane." From e6cb3c351efc86ee5e857f755c4855cc896ada7e Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 18:04:32 +0100 Subject: [PATCH 16/29] test(helm): cover config resource coherence Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/_toml.tpl | 7 ++ .../helm/test-gateway-resource-coherence.sh | 74 +++++++++++++++++++ e2e/with-kube-gateway.sh | 52 +++++++++++++ tasks/helm.toml | 1 + tasks/test.toml | 1 + 5 files changed, 135 insertions(+) create mode 100755 deploy/helm/test-gateway-resource-coherence.sh diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index ec1154563e..3049ab03a7 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -108,6 +108,13 @@ field must not require a Helm template change. {{- $_ := unset $kubernetes "client_tls_secret_name" -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} {{- end -}} +{{- if and (not .Values.server.disableTls) .Values.certManager.serverIssuerRef.name -}} +{{- $gatewayTls := get $config "openshell.gateway.tls" | default dict -}} +{{- $_ := set $gatewayTls "external_cert_path" "/etc/openshell-tls/server-external/tls.crt" -}} +{{- $_ := set $gatewayTls "external_key_path" "/etc/openshell-tls/server-external/tls.key" -}} +{{- $_ := set $gatewayTls "external_server_names" (deepCopy (.Values.certManager.serverDnsNames | default list)) -}} +{{- $_ := set $config "openshell.gateway.tls" $gatewayTls -}} +{{- end -}} {{- range $tableName := keys $config | sortAlpha -}} {{- $fields := get $config $tableName -}} {{- if ne $fields nil -}} diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh new file mode 100755 index 0000000000..20183c29fd --- /dev/null +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Assert relationships that span rendered Kubernetes objects and gateway.toml. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +chart="${repo_root}/deploy/helm/openshell" +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT + +render() { + local name="$1" + shift + helm template resource-coherence "${chart}" \ + --namespace resource-namespace \ + --set agentSandbox.preflight.enabled=false \ + "$@" >"${work_dir}/${name}.yaml" +} + +toml() { + yq ea -e -r \ + 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .data."gateway.toml"' \ + "$1" >"$2" +} + +workload_value() { + local manifest="$1" + local expression="$2" + yq ea -e -r "select(.kind == \"StatefulSet\" or .kind == \"Deployment\") | ${expression}" "${manifest}" +} + +render default +default_manifest="${work_dir}/default.yaml" +default_toml="${work_dir}/default.toml" +toml "${default_manifest}" "${default_toml}" +service_name="$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${default_manifest}")" +service_port="$(yq ea -e -r 'select(.kind == "Service") | .spec.ports[] | select(.name == "grpc") | .port' "${default_manifest}")" +workload_port="$(workload_value "${default_manifest}" '.spec.template.spec.containers[] | select(.name == "openshell-gateway") | .ports[] | select(.name == "grpc") | .containerPort')" +config_map="$(yq ea -e -r 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .metadata.name' "${default_manifest}")" +mounted_config_map="$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "gateway-config") | .configMap.name')" +[[ "${service_port}" == "${workload_port}" && "${config_map}" == "${mounted_config_map}" ]] +grep -F "bind_address = \"0.0.0.0:${service_port}\"" "${default_toml}" >/dev/null +grep -F "grpc_endpoint = \"https://${service_name}.resource-namespace.svc.cluster.local:${service_port}\"" "${default_toml}" >/dev/null +grep -F '[openshell.gateway.tls]' "${default_toml}" >/dev/null +[[ "$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-cert") | .secret.secretName')" == "openshell-server-tls" ]] +[[ "$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-client-ca") | .secret.secretName')" == "openshell-server-tls" ]] + +render tls-disabled --values "${chart}/ci/values-tls-disabled.yaml" +tls_disabled_manifest="${work_dir}/tls-disabled.yaml" +tls_disabled_toml="${work_dir}/tls-disabled.toml" +toml "${tls_disabled_manifest}" "${tls_disabled_toml}" +grep -F 'disable_tls = true' "${tls_disabled_toml}" >/dev/null +if grep -Fq '[openshell.gateway.tls]' "${tls_disabled_toml}" \ + || workload_value "${tls_disabled_manifest}" '.spec.template.spec.volumes[]?.name' | grep -Eq '^(tls-cert|tls-client-ca)$'; then + echo "TLS-disabled runtime configuration and workload mounts disagree" >&2 + exit 1 +fi + +render openshift-route --values "${chart}/ci/values-openshift-route-cert-manager.yaml" +route_manifest="${work_dir}/openshift-route.yaml" +route_toml="${work_dir}/openshift-route.toml" +toml "${route_manifest}" "${route_toml}" +route_service="$(yq ea -e -r 'select(.kind == "Route") | .spec.to.name' "${route_manifest}")" +route_port="$(yq ea -e -r 'select(.kind == "Route") | .spec.port.targetPort' "${route_manifest}")" +[[ "${route_service}" == "$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${route_manifest}")" && "${route_port}" == "grpc" ]] +[[ "$(workload_value "${route_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-external-cert") | .secret.secretName')" == "${route_service}-server-external-tls" ]] +[[ "$(yq ea -e -r 'select(.kind == "Certificate") | .spec.secretName' "${route_manifest}" | grep -Fx "${route_service}-server-external-tls")" == "${route_service}-server-external-tls" ]] +grep -F 'external_cert_path = "/etc/openshell-tls/server-external/tls.crt"' "${route_toml}" >/dev/null +grep -F 'external_key_path = "/etc/openshell-tls/server-external/tls.key"' "${route_toml}" >/dev/null +grep -F 'external_server_names = ["openshell.example.com"]' "${route_toml}" >/dev/null + +echo "gateway Service, TLS, PKI, Route, workload, and runtime config are coherent" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 5d00e11adb..560ac47749 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -157,6 +157,54 @@ kube_workload_ref() { return 1 } +# Verify the ConfigMap checksum causes a live gateway rollout. This belongs in +# the harness, before port-forwards are established, because replacing a pod +# necessarily interrupts any existing port-forward to it. +verify_gateway_config_rollout() { + local workload_ref old_checksum new_checksum old_pod_uid new_pod_uid attempt + local pod_selector="app.kubernetes.io/instance=${RELEASE_NAME},app.kubernetes.io/name=openshell" + + workload_ref="$(kube_workload_ref "${RELEASE_NAME}")" + old_checksum="$(kctl -n "${NAMESPACE}" get "${workload_ref}" -o jsonpath='{.spec.template.metadata.annotations.checksum/gateway-config}')" + old_pod_uid="$(kctl -n "${NAMESPACE}" get pods -l "${pod_selector}" -o jsonpath='{.items[0].metadata.uid}')" + if [[ -z "${old_checksum}" || -z "${old_pod_uid}" ]]; then + echo "ERROR: gateway workload is missing its ConfigMap checksum or ready pod" >&2 + return 1 + fi + + echo "Verifying ConfigMap-only gateway configuration rollout..." + helmctl upgrade "${RELEASE_NAME}" "${ROOT}/deploy/helm/openshell" \ + --namespace "${NAMESPACE}" \ + --reuse-values \ + "${helm_values_args[@]}" \ + --set "fullnameOverride=openshell" \ + --set "image.repository=${REGISTRY_VALUE}/gateway" \ + --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ + --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${helm_extra_args[@]}" \ + "${helm_post_renderer_args[@]}" \ + --set-string 'gatewayConfig.openshell\.gateway.log_level=debug' \ + --wait --timeout 5m + + new_checksum="$(kctl -n "${NAMESPACE}" get "${workload_ref}" -o jsonpath='{.spec.template.metadata.annotations.checksum/gateway-config}')" + if [[ -z "${new_checksum}" || "${new_checksum}" == "${old_checksum}" ]]; then + echo "ERROR: ConfigMap-only gateway configuration change did not update workload checksum" >&2 + return 1 + fi + kctl -n "${NAMESPACE}" rollout status "${workload_ref}" --timeout=5m + + for attempt in $(seq 1 60); do + new_pod_uid="$(kctl -n "${NAMESPACE}" get pods -l "${pod_selector}" -o jsonpath='{.items[0].metadata.uid}')" + if [[ -n "${new_pod_uid}" && "${new_pod_uid}" != "${old_pod_uid}" ]]; then + return 0 + fi + sleep 1 + done + echo "ERROR: gateway workload rolled out without replacing its pod" >&2 + return 1 +} + deploy_postgres_fixture() { local secret_name="$1" local pg_uri @@ -1392,6 +1440,10 @@ else --wait --timeout 5m HELM_INSTALLED=1 + if [ "${OPENSHELL_E2E_KUBE_CONFIG_ROLLOUT:-0}" = "1" ]; then + verify_gateway_config_rollout || exit 1 + fi + if [ -n "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET:-}" ]; then kctl -n "${NAMESPACE}" create secret docker-registry \ "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ diff --git a/tasks/helm.toml b/tasks/helm.toml index c95247597f..feaf624e90 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -66,6 +66,7 @@ run = """ helm unittest deploy/helm/openshell-workspace deploy/helm/test-split-ownership.sh deploy/helm/test-gateway-config-parser.sh + deploy/helm/test-gateway-resource-coherence.sh """ run_windows = "echo Skipping helm:test: Helm validation is not part of the native Windows lane." diff --git a/tasks/test.toml b/tasks/test.toml index c45826f1db..1f41dca535 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -200,6 +200,7 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu ["e2e:kubernetes"] description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST= to scope to one test)" +env = { OPENSHELL_E2E_KUBE_CONFIG_ROLLOUT = "1" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" From 49f226a268b36fefa6efaafd0ee8cf67e7d6cc8f Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Tue, 15 Sep 2026 18:56:31 +0100 Subject: [PATCH 17/29] fix(helm): make Kubernetes E2E deployable Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/templates/backend-tls-policy.yaml | 2 +- deploy/helm/openshell/templates/certgen.yaml | 2 +- deploy/helm/openshell/templates/clusterrole.yaml | 2 +- deploy/helm/openshell/templates/route.yaml | 2 +- deploy/helm/test-gateway-resource-coherence.sh | 9 +++++++++ e2e/with-kube-gateway.sh | 4 ++++ tasks/scripts/setup-zig-cc-wrapper.sh | 5 +++-- 7 files changed, 20 insertions(+), 6 deletions(-) diff --git a/deploy/helm/openshell/templates/backend-tls-policy.yaml b/deploy/helm/openshell/templates/backend-tls-policy.yaml index 8d7a4fd557..1c7e59ec3d 100644 --- a/deploy/helm/openshell/templates/backend-tls-policy.yaml +++ b/deploy/helm/openshell/templates/backend-tls-policy.yaml @@ -1,7 +1,7 @@ +{{- if .Values.grpcRoute.backendTLSPolicy.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.grpcRoute.backendTLSPolicy.enabled }} {{- if .Values.server.disableTls }} {{- fail "grpcRoute.backendTLSPolicy requires the gateway pod to serve TLS; set server.disableTls=false" }} {{- end }} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index f7c9a751d3..db12bec905 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -1,7 +1,7 @@ +{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} {{- $hookName := printf "%s-certgen" (include "openshell.fullname" .) }} {{- $ns := .Release.Namespace }} apiVersion: v1 diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index eeb01c626a..92005b129b 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -9,7 +9,7 @@ {{- $managedSshIngressEnabled := true -}} {{- if hasKey $managedSshIngress "enabled" -}} {{- $managedSshIngressEnabled = get $managedSshIngress "enabled" -}} -{{- end -}} +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/deploy/helm/openshell/templates/route.yaml b/deploy/helm/openshell/templates/route.yaml index 459a0ac462..018e32085b 100644 --- a/deploy/helm/openshell/templates/route.yaml +++ b/deploy/helm/openshell/templates/route.yaml @@ -1,7 +1,7 @@ +{{- if .Values.openshiftRoute.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.openshiftRoute.enabled }} {{- if .Values.server.disableTls }} {{- fail "openshiftRoute.enabled=true requires TLS (server.disableTls must be false) \u2014 a passthrough Route forwards encrypted traffic by SNI, so the gateway must terminate its own TLS." }} {{- end }} diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index 20183c29fd..10301244b6 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -17,6 +17,15 @@ render() { --namespace resource-namespace \ --set agentSandbox.preflight.enabled=false \ "$@" >"${work_dir}/${name}.yaml" + + if ! awk 'BEGIN { RS="---" } + /^\n?# Source:/ && $0 !~ /\napiVersion:/ { + print "rendered an empty or invalid Kubernetes document:" $0 > "/dev/stderr" + exit 1 + }' "${work_dir}/${name}.yaml"; then + echo "${name}: rendered an invalid Kubernetes document" >&2 + exit 1 + fi } toml() { diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 560ac47749..005166e8f0 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1218,6 +1218,10 @@ fi helm_extra_args=() helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") +# Sandboxes pull the supervisor image named by the runtime TOML, not the +# gateway workload's `supervisor.image` field. Keep that runtime image aligned +# with the locally built/imported image (and with CI's registry/tag overrides). +helm_extra_args+=(--set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 diff --git a/tasks/scripts/setup-zig-cc-wrapper.sh b/tasks/scripts/setup-zig-cc-wrapper.sh index c21e0a3487..9a80354697 100755 --- a/tasks/scripts/setup-zig-cc-wrapper.sh +++ b/tasks/scripts/setup-zig-cc-wrapper.sh @@ -23,8 +23,9 @@ if [[ $bare_cargo_target =~ ^(.+)\.[0-9]+\.[0-9]+$ ]]; then fi # cargo-zigbuild accepts Rust target triples with glibc suffixes, for example -# x86_64-unknown-linux-gnu.2.28. Zig's C/C++ driver expects the vendorless form. -zig_cc_target=${zig_target/-unknown-linux-/-linux-} +# x86_64-unknown-linux-gnu.2.28. Zig's C/C++ driver expects a vendorless +# target without that Rust-only suffix. +zig_cc_target=${bare_cargo_target/-unknown-linux-/-linux-} if [[ -n ${ZIG:-} ]]; then zig=$ZIG From 9b1199904dd5419d6ef5b6d5eeb13dcfcd3bc880 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 10:00:06 +0100 Subject: [PATCH 18/29] fix(e2e): pass host aliases through gateway config Signed-off-by: Gaizka Menendez Hernandez --- e2e/rust/e2e-kubernetes.sh | 8 ++++---- e2e/with-kube-gateway.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index e49c7a3b7b..5c4d7d6c2b 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -10,10 +10,10 @@ # # Features: the default set includes `e2e-host-gateway` so tests that rely on # the sandbox-side `host.openshell.internal` alias compile and run. The -# wrapper detects the cluster's host-routable IP and wires it into the chart -# via `server.hostGatewayIP`. Targeting a cluster where the test host is -# unreachable from pods? Set OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the -# alias-dependent tests entirely. +# wrapper detects the cluster's host-routable IP and wires it into +# `gatewayConfig.openshell.drivers.kubernetes.host_gateway_ip`. Targeting a +# cluster where the test host is unreachable from pods? Set +# OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the alias-dependent tests. # # Results: `run_suite` writes a JUnit + HTML report under `results/`. Set # `OPENSHELL_E2E_REPORT_NAME` to name it per run when invoking this script repeatedly. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 005166e8f0..b024f57b74 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1233,7 +1233,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then ) fi if [ -n "${HOST_GATEWAY_IP}" ]; then - helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") + helm_extra_args+=(--set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.host_gateway_ip=${HOST_GATEWAY_IP}") fi helm_values_args=(--values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml") From 6ce2170db992baf9c303ef4dfd920e01f896ae5c Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 10:30:38 +0100 Subject: [PATCH 19/29] docs(e2e): reference gateway config host alias Signed-off-by: Gaizka Menendez Hernandez --- e2e/rust/Cargo.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 4d0522ea86..a32f4766da 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -20,9 +20,10 @@ publish = false e2e = [] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the -# alias unconditionally; the kube driver only does so when the chart's -# `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and -# the helm wrapper opts in explicitly when it has resolved an IP. +# alias unconditionally; the kube driver only does so when +# `gatewayConfig.openshell.drivers.kubernetes.host_gateway_ip` is set, so +# `e2e-kubernetes` does NOT imply this and the Helm wrapper opts in explicitly +# when it has resolved an IP. e2e-host-gateway = ["e2e"] e2e-local-container-driver = ["e2e"] e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] From 03f5abea3157148b7eb8285e67b6fe860c91e8e3 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 10:44:30 +0100 Subject: [PATCH 20/29] fix(helm): enforce gateway resource ownership Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/README.md | 34 +++++++++++++++-- deploy/helm/openshell/README.md.gotmpl | 8 ++++ .../openshell/templates/_gateway-workload.tpl | 4 +- deploy/helm/openshell/templates/_toml.tpl | 37 +++++++++++++++++-- .../gateway_pod_security_context_test.yaml | 15 ++++++++ docs/kubernetes/migrate-gateway-config.mdx | 15 ++++++-- skills/debug-openshell-cluster/SKILL.md | 12 +++--- 7 files changed, 107 insertions(+), 18 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index df3ea173ad..a77bcb456f 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -239,9 +239,37 @@ database. The chart creates a retained Kubernetes Secret with the shared key-encryption key and injects that key into every gateway pod, so the same default works for single-replica and external database-backed HA deployments. -Use `kubernetes-secrets` or `vault` instead when credentials should live in a -cluster or external secret backend. Enabling one external credential driver -disables the default credential-storage key-encryption key Secret and env injection. +Use `gatewayConfig` to select `kubernetes-secrets` or `vault` when credentials +should live in a cluster or external secret backend. Selecting an external +credential driver disables the default credential-storage key-encryption-key +Secret and environment injection. The map is rendered directly as gateway TOML, +so it uses the gateway's snake_case field names: + +```yaml +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: http://vault.vault.svc.cluster.local:8200 + mount: secret + kv_version: "2" + auth_method: kubernetes + role: openshell-gateway +``` + +> `gatewayConfig` must contain only non-secret values. Helm serializes unknown +> fields generically and cannot determine whether an arbitrary string, such as +> `api_token`, is confidential. Do not put passwords, tokens, private keys, +> database URLs, or other secret material in this map. Use Secret-backed +> environment variables, files, volumes, or gateway credential drivers instead. +> The chart rejects known unsafe forms such as `database_url`, inline URL +> credentials, and PEM private keys; it is not a general secret scanner. + +For the Kubernetes Secret driver, use +`openshell.credential_drivers.kubernetes-secrets.namespace` in the same map. +The chart derives any required RBAC from the selected driver; use a dedicated +namespace to limit access to OpenShell-managed Secrets. #### OpenShift diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index ce94283d41..127c824c9b 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -259,6 +259,14 @@ gatewayConfig: role: openshell-gateway ``` +> `gatewayConfig` must contain only non-secret values. Helm serializes unknown +> fields generically and cannot determine whether an arbitrary string, such as +> `api_token`, is confidential. Do not put passwords, tokens, private keys, +> database URLs, or other secret material in this map. Use Secret-backed +> environment variables, files, volumes, or gateway credential drivers instead. +> The chart rejects known unsafe forms such as `database_url`, inline URL +> credentials, and PEM private keys; it is not a general secret scanner. + For the Kubernetes Secret driver, use `openshell.credential_drivers.kubernetes-secrets.namespace` in the same map. The chart derives any required RBAC from the selected driver; use a dedicated diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 019455f46a..53dc678dd1 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -32,9 +32,9 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} serviceAccountName: {{ include "openshell.serviceAccountName" . }} - {{- if get $kubernetesRuntimeConfig "host_gateway_ip" }} + {{- if .Values.server.hostGatewayIP }} hostAliases: - - ip: {{ get $kubernetesRuntimeConfig "host_gateway_ip" | quote }} + - ip: {{ .Values.server.hostGatewayIP | quote }} hostnames: - host.docker.internal - host.openshell.internal diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 3049ab03a7..42076dafa7 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -99,20 +99,49 @@ field must not require a Helm template change. {{- define "openshell.gatewayConfigToml" -}} {{- $root := . -}} {{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} -{{- if .Values.server.disableTls -}} +{{/* Kubernetes packaging owns host aliases. Do not permit a second runtime +source to make sandbox callback hostnames disagree with the pod spec. */}} +{{- $kubernetes := get $config "openshell.drivers.kubernetes" | default dict -}} +{{- if .Values.server.hostGatewayIP -}} +{{- $_ := set $kubernetes "host_gateway_ip" .Values.server.hostGatewayIP -}} +{{- else -}} +{{- $_ := unset $kubernetes "host_gateway_ip" -}} +{{- end -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} + +{{/* TLS resources, mounts, and their corresponding runtime fields have one +owner: server.*. Override any gatewayConfig copies before serializing TOML. */}} {{- $gateway := get $config "openshell.gateway" | default dict -}} -{{- $_ := set $gateway "disable_tls" true -}} +{{- $_ := set $gateway "disable_tls" .Values.server.disableTls -}} {{- $_ := set $config "openshell.gateway" $gateway -}} +{{- if .Values.server.disableTls -}} {{- $_ := unset $config "openshell.gateway.tls" -}} -{{- $kubernetes := get $config "openshell.drivers.kubernetes" | default dict -}} {{- $_ := unset $kubernetes "client_tls_secret_name" -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} +{{- else -}} +{{- if .Values.server.tls.clientTlsSecretName -}} +{{- $_ := set $kubernetes "client_tls_secret_name" .Values.server.tls.clientTlsSecretName -}} +{{- else -}} +{{- $_ := unset $kubernetes "client_tls_secret_name" -}} {{- end -}} -{{- if and (not .Values.server.disableTls) .Values.certManager.serverIssuerRef.name -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} {{- $gatewayTls := get $config "openshell.gateway.tls" | default dict -}} +{{- $_ := set $gatewayTls "cert_path" "/etc/openshell-tls/server/tls.crt" -}} +{{- $_ := set $gatewayTls "key_path" "/etc/openshell-tls/server/tls.key" -}} +{{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" -}} +{{- $_ := set $gatewayTls "client_ca_path" "/etc/openshell-tls/client-ca/ca.crt" -}} +{{- else -}} +{{- $_ := unset $gatewayTls "client_ca_path" -}} +{{- end -}} +{{- if .Values.certManager.serverIssuerRef.name -}} {{- $_ := set $gatewayTls "external_cert_path" "/etc/openshell-tls/server-external/tls.crt" -}} {{- $_ := set $gatewayTls "external_key_path" "/etc/openshell-tls/server-external/tls.key" -}} {{- $_ := set $gatewayTls "external_server_names" (deepCopy (.Values.certManager.serverDnsNames | default list)) -}} +{{- else -}} +{{- $_ := unset $gatewayTls "external_cert_path" -}} +{{- $_ := unset $gatewayTls "external_key_path" -}} +{{- $_ := unset $gatewayTls "external_server_names" -}} +{{- end -}} {{- $_ := set $config "openshell.gateway.tls" $gatewayTls -}} {{- end -}} {{- range $tableName := keys $config | sortAlpha -}} diff --git a/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml b/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml index cdb70a05eb..6c7c45bb28 100644 --- a/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml +++ b/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml @@ -34,3 +34,18 @@ tests: asserts: - notExists: path: spec.template.spec.securityContext + + - it: derives host aliases from server.hostGatewayIP and not gatewayConfig + template: templates/statefulset.yaml + set: + server.hostGatewayIP: 10.23.45.67 + gatewayConfig: + openshell.drivers.kubernetes: + host_gateway_ip: 192.0.2.1 + asserts: + - equal: + path: spec.template.spec.hostAliases[0].ip + value: 10.23.45.67 + - contains: + path: spec.template.spec.hostAliases[0].hostnames + content: host.openshell.internal diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx index db6b303ea3..afe7d8f410 100644 --- a/docs/kubernetes/migrate-gateway-config.mdx +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -10,10 +10,15 @@ and mounts) remain chart values. Remove every application-only `server.*`, `supervisor.*`, and `upstreamProxy.*` setting before upgrading. There is no compatibility translation. Kubernetes pull policy spellings such as `Always` are not aliases: use `always`, -`if_not_present`, or `never`. Do not put passwords, private keys, or -`database_url` in `gatewayConfig`; use Secret-backed environment configuration. +`if_not_present`, or `never`. `gatewayConfig` is strictly a non-secret +configuration contract: do not put passwords, tokens, private keys, +`database_url`, or other secret material in it; use Secret-backed environment, +file, or volume configuration instead. Helm serializes unknown fields +generically and cannot determine whether an arbitrary string such as +`api_token` is confidential. It rejects `database_url`, inline URL credentials, +and PEM private keys, but is not a general secret scanner. -Keep `server.disableTls`, TLS Secret names, `server.externalDbSecret`, +Keep `server.disableTls`, TLS Secret names, `server.hostGatewayIP`, `server.externalDbSecret`, `server.sandboxNamespace`, `oidc.caConfigMapName`, certificate settings and Service settings: these own Kubernetes resources. The chart derives their runtime counterparts. @@ -32,7 +37,9 @@ runtime counterparts. | `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.gateway.gateway_jwt` or `openshell.drivers.kubernetes.sa_token_ttl_secs` | | `supervisor.*` | `openshell.drivers.kubernetes` and `.sidecar` | | `upstreamProxy.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}` | -| credential-driver runtime settings | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | +| `server.credentialDrivers.{kubernetesSecrets,vault}.*` | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | +| `server.providerTokenGrants.spiffe.{enabled,workloadApiSocketPath}` | `openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path`; omit it to disable SPIFFE provider grants. | +| `server.hostGatewayIP` | Retained as the chart-owned host-alias input; the chart derives `openshell.drivers.kubernetes.host_gateway_ip`. | ## Examples diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 6382428946..a890470743 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -444,8 +444,9 @@ retained Kubernetes Secret for the shared KEK, injects it into gateway pods, and stores encrypted credential envelopes in the OpenShell database. For `workload.kind=deployment` or multi-replica gateways, confirm `server.externalDbSecret` points at a shared database. A render/install error -mentioning `server.credentialDrivers` means the values selected multiple -external credential backends. +mentioning multiple credential drivers means the +`gatewayConfig.openshell.gateway.credential_drivers` list selected more than +one external credential backend. For HA or PostgreSQL-backed installs, also check the external database Secret referenced by `server.externalDbSecret` and the PostgreSQL workload when it is @@ -583,8 +584,9 @@ kubectl -n openshell get statefulset openshell -o jsonpath='{.spec.template.spec # Should show items filter for ca.crt from openshell-server-tls ``` -If `server.providerTokenGrants.spiffe.enabled=true`, the gateway should still -render `[openshell.gateway.gateway_jwt]` and mount the `sandbox-jwt` Secret. +If `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` +is set, the gateway should still render `[openshell.gateway.gateway_jwt]` and +mount the `sandbox-jwt` Secret. SPIRE is used by both the gateway and sandbox supervisors for dynamic provider token grants. The gateway pod must mount the `spiffe-workload-api` CSI volume and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; supervisor Pods must @@ -595,7 +597,7 @@ Verify that SPIRE is installed, the CSI driver is available, and the Kubernetes driver config includes `provider_spiffe_workload_api_socket_path`: ```bash -helm -n openshell get values openshell | grep -E 'providerTokenGrants|workloadApiSocketPath' +helm -n openshell get values openshell | grep provider_spiffe_workload_api_socket_path kubectl get pods -A | grep -E 'spire|spiffe' kubectl -n openshell get configmap openshell-config -o yaml | grep provider_spiffe_workload_api_socket_path kubectl -n openshell get pod -l app.kubernetes.io/name=helm-chart -o jsonpath="{.items[*].spec.containers[*].env[?(@.name==\"OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET\")].value}{\"\n\"}" From e63d4a0127187c08e6a7761256a48a1f13f17f92 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 10:53:51 +0100 Subject: [PATCH 21/29] test(e2e): cover chart host gateway input Signed-off-by: Gaizka Menendez Hernandez --- e2e/with-kube-gateway.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index b024f57b74..139e5ad09d 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1233,7 +1233,10 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then ) fi if [ -n "${HOST_GATEWAY_IP}" ]; then - helm_extra_args+=(--set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.host_gateway_ip=${HOST_GATEWAY_IP}") + # server.hostGatewayIP owns both the gateway runtime field and sandbox-pod + # hostAliases. Exercise the public chart input rather than bypassing it with + # a direct gatewayConfig override. + helm_extra_args+=(--set-string "server.hostGatewayIP=${HOST_GATEWAY_IP}") fi helm_values_args=(--values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml") From d8b9aa8483833cd21e96df0c52f4623f9d686adb Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 11:40:13 +0100 Subject: [PATCH 22/29] fix(helm): address gateway config review findings Signed-off-by: Gaizka Menendez Hernandez --- .../openshell/ci/values-corporate-proxy-e2e.yaml | 14 ++++++++------ deploy/helm/openshell/templates/_helpers.tpl | 3 +++ deploy/helm/openshell/templates/_toml.tpl | 2 +- .../tests/gateway_secret_boundary_test.yaml | 16 ++++++++++++++++ deploy/helm/test-gateway-config-parser.sh | 9 +-------- docs/kubernetes/access-control.mdx | 16 ++++------------ docs/kubernetes/ingress.mdx | 2 ++ e2e/with-kube-gateway.sh | 2 +- 8 files changed, 36 insertions(+), 28 deletions(-) diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index d9803da1ca..c015775efd 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,9 +3,11 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -gatewayConfig: - openshell.drivers.kubernetes: - topology: sidecar - proxy_auth_secret_name: openshell-e2e-proxy-auth - proxy_auth_secret_key: proxy-auth - proxy_auth_allow_insecure: true +upstreamProxy: + # The e2e wrapper replaces this endpoint with its dynamically allocated port. + # Keep the overlay valid when rendered independently as well. + url: http://host.openshell.internal:8080 + authSecret: + name: openshell-e2e-proxy-auth + key: proxy-auth + authAllowInsecure: true diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 9d5180e7d4..33120137c7 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -401,6 +401,9 @@ passes through Helm values into gateway.toml; only this reference is rendered. {{- if not (kindIs "string" $name) -}} {{- fail (printf "%s must be a Kubernetes Secret name, got %s" $path (kindOf $name)) -}} {{- end -}} +{{- if gt (len $name) 253 -}} +{{- fail (printf "%s must be no more than 253 characters" $path) -}} +{{- end -}} {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" $name) -}} {{- fail (printf "%s must be a valid Kubernetes Secret name" $path) -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 42076dafa7..0210bd3fa3 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -26,7 +26,7 @@ field must not require a Helm template change. {{- if regexMatch "-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----" $rendered -}} {{- fail "gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount" -}} {{- end -}} -{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]+:[^/@[:space:]]+@" $rendered -}} +{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]*:[^/@[:space:]]+@" $rendered -}} {{- fail "gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume" -}} {{- end -}} {{- $rendered | quote -}} diff --git a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml index 0b7c97f30c..2afb9918b1 100644 --- a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml +++ b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml @@ -31,6 +31,14 @@ tests: - failedTemplate: errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + - it: rejects URL credentials with an empty username + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://:password@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + - it: keeps external database credentials out of gateway.toml template: templates/gateway-config.yaml set: @@ -61,3 +69,11 @@ tests: asserts: - failedTemplate: errorMessage: server.credentialStorage.existingSecret must be a valid Kubernetes Secret name + + - it: rejects Secret references longer than the Kubernetes limit + template: templates/statefulset.yaml + set: + server.credentialStorage.existingSecret: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - failedTemplate: + errorMessage: server.credentialStorage.existingSecret must be no more than 253 characters diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 538db9cc46..1a9227ca1c 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -106,14 +106,7 @@ fi # All maintained CI/dev overlays must render a loader-valid configuration. for values in "${chart}"/ci/values-*.yaml; do name="$(basename "${values}" .yaml)" - if [[ "${name}" == "values-corporate-proxy-e2e" ]]; then - # The e2e wrapper supplies this generated URL alongside the overlay. - render "${work_dir}/${name}.yaml" \ - --values "${values}" \ - --set-string 'gatewayConfig.openshell\.drivers\.kubernetes.https_proxy=http://proxy.corp.example:8080' - else - render "${work_dir}/${name}.yaml" --values "${values}" - fi + render "${work_dir}/${name}.yaml" --values "${values}" extract_toml "${work_dir}/${name}.yaml" "${work_dir}/${name}.toml" preflight "${work_dir}/${name}.toml" done diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 99e49d912d..58bba7ea5c 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -113,7 +113,7 @@ OIDC RBAC is method-level authorization. It controls which API operations a call ## Reverse-Proxy Auth Termination -When an access proxy, such as Cloudflare Access, ngrok, or a corporate SSO gateway, handles authentication in front of the OpenShell gateway, you can explicitly allow unauthenticated user calls at the gateway: +When an access proxy, such as Cloudflare Access, ngrok, or a corporate SSO gateway, terminates TLS and handles authentication in front of the OpenShell gateway, configure the gateway for plaintext traffic from that trusted proxy and explicitly allow unauthenticated user calls: ```shell helm upgrade openshell \ @@ -126,22 +126,14 @@ helm upgrade openshell \ Create `reverse-proxy-values.yaml`: ```yaml +server: + disableTls: true gatewayConfig: openshell.gateway.auth: allow_unauthenticated_users: true ``` -The gateway still serves TLS and sandbox supervisors still authenticate with gateway-minted sandbox JWTs. User-facing CLI/API calls without OIDC or mTLS credentials are accepted as an unauthenticated local developer principal. The proxy is responsible for authenticating callers and forwarding only authorized traffic. - -When the gateway terminates TLS directly and callers connect without client certificates, also set `server.tls.clientCaSecretName=""` as described in the OIDC section above. - -To also disable TLS entirely (when the proxy terminates TLS before the request reaches the gateway): - -```yaml -gatewayConfig: - openshell.gateway.auth: - allow_unauthenticated_users: true -``` +The gateway listens on plaintext traffic from the proxy, while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. User-facing CLI/API calls without OIDC or mTLS credentials are accepted as an unauthenticated local developer principal. The proxy is responsible for authenticating callers and forwarding only authorized traffic. Only enable unauthenticated users when the gateway is not reachable from outside a trusted local development environment or the proxy path is fully trusted. Never expose a plaintext, auth-disabled gateway to a public network. diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 192a3cfb50..54ec3e07eb 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -109,6 +109,8 @@ The Secret may also be issued by cert-manager, or you can reference the chart's Enable an HTTPS listener, point it at the Secret, configure the gateway for a plaintext backend, and provide OIDC settings in `oidc-values.yaml`: ```yaml +server: + disableTls: true gatewayConfig: openshell.gateway.oidc: issuer: https://keycloak.example.com/realms/openshell diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 139e5ad09d..20be9c48ca 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -192,7 +192,7 @@ verify_gateway_config_rollout() { echo "ERROR: ConfigMap-only gateway configuration change did not update workload checksum" >&2 return 1 fi - kctl -n "${NAMESPACE}" rollout status "${workload_ref}" --timeout=5m + kctl -n "${NAMESPACE}" rollout status "${workload_ref}" --timeout=5m || return 1 for attempt in $(seq 1 60); do new_pod_uid="$(kctl -n "${NAMESPACE}" get pods -l "${pod_selector}" -o jsonpath='{.items[0].metadata.uid}')" From d7ac0fa552dab7dd0c6512ab22333ea35bd7f1c5 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 11:52:48 +0100 Subject: [PATCH 23/29] fix(helm): repair RFC 0012 migration integration Signed-off-by: Gaizka Menendez Hernandez --- .../ci/values-credential-driver-vault.yaml | 13 +-- .../openshell/ci/values-openshift-e2e.yaml | 11 ++- .../openshell/templates/_gateway-workload.tpl | 99 +++---------------- deploy/helm/openshell/templates/_toml.tpl | 44 ++++++++- .../tests/gateway_secret_boundary_test.yaml | 8 ++ deploy/helm/openshell/values.yaml | 8 ++ deploy/helm/test-gateway-config-parser.sh | 8 ++ .../helm/test-gateway-resource-coherence.sh | 3 +- docs/how-it-works/gateways/configuration.mdx | 4 +- docs/kubernetes/migrate-gateway-config.mdx | 17 ++-- tasks/test.toml | 8 +- 11 files changed, 110 insertions(+), 113 deletions(-) diff --git a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml index 9380dea613..83a891dd2a 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml @@ -8,16 +8,17 @@ # # The profile assumes another process has already deployed a Vault-compatible # backend. Local e2e validation deploys OpenBao in the `openbao` namespace with -# TLS enabled, publishes its private CA in the `openbao-ca` ConfigMap, and -# creates an `openbao-0` DNS alias matching the OpenBao dev certificate. It also -# configures a Kubernetes auth role named `openshell-gateway` bound to the -# OpenShell gateway ServiceAccount in the `openshell` namespace. +# a Kubernetes auth role named `openshell-gateway` bound to the OpenShell +# gateway ServiceAccount in the `openshell` namespace. gatewayConfig: openshell.gateway: credential_drivers: - vault openshell.credential_drivers.vault: - address: https://openbao-0:8200 - ca_bundle: /etc/openshell-tls/vault-ca/ca.crt + address: https://openbao.openbao.svc.cluster.local:8200 role: openshell-gateway + +credentialDrivers: + vault: + caConfigMapName: openbao-ca diff --git a/deploy/helm/openshell/ci/values-openshift-e2e.yaml b/deploy/helm/openshell/ci/values-openshift-e2e.yaml index 2602de3c9d..bbe68c3e1c 100644 --- a/deploy/helm/openshell/ci/values-openshift-e2e.yaml +++ b/deploy/helm/openshell/ci/values-openshift-e2e.yaml @@ -36,12 +36,17 @@ server: disableTls: false gatewayConfig: - openshell.drivers.kubernetes: - supervisor_image_pull_policy: always - image_pull_policy: always openshell.gateway.auth: allow_unauthenticated_users: true +sandboxRuntime: + image: + pullPolicy: Always + +supervisor: + image: + pullPolicy: Always + openshiftRoute: enabled: true # host is supplied via --set at install time (cluster-derived Route hostname). diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 53dc678dd1..97569470b8 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -9,7 +9,12 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. {{- $gatewayRuntimeConfig := get $gatewayConfig "openshell.gateway" | default dict -}} {{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} {{- $kubernetesRuntimeConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $spiffeSocketPath := get $kubernetesRuntimeConfig "provider_spiffe_workload_api_socket_path" -}} {{- $hasExternalCredentialDriver := or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") -}} +{{- $vaultCredentialDriverEnabled := eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true" -}} +{{- $credentialDrivers := .Values.credentialDrivers | default dict -}} +{{- $vaultResources := get $credentialDrivers "vault" | default dict -}} +{{- $vaultCaConfigMapName := get $vaultResources "caConfigMapName" -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -57,50 +62,6 @@ spec: - {{ .Values.server.dbUrl | quote }} {{- end }} env: - - name: OPENSHELL_REPLICA_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPENSHELL_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPENSHELL_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - {{- if eq (include "openshell.workloadKind" .) "deployment" }} - - name: OPENSHELL_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: OPENSHELL_PEER_ENDPOINT - value: {{ printf "%s://$(OPENSHELL_POD_IP):%d" (ternary "http" "https" (default false .Values.server.disableTls)) (int .Values.service.port) | quote }} - {{- end }} - - name: OPENSHELL_SERVICE_ACCOUNT_NAME - value: {{ include "openshell.serviceAccountName" . | quote }} - - name: OPENSHELL_PEER_SERVICE_NAME - value: {{ include "openshell.peerServiceName" . | quote }} - - name: OPENSHELL_PEER_TOKEN_AUDIENCE - value: "openshell-gateway-peer" - - name: OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE - value: /var/run/secrets/openshell-peer/token - - name: OPENSHELL_PEER_POD_LABELS - value: {{ printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "openshell.name" .) .Release.Name | quote }} - {{- if not .Values.server.disableTls }} - - name: OPENSHELL_PEER_TLS_SERVER_NAME - value: {{ printf "%s.%s.svc.cluster.local" (include "openshell.fullname" .) .Release.Namespace | quote }} - {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} - - name: OPENSHELL_PEER_TLS_CA_FILE - value: /etc/openshell-tls/server/ca.crt - {{- end }} - {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} - - name: OPENSHELL_PEER_TLS_CERT_FILE - value: /etc/openshell-tls/peer-client/tls.crt - - name: OPENSHELL_PEER_TLS_KEY_FILE - value: /etc/openshell-tls/peer-client/tls.key - {{- end }} - {{- end }} {{- if not $hasExternalCredentialDriver }} - name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }} valueFrom: @@ -129,9 +90,9 @@ spec: {{- end }} - name: OPENSHELL_TELEMETRY_ENABLED value: {{ .Values.server.telemetryEnabled | quote }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET - value: {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} + value: {{ $spiffeSocketPath | quote }} {{- end }} volumeMounts: {{- if eq (include "openshell.workloadKind" .) "statefulset" }} @@ -148,9 +109,6 @@ spec: - name: sandbox-jwt mountPath: /etc/openshell-jwt readOnly: true - - name: gateway-peer-token - mountPath: /var/run/secrets/openshell-peer - readOnly: true {{- if not .Values.server.disableTls }} - name: tls-cert mountPath: /etc/openshell-tls/server @@ -161,9 +119,6 @@ spec: readOnly: true {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} - - name: peer-client-tls - mountPath: /etc/openshell-tls/peer-client - readOnly: true - name: tls-client-ca mountPath: /etc/openshell-tls/client-ca readOnly: true @@ -174,19 +129,14 @@ spec: mountPath: /etc/openshell-tls/oidc-ca readOnly: true {{- end }} - {{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }} + {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca - mountPath: /etc/openshell-tls/vault-ca + mountPath: /etc/openshell-tls/vault readOnly: true {{- end }} - {{- if .Values.upstreamProxy.caBundle.configMapName }} - - name: upstream-proxy-ca - mountPath: /etc/openshell-tls/proxy-ca - readOnly: true - {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: spiffe-workload-api - mountPath: {{ dir .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} + mountPath: {{ dir $spiffeSocketPath | quote }} readOnly: true {{- end }} {{- with .Values.server.extraVolumeMounts }} @@ -237,14 +187,6 @@ spec: secret: secretName: {{ include "openshell.sandboxJwtSecretName" . }} defaultMode: {{ .Values.server.sandboxJwt.secretDefaultMode | default 0400 }} - - name: gateway-peer-token - projected: - defaultMode: 0400 - sources: - - serviceAccountToken: - path: token - audience: openshell-gateway-peer - expirationSeconds: 3600 {{- if not .Values.server.disableTls }} - name: tls-cert secret: @@ -255,9 +197,6 @@ spec: secretName: {{ include "openshell.fullname" . }}-server-external-tls {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} - - name: peer-client-tls - secret: - secretName: {{ .Values.server.tls.clientTlsSecretName }} - name: tls-client-ca secret: {{- if or (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }} @@ -275,25 +214,15 @@ spec: configMap: name: {{ .Values.oidc.caConfigMapName }} {{- end }} - {{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }} + {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca configMap: - name: {{ .Values.server.credentialDrivers.vault.caConfigMapName }} + name: {{ $vaultCaConfigMapName }} items: - key: ca.crt path: ca.crt {{- end }} - {{- if .Values.upstreamProxy.caBundle.configMapName }} - - name: upstream-proxy-ca - configMap: - name: {{ .Values.upstreamProxy.caBundle.configMapName | quote }} - items: - # The mounted filename stays fixed so the rendered proxy_ca_bundle - # path does not depend on the operator's ConfigMap key. - - key: {{ .Values.upstreamProxy.caBundle.key | default "ca.crt" | quote }} - path: ca.crt - {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: spiffe-workload-api csi: driver: csi.spiffe.io diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 0210bd3fa3..da3e5aecee 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -26,7 +26,7 @@ field must not require a Helm template change. {{- if regexMatch "-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----" $rendered -}} {{- fail "gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount" -}} {{- end -}} -{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]*:[^/@[:space:]]+@" $rendered -}} +{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]*@" $rendered -}} {{- fail "gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume" -}} {{- end -}} {{- $rendered | quote -}} @@ -109,6 +109,48 @@ source to make sandbox callback hostnames disagree with the pod spec. */}} {{- end -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} +{{/* RFC 0012 packaging inputs are authoritative for paired runtime images, +the network-fence acknowledgement, and corporate proxy Secret wiring. */}} +{{- $runtime := .Values.sandboxRuntime | default dict -}} +{{- $runtimeImage := get $runtime "image" | default dict -}} +{{- $supervisor := .Values.supervisor | default dict -}} +{{- $supervisorImage := get $supervisor "image" | default dict -}} +{{- $runtimeTag := get $runtimeImage "tag" | default .Values.image.tag | default .Chart.AppVersion -}} +{{- $supervisorTag := get $supervisorImage "tag" | default .Values.image.tag | default .Chart.AppVersion -}} +{{- $_ := set $kubernetes "sandbox_runtime_image" (printf "%s:%s" (get $runtimeImage "repository" | default "ghcr.io/nvidia/openshell/sandbox") $runtimeTag) -}} +{{- $_ := set $kubernetes "supervisor_image" (printf "%s:%s" (get $supervisorImage "repository" | default "ghcr.io/nvidia/openshell/supervisor") $supervisorTag) -}} +{{- if get $runtimeImage "pullPolicy" -}} +{{- $_ := set $kubernetes "sandbox_runtime_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $runtimeImage "pullPolicy")) -}} +{{- else -}}{{- $_ := unset $kubernetes "sandbox_runtime_image_pull_policy" -}}{{- end -}} +{{- if get $supervisorImage "pullPolicy" -}} +{{- $_ := set $kubernetes "supervisor_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $supervisorImage "pullPolicy")) -}} +{{- else -}}{{- $_ := unset $kubernetes "supervisor_image_pull_policy" -}}{{- end -}} +{{- $runtimeConfig := get $supervisor "sandboxRuntime" | default dict -}} +{{- $_ := set $kubernetes "sandbox_runtime" (dict "network_policy_enforced" (get $runtimeConfig "networkPolicyEnforced") "boundary_port" (get $runtimeConfig "boundaryPort" | default 5500)) -}} +{{- $proxy := .Values.upstreamProxy | default dict -}} +{{- range $runtimeKey := list "https_proxy" "no_proxy" "proxy_auth_secret_name" "proxy_auth_secret_key" "proxy_auth_allow_insecure" "proxy_connect_by_hostname" -}}{{- $_ := unset $kubernetes $runtimeKey -}}{{- end -}} +{{- if get $proxy "url" -}}{{- $_ := set $kubernetes "https_proxy" (get $proxy "url") -}}{{- end -}} +{{- if get $proxy "noProxy" -}}{{- $_ := set $kubernetes "no_proxy" (get $proxy "noProxy") -}}{{- end -}} +{{- $proxySecret := get $proxy "authSecret" | default dict -}} +{{- if get $proxySecret "name" -}}{{- $_ := set $kubernetes "proxy_auth_secret_name" (get $proxySecret "name") -}}{{- end -}} +{{- if get $proxySecret "key" -}}{{- $_ := set $kubernetes "proxy_auth_secret_key" (get $proxySecret "key") -}}{{- end -}} +{{- if or (get $proxySecret "name") (get $proxySecret "key") -}}{{- $_ := set $kubernetes "proxy_auth_allow_insecure" (get $proxy "authAllowInsecure") -}}{{- end -}} +{{- if get $proxy "connectByHostname" -}}{{- $_ := set $kubernetes "proxy_connect_by_hostname" true -}}{{- end -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} + +{{/* A Vault CA is a Kubernetes resource reference, not a free-form runtime +path. Derive its mounted path only from the chart-owned ConfigMap reference. */}} +{{- $credentialDrivers := .Values.credentialDrivers | default dict -}} +{{- $vaultResources := get $credentialDrivers "vault" | default dict -}} +{{- if hasKey $config "openshell.credential_drivers.vault" -}} +{{- $vaultConfig := get $config "openshell.credential_drivers.vault" | default dict -}} +{{- $_ := unset $vaultConfig "ca_bundle" -}} +{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") (get $vaultResources "caConfigMapName") -}} +{{- $_ := set $vaultConfig "ca_bundle" "/etc/openshell-tls/vault/ca.crt" -}} +{{- end -}} +{{- $_ := set $config "openshell.credential_drivers.vault" $vaultConfig -}} +{{- end -}} + {{/* TLS resources, mounts, and their corresponding runtime fields have one owner: server.*. Override any gatewayConfig copies before serializing TOML. */}} {{- $gateway := get $config "openshell.gateway" | default dict -}} diff --git a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml index 2afb9918b1..71d5ea1810 100644 --- a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml +++ b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml @@ -39,6 +39,14 @@ tests: - failedTemplate: errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + - it: rejects URL userinfo without a password + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://username@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + - it: keeps external database credentials out of gateway.toml template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index cb45e12c0c..1c1b484976 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -583,6 +583,14 @@ server: # issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). caConfigMapName: "" +# Kubernetes resource references for external credential drivers. Runtime +# driver settings remain in gatewayConfig. +credentialDrivers: + vault: + # -- ConfigMap containing the private Vault/OpenBao CA certificate under + # the ca.crt key. Helm mounts it only when the Vault driver is selected. + caConfigMapName: "" + # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. networkPolicy: # -- Restrict SSH ingress on sandbox pods to the gateway. In managed mode, diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 1a9227ca1c..93b25ef3c8 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -38,6 +38,14 @@ preflight() { "${gateway_bin}" config preflight --path "${toml}" } +if helm template parser-validation "${chart}" --namespace parser-namespace \ + --set agentSandbox.preflight.enabled=false >"${work_dir}/unacknowledged-default.yaml" 2>"${work_dir}/unacknowledged-default.err"; then + echo "the chart must require explicit NetworkPolicy enforcement acknowledgement" >&2 + exit 1 +fi +grep -F 'supervisor.sandboxRuntime.networkPolicyEnforced must be true' "${work_dir}/unacknowledged-default.err" >/dev/null + +# The runtime default is valid after the required infrastructure acknowledgement. render "${work_dir}/default.yaml" extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" preflight "${work_dir}/default.toml" diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index 10301244b6..2f5a603935 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -16,10 +16,11 @@ render() { helm template resource-coherence "${chart}" \ --namespace resource-namespace \ --set agentSandbox.preflight.enabled=false \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ "$@" >"${work_dir}/${name}.yaml" if ! awk 'BEGIN { RS="---" } - /^\n?# Source:/ && $0 !~ /\napiVersion:/ { + /^\n?# Source:/ && $0 !~ /\napiVersion:/ && $0 !~ /network-policy-ack\.yaml/ { print "rendered an empty or invalid Kubernetes document:" $0 > "/dev/stderr" exit 1 }' "${work_dir}/${name}.yaml"; then diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index f81e3d34a8..9c6e1dc976 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -527,7 +527,7 @@ credential_drivers = ["vault"] [openshell.credential_drivers.vault] address = "https://vault.vault.svc.cluster.local:8200" -ca_bundle = "/etc/openshell/vault/ca.pem" +ca_bundle = "/etc/openshell-tls/vault/ca.crt" mount = "secret" kv_version = "2" auth_method = "kubernetes" @@ -549,7 +549,7 @@ For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secr For `vault`, `address` points at the Vault service. Non-loopback endpoints must use HTTPS; plaintext HTTP is accepted only for `localhost` or an IP loopback address during local development. Vault requests do not follow redirects, preventing credentials from being replayed to a downgraded or substituted endpoint. HTTPS uses platform trust roots by default. Set `ca_bundle` to a certificate-only PEM bundle for a private Vault CA; the bundle augments platform roots and normal hostname verification remains enabled. `mount` and `kv_version` describe the KV engine where OpenShell-managed provider secrets are stored, and `auth_method = "kubernetes"` logs in with the gateway Pod's ServiceAccount token. For local or development validation, use `auth_method = "token_file"` with `token_path = "/path/to/token"`. Do not put literal Vault tokens in TOML. -For Helm deployments, set `server.credentialDrivers.vault.caConfigMapName` to a ConfigMap containing the private CA bundle under the `ca.crt` key. The chart mounts that key and renders `ca_bundle` automatically. +For Helm deployments, set `credentialDrivers.vault.caConfigMapName` to a ConfigMap containing the private CA bundle under the `ca.crt` key. The chart mounts that key and derives `ca_bundle` automatically. Provider records that already contain inline database credentials remain readable for upgrade compatibility. New provider create/update requests still submit credential values through the normal API, but the gateway stores those values through the active credential storage path and persists only handles. Before OpenShell 0.1.0, OpenShell does not automatically migrate inline refresh material or credential handles between drivers. Reconfigure refresh grants after an upgrade. Before changing credential drivers, remove affected credentials while the original driver is still available, then select the new driver and create them again. Do not run mixed gateway versions against the same refresh records. diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx index afe7d8f410..b92cc518ce 100644 --- a/docs/kubernetes/migrate-gateway-config.mdx +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -7,8 +7,9 @@ and mounts) remain chart values. ## Breaking upgrade -Remove every application-only `server.*`, `supervisor.*`, and `upstreamProxy.*` -setting before upgrading. There is no compatibility translation. Kubernetes pull +Remove every application-only `server.*` setting before upgrading. There is no +compatibility translation. Retain chart-owned `upstreamProxy.*` settings; Helm +derives their runtime Kubernetes-driver fields. Kubernetes pull policy spellings such as `Always` are not aliases: use `always`, `if_not_present`, or `never`. `gatewayConfig` is strictly a non-secret configuration contract: do not put passwords, tokens, private keys, @@ -32,11 +33,11 @@ runtime counterparts. | `server.otlp.{endpoint,serviceName}` | `openshell.gateway.otlp.{endpoint,service_name}` | | `server.auth.allowUnauthenticatedUsers` | `openshell.gateway.auth.allow_unauthenticated_users` | | `server.oidc.*` | `openshell.gateway.oidc` with snake_case keys | -| `server.sandboxImage*`, workspace storage/runtime/AppArmor/user namespace settings | `openshell.drivers.kubernetes` | +| `server.sandboxImage*`, workspace storage/runtime and user namespace settings | `openshell.drivers.kubernetes`; isolated runtime images use `sandboxRuntime.image` and `supervisor.image`. Kubernetes AppArmor configuration is removed by RFC 0012. | | `server.drivers.kubernetes.*` | `openshell.drivers.kubernetes` | | `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.gateway.gateway_jwt` or `openshell.drivers.kubernetes.sa_token_ttl_secs` | -| `supervisor.*` | `openshell.drivers.kubernetes` and `.sidecar` | -| `upstreamProxy.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}` | +| `supervisor.image.*` | Chart-owned `supervisor.image.*`, derived into the Kubernetes driver. `supervisor.topology` and `supervisor.sidecar.*` are removed by RFC 0012 with no replacement. | +| `upstreamProxy.*` | Retained as chart-owned packaging input; Helm derives `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}`. | | `server.credentialDrivers.{kubernetesSecrets,vault}.*` | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | | `server.providerTokenGrants.spiffe.{enabled,workloadApiSocketPath}` | `openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path`; omit it to disable SPIFFE provider grants. | | `server.hostGatewayIP` | Retained as the chart-owned host-alias input; the chart derives `openshell.drivers.kubernetes.host_gateway_ip`. | @@ -65,8 +66,8 @@ gatewayConfig: ``` For Vault, set `openshell.gateway.credential_drivers: [vault]` and configure -`openshell.credential_drivers.vault`. For a proxy, set `https_proxy` and the -`proxy_auth_secret_name`/`proxy_auth_secret_key` references in the Kubernetes -driver table. TLS remains Secret-backed; set `server.disableTls: true` only for +`openshell.credential_drivers.vault`; use `credentialDrivers.vault.caConfigMapName` +for a private Vault CA. For a proxy, use chart-owned `upstreamProxy.url`, +`noProxy`, and `authSecret` settings. TLS remains Secret-backed; set `server.disableTls: true` only for trusted external termination. A null map field omits it; null array elements are invalid. Strings evaluate Helm `tpl`; other values do not. diff --git a/tasks/test.toml b/tasks/test.toml index 1f41dca535..27c13c2f38 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -104,7 +104,6 @@ run = [ # with test-only helpers enabled. "cargo test --workspace --exclude openshell-server", "cargo test -p openshell-server --features test-support", - "cargo nextest run --config-file .config/nextest.toml --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml", ] run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native" hide = true @@ -218,11 +217,6 @@ run = [ "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] -["e2e:kubernetes:isolation"] -description = "Run Kubernetes e2e with the workload network fence and separate supervisor" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" - ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } @@ -281,7 +275,7 @@ depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" ["e2e:kubernetes:external-driver"] -description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver" +description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver sidecar" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" From b9f02f276c433ae3dd518bccc6f3dca0cd0fc492 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 16 Sep 2026 12:03:26 +0100 Subject: [PATCH 24/29] fix(e2e): align Kubernetes parity with RFC 0012 Signed-off-by: Gaizka Menendez Hernandez --- docs/kubernetes/migrate-gateway-config.mdx | 2 +- e2e/parity/kubernetes-options-test.sh | 14 ++++++++++--- e2e/parity/kubernetes-options.sh | 24 +++++++++++++--------- e2e/rust/e2e-kubernetes.sh | 5 +++-- e2e/with-kube-gateway.sh | 9 ++++---- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx index b92cc518ce..fab2ea1258 100644 --- a/docs/kubernetes/migrate-gateway-config.mdx +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -28,7 +28,7 @@ runtime counterparts. | Removed value | New location | | --- | --- | -| `server.name`, `logLevel`, `enableLoopbackServiceHttp`, `policyValidationFailureMode` | `openshell.gateway.{name,log_level,enable_loopback_service_http,policy_validation_failure_mode}` | +| `server.name`, `server.logLevel`, `server.enableLoopbackServiceHttp`, `server.policyValidationFailureMode` | `openshell.gateway.{name,log_level,enable_loopback_service_http,policy_validation_failure_mode}` | | `server.grpcRateLimit.{requests,windowSeconds}` | `openshell.gateway.{grpc_rate_limit_requests,grpc_rate_limit_window_seconds}` | | `server.otlp.{endpoint,serviceName}` | `openshell.gateway.otlp.{endpoint,service_name}` | | `server.auth.allowUnauthenticatedUsers` | `openshell.gateway.auth.allow_unauthenticated_users` | diff --git a/e2e/parity/kubernetes-options-test.sh b/e2e/parity/kubernetes-options-test.sh index 4f14687e7a..ff455ef4fc 100644 --- a/e2e/parity/kubernetes-options-test.sh +++ b/e2e/parity/kubernetes-options-test.sh @@ -9,6 +9,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SCRIPT="${ROOT}/e2e/parity/kubernetes-options.sh" TMP="$(mktemp -d)" trap 'rm -rf "${TMP}"' EXIT +TRUE_BIN="$(type -P true)" OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config baseline >"${TMP}/baseline.toml" OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config candidate >"${TMP}/candidate.toml" @@ -28,6 +29,13 @@ check(shared.isdisjoint(candidate['gateway'].keys()),'candidate leaked driver fi check(shared <= candidate['drivers']['kubernetes'].keys(),'candidate driver fields missing') b=dict(baseline['drivers']['kubernetes']); b.update({key:baseline['gateway'][key] for key in shared}) c=dict(candidate['drivers']['kubernetes']) +# These fields belong only to the frozen schema-v1 baseline. RFC 0012 removed +# their configuration surface; the Agent Sandbox controller owns its internals. +legacy_v1_only={'supervisor_sideload_method','topology','app_armor_profile'} +check(legacy_v1_only <= b.keys(),'baseline legacy fields missing') +check(legacy_v1_only.isdisjoint(c.keys()),'candidate retained RFC 0012 fields') +for field in legacy_v1_only: + b.pop(field) for projection in (b,c): projection['gateway_id']='' projection['grpc_endpoint']='http://host.openshell.internal:' @@ -39,9 +47,9 @@ PY : >"${TMP}/kubeconfig" set +e OPENSHELL_PARITY_BASELINE_ROOT="${TMP}/not-a-worktree" \ -OPENSHELL_PARITY_BASELINE_GATEWAY=/bin/true \ -OPENSHELL_PARITY_CANDIDATE_GATEWAY=/bin/true \ -OPENSHELL_PARITY_CLI=/bin/true \ +OPENSHELL_PARITY_BASELINE_GATEWAY="${TRUE_BIN}" \ +OPENSHELL_PARITY_CANDIDATE_GATEWAY="${TRUE_BIN}" \ +OPENSHELL_PARITY_CLI="${TRUE_BIN}" \ OPENSHELL_PARITY_KUBECONFIG="${TMP}/kubeconfig" \ OPENSHELL_PARITY_KUBE_CONTEXT=default/external-production-cluster \ OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 \ diff --git a/e2e/parity/kubernetes-options.sh b/e2e/parity/kubernetes-options.sh index 3ddca0355e..5da6fe0c60 100644 --- a/e2e/parity/kubernetes-options.sh +++ b/e2e/parity/kubernetes-options.sh @@ -52,6 +52,14 @@ write_config() { local run_dir=$5 local gateway_id="step8-${variant}-${RUN_ID}" local pull_policy + local print_to_stdout=false + + # macOS does not permit redirecting a heredoc directly to /dev/stdout. + # Keep --print-config portable for the deterministic contract test. + if [ "${path}" = /dev/stdout ]; then + path="$(mktemp "${TMPDIR:-/tmp}/openshell-parity-config.XXXXXX")" + print_to_stdout=true + fi if [ "${variant}" = baseline ]; then pull_policy=IfNotPresent @@ -132,8 +140,6 @@ image_pull_secrets = ["parity-pull-secret"] service_account_name = "parity-sandbox" supervisor_image = "${SUPERVISOR_IMAGE}" supervisor_image_pull_policy = "${pull_policy}" -supervisor_sideload_method = "init-container" -topology = "combined" grpc_endpoint = "http://host.openshell.internal:${port}" ssh_socket_path = "/run/openshell/parity-kubernetes-ssh.sock" client_tls_secret_name = "parity-client-tls" @@ -143,11 +149,15 @@ sa_token_ttl_secs = 600 workspace_default_storage_size = "64Mi" workspace_storage_class = "standard" default_runtime_class_name = "${RUNTIME_CLASS}" -app_armor_profile = "Unconfined" sandbox_uid = 1000 sandbox_gid = 1000 EOF fi + + if ${print_to_stdout}; then + cat "${path}" + rm -f "${path}" + fi } if [ "${1:-}" = --print-config ]; then @@ -358,7 +368,6 @@ check(env['OPENSHELL_SSH_SOCKET_PATH']=='/run/openshell/parity-kubernetes-ssh.so check(env['OPENSHELL_SANDBOX_UID']=='1000' and env['OPENSHELL_SANDBOX_GID']=='1000','sandbox identity differs') check(('host.openshell.internal',host_ip) in hosts and ('host.docker.internal',host_ip) in hosts,'host aliases differ') check(spec['runtimeClassName']==runtime_class and spec.get('hostUsers',True) is not False,'RuntimeClass or user namespace posture differs') -check(agent['securityContext']['appArmorProfile']['type']=='Unconfined','AppArmor profile differs') check(agent['resources']['requests']=={'cpu':'250m','memory':'128Mi'},'resource requests differ') check(agent['resources']['limits']=={'cpu':'250m','memory':'128Mi'},'resource limits differ') check(vols['openshell-sa-token']['projected']['sources'][0]['serviceAccountToken']['expirationSeconds']==600,'ServiceAccount token TTL differs') @@ -369,22 +378,17 @@ check(pvc['spec']['resources']['requests']['storage']=='64Mi','PVC storage reque labels=sb['metadata']['labels'] for key in ('openshell.ai/sandbox-id','openshell.ai/sandbox-name','openshell.ai/sandbox-workspace','openshell.ai/gateway-id','openshell.ai/managed-by'): check(labels.get(key),f'managed label {key} missing') -observed_sideload='init-container' if 'openshell-supervisor-install' in inits else 'unknown' -observed_topology='combined' if [c['name'] for c in spec['containers']]==['agent'] else 'other' observed_workspace_mode='shared' if pvc['metadata']['namespace']==pod['metadata']['namespace'] and pvc['metadata']['name'].startswith('workspace-default--') else 'other' -check(observed_sideload=='init-container','supervisor sideload method differs') -check(observed_topology=='combined','supervisor topology differs') check(observed_workspace_mode=='shared','workspace placement differs') normalized={ 'scenario':'kubernetes-core-options','pod_phase':'Running','sandbox_ready':True, 'sandbox_image':agent['image'],'sandbox_image_pull_policy':agent['imagePullPolicy'], 'image_pull_secrets':['parity-pull-secret'],'service_account':'parity-sandbox', 'supervisor_image':install['image'],'supervisor_image_pull_policy':install['imagePullPolicy'], - 'supervisor_sideload_method':observed_sideload,'topology':observed_topology, 'callback_endpoint_host':'host.openshell.internal','callback_exec':True, 'ssh_socket_path':env['OPENSHELL_SSH_SOCKET_PATH'],'client_tls_secret':'parity-client-tls', 'host_gateway_ip':host_ip,'sa_token_ttl_secs':600,'runtime_class_handler':'runc', - 'enable_user_namespaces':False,'app_armor_profile':'Unconfined','sandbox_uid':1000,'sandbox_gid':1000, + 'enable_user_namespaces':False,'sandbox_uid':1000,'sandbox_gid':1000, 'workspace_mode':observed_workspace_mode,'workspace_storage':'64Mi','workspace_storage_class':'standard','pvc_phase':'Bound', 'cpu':'250m','memory':'128Mi','managed_labels':True, } diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 5c4d7d6c2b..95570d4bc8 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -10,8 +10,9 @@ # # Features: the default set includes `e2e-host-gateway` so tests that rely on # the sandbox-side `host.openshell.internal` alias compile and run. The -# wrapper detects the cluster's host-routable IP and wires it into -# `gatewayConfig.openshell.drivers.kubernetes.host_gateway_ip`. Targeting a +# wrapper detects the cluster's host-routable IP and passes it through the +# chart-owned server.hostGatewayIP input, which derives the runtime field. +# Targeting a # cluster where the test host is unreachable from pods? Set # OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the alias-dependent tests. # diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 20be9c48ca..b6219879ff 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1218,10 +1218,11 @@ fi helm_extra_args=() helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") -# Sandboxes pull the supervisor image named by the runtime TOML, not the -# gateway workload's `supervisor.image` field. Keep that runtime image aligned -# with the locally built/imported image (and with CI's registry/tag overrides). -helm_extra_args+=(--set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") +helm_extra_args+=(--set supervisor.sandboxRuntime.networkPolicyEnforced=true) +# `supervisor.image` is the chart-owned input and is derived into the runtime +# TOML. Keep it aligned with the locally built/imported image and CI overrides. +helm_extra_args+=(--set-string "supervisor.image.repository=${REGISTRY_VALUE}/supervisor") +helm_extra_args+=(--set-string "supervisor.image.tag=${IMAGE_TAG_VALUE}") if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 From b67982d954fe89ea8e434a070ff6904a006aeccc Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Fri, 18 Sep 2026 12:49:48 +0100 Subject: [PATCH 25/29] fix(helm): complete gateway config migration Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/README.md | 88 ++++++++++--------- .../ci/values-corporate-proxy-e2e.yaml | 16 ++-- .../openshell/ci/values-openshift-e2e.yaml | 11 +-- deploy/helm/openshell/templates/_toml.tpl | 37 ++------ .../openshell/tests/clusterrole_test.yaml | 8 +- .../tests/credential_drivers_test.yaml | 3 + .../fixtures/parser-validation-values.yaml | 9 +- deploy/helm/test-gateway-config-parser.sh | 5 +- .../helm/test-gateway-resource-coherence.sh | 2 +- docs/kubernetes/migrate-gateway-config.mdx | 17 ++-- 10 files changed, 88 insertions(+), 108 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index a77bcb456f..d426d16422 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -149,7 +149,12 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell +<<<<<<< HEAD helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version +======= +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) ``` ## Install on OpenShift @@ -157,14 +162,18 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ +<<<<<<< HEAD --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null +======= + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) ``` On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the @@ -222,6 +231,10 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ +<<<<<<< HEAD +======= + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -276,33 +289,10 @@ namespace to limit access to OpenShell-managed Secrets. Append these flags to any of the PostgreSQL commands above for OpenShift: ``` ---set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` -### High availability - -Set `replicaCount` above `1` only with `server.externalDbSecret`; the default -SQLite database is per pod and cannot coordinate multiple gateway replicas. -The chart creates a headless peer Service for gateway-to-gateway relay traffic. -StatefulSet pods use stable pod DNS names through that headless Service. -Deployment pods advertise their pod IP with `OPENSHELL_PEER_ENDPOINT`, because -Kubernetes does not assign stable per-pod DNS names to Deployment replicas. - -Gateway peer traffic uses Kubernetes ServiceAccount identity. Each gateway pod -mounts a projected, pod-bound ServiceAccount token with audience -`openshell-gateway-peer`; receiving replicas validate that token with the -Kubernetes TokenReview API, verify the live pod UID and Helm selector labels, -and authorize only the internal `PeerRelay` RPC. The chart does not create or -accept a shared gateway peer Secret. - -With gateway TLS enabled, peer calls use the chart CA and client TLS Secret for -server verification and mTLS. The client verifies the stable gateway Service -DNS name while connecting directly to the owning pod. Custom TLS Secrets must -include that Service DNS name in the server certificate and provide the CA and -client credentials configured by `server.tls`. - ## Secret bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` @@ -319,12 +309,16 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants -Set `server.providerTokenGrants.spiffe.enabled=true` to let the gateway and -sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The -chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs, -mounts the SPIFFE CSI socket into the gateway pod, exports -`OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`, and passes the socket path to -the Kubernetes driver so sandbox pods can mount the same socket. +Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` +to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +grants. The chart keeps supervisor-to-gateway authentication on gateway-minted +sandbox JWTs and passes the configured socket path to the Kubernetes driver. + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock +``` For local development, uncomment the SPIRE Helm releases in `skaffold.yaml` and add `ci/values-spire.yaml` to the OpenShell release values files. @@ -347,7 +341,9 @@ discovery endpoint or its TLS CA. | certManager.serverDnsNames | list | `["openshell","openshell.openshell.svc","openshell.openshell.svc.cluster.local","localhost","openshell.localhost","*.openshell.localhost","host.docker.internal"]` | DNS SANs on the cert-manager-issued server certificate. | | certManager.serverIpAddresses | list | `["127.0.0.1"]` | IP SANs on the cert-manager-issued server certificate. | | certManager.serverIssuerRef | object | `{"group":"","kind":"","name":""}` | Override the issuerRef for the external server Certificate (e.g. a real LetsEncrypt/ACME ClusterIssuer for a publicly-trusted cert on an external hostname). When set, the chart creates a second server certificate from this issuer with only the hostnames in serverDnsNames; the internal server certificate is always signed by the chart's own CA. Leave name empty to use the chart CA for all server certificates (default). Requires certManager.enabled=true. | +| credentialDrivers.vault.caConfigMapName | string | `""` | ConfigMap containing the private Vault/OpenBao CA certificate under the ca.crt key. Helm mounts it only when the Vault driver is selected. | | fullnameOverride | string | `""` | Override the full generated resource name. | +<<<<<<< HEAD | gateway.image.digest | string | `""` | Gateway image digest. When set, this takes precedence over tag. | | gateway.image.pullPolicy | string | `nil` | Gateway image pull policy. Empty uses global.image.pullPolicy. | | gateway.image.registry | string | `""` | Gateway image registry. Empty uses global.image.registry. | @@ -356,6 +352,9 @@ discovery endpoint or its TLS CA. | global.image.pullPolicy | string | `"IfNotPresent"` | Shared OpenShell image pull policy. Individual image pull policies take precedence. | | global.image.registry | string | `"ghcr.io/nvidia"` | Shared OpenShell image registry. Individual image registries take precedence. | | global.image.tag | string | `""` | Shared OpenShell image tag. Defaults to the chart appVersion when empty. | +======= +| gatewayConfig | object | `{"openshell":{"version":2},"openshell.drivers.kubernetes":{"client_tls_secret_name":"{{ .Values.server.tls.clientTlsSecretName }}","default_image":"ghcr.io/nvidia/openshell-community/sandboxes/base:latest","gateway_id":"{{ include \"openshell.fullname\" . }}","grpc_endpoint":"{{ include \"openshell.grpcEndpoint\" . }}","namespace":"{{ include \"openshell.sandboxNamespace\" . }}","sa_token_ttl_secs":3600,"sandbox_runtime":{"boundary_port":5500,"network_policy_enforced":false},"sandbox_runtime_image":"ghcr.io/nvidia/openshell/sandbox:{{ .Chart.AppVersion }}","service_account_name":"{{ include \"openshell.sandboxServiceAccountName\" . }}","supervisor_image":"ghcr.io/nvidia/openshell/supervisor:{{ .Chart.AppVersion }}","workspace_mode":"shared"},"openshell.drivers.kubernetes.managed_ssh_ingress":{"enabled":true,"gateway_namespace":"{{ .Release.Namespace }}","gateway_pod_selector":{"app.kubernetes.io/instance":"{{ .Release.Name }}","app.kubernetes.io/name":"{{ include \"openshell.name\" . }}"}},"openshell.gateway":{"bind_address":"0.0.0.0:{{ .Values.service.port }}","compute_driver":"kubernetes","enable_loopback_service_http":true,"health_bind_address":"0.0.0.0:{{ .Values.service.healthPort }}","log_level":"info","metrics_bind_address":"0.0.0.0:{{ .Values.service.metricsPort }}","name":"{{ include \"openshell.fullname\" . }}","policy_validation_failure_mode":"fail_closed"},"openshell.gateway.credential_storage":{"key_encryption_key_env":"{{ include \"openshell.credentialStorageKeyEncryptionKeyEnvName\" . }}"},"openshell.gateway.gateway_jwt":{"gateway_id":"{{ include \"openshell.fullname\" . }}","kid_path":"/etc/openshell-jwt/kid","public_key_path":"/etc/openshell-jwt/public.pem","signing_key_path":"/etc/openshell-jwt/signing.pem","ttl_secs":3600},"openshell.gateway.tls":{"cert_path":"/etc/openshell-tls/server/tls.crt","client_ca_path":"/etc/openshell-tls/client-ca/ca.crt","key_path":"/etc/openshell-tls/server/tls.key"}}` | Non-secret gateway application configuration. Top-level keys name TOML tables and are rendered into the mounted gateway.toml file. Kubernetes resource inputs remain outside this map; template expressions derive the corresponding runtime values from their resource owner. | +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | grpcRoute.backendTLSPolicy.caCertificateConfigMapName | string | `""` | Name of the ConfigMap containing the CA certificate (key: ca.crt) used to validate the gateway pod's TLS certificate. Defaults to `-backend-ca` when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds waiting for cert-manager to issue the server certificate, then creates the ConfigMap. A single install usually succeeds; if cert-manager takes longer, increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), the install fails if the timeout is reached; set failOnTimeout=false to allow the install to succeed and run `helm upgrade` after the certificate is issued. | | grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | | grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (`..svc.cluster.local`) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | @@ -373,6 +372,7 @@ discovery endpoint or its TLS CA. | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | | networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | +| oidc.caConfigMapName | string | `""` | | | openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | | openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | @@ -402,6 +402,7 @@ discovery endpoint or its TLS CA. | rbac.create | bool | `true` | Create the RBAC objects that grant the gateway ServiceAccount access. Disable to supply the namespaced sandbox and peer Role/RoleBinding and the cluster-scoped ClusterRole/ClusterRoleBinding out of band. The certgen hook and credential driver RBAC keep their own flags. | | replicaCount | int | `1` | Number of OpenShell gateway replicas. Values greater than 1 require server.externalDbSecret because the default SQLite backend is per pod. | | resources | object | `{}` | Gateway pod resource requests and limits. | +<<<<<<< HEAD | sandbox.image.digest | string | `""` | Sandbox image digest. When set, this takes precedence over tag. | | sandbox.image.pullPolicy | string | `nil` | Sandbox image pull policy. Leave unset to use the Kubernetes image default. | | sandbox.image.repository | string | `"nvcr.io/nvidia/base/ubuntu"` | Default standalone sandbox image repository. | @@ -411,6 +412,8 @@ discovery endpoint or its TLS CA. | sandboxRuntime.image.registry | string | `""` | Sandbox runtime image registry. Empty uses global.image.registry. | | sandboxRuntime.image.repository | string | `"openshell/sandbox"` | Sandbox runtime image repository. | | sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag. Defaults to the chart appVersion when empty. | +======= +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | @@ -418,6 +421,7 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | +<<<<<<< HEAD | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | | server.credentialDrivers.kubernetesSecrets.createNamespace | bool | `false` | Create the credential namespace. Requires a namespace other than the release namespace. The Namespace is retained on uninstall so stored credentials survive; an existing Namespace not owned by this release is left untouched. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | @@ -481,17 +485,21 @@ discovery endpoint or its TLS CA. | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | +======= +| server.credentialStorage.existingSecret | string | `""` | Name of a pre-existing Secret containing the key-encryption key. When set, the chart does NOT generate a new Secret; it references this one instead. The Secret must contain a key named "key-encryption-key" with a base64-encoded 32-byte value. Required for GitOps workflows that render manifests with `helm template` (where `lookup` is unavailable). | +| server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | +| server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | +| server.hostGatewayIP | string | `""` | Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get hostAliases entries mapping host.docker.internal and host.openshell.internal to this IP, allowing them to reach services running on the Docker host. Auto-detected by the cluster entrypoint script. | +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | server.sandboxJwt.secretDefaultMode | string | `""` | File mode for the mounted JWT signing key Secret. Default 0400 (owner-read only). Override to 0440 or 0444 if the container UID does not match the volume file owner. | | server.sandboxJwt.signingSecretName | string | `""` | Name of the Opaque Secret holding the signing key material. Empty falls back to the chart fullname with "-jwt-keys" appended. | -| server.sandboxJwt.ttlSecs | int | `3600` | Token TTL in seconds. Defaults to 3600 (1h). | | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | | server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to disable client certificate verification for HTTPS-only mode. | | server.tls.clientTlsSecretName | string | `"openshell-client-tls"` | K8s secret mounted into sandbox pods for mTLS to the server. | | server.tls.enableMtls | bool | `true` | Enable mTLS client certificate authentication. When false, the gateway runs HTTPS-only without requiring client certificates (use OIDC for auth instead). Must be false when using BackendTLSPolicy because ingress proxies cannot present client certificates to the backend. | -| server.workspaceDefaultStorageSize | string | `""` | Default storage size for the workspace PVC in sandbox pods. Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). Empty = built-in default (2Gi). | -| server.workspaceStorageClass | string | `""` | Kubernetes StorageClass for the workspace PVC in sandbox pods. Empty (default) = omit storageClassName, using the cluster's default StorageClass. Set this on clusters with no default StorageClass, otherwise the workspace PVC stays Pending and the sandbox never starts. | | service.healthPort | int | `8081` | Gateway health service port. | | service.metricsPort | int | `9090` | Gateway metrics service port. | | service.port | int | `8080` | Gateway gRPC/HTTP service port. | @@ -499,22 +507,16 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | +<<<<<<< HEAD | supervisor.image.digest | string | `""` | Supervisor image digest. When set, this takes precedence over tag. | | supervisor.image.pullPolicy | string | `nil` | Supervisor image pull policy. Empty uses global.image.pullPolicy. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.registry | string | `""` | Supervisor image registry. Empty uses global.image.registry. | | supervisor.image.repository | string | `"openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | +======= +>>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | tolerations | list | `[]` | Tolerations for the gateway pod. | -| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"caBundle":{"configMapName":"","key":"ca.crt"},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | -| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | -| upstreamProxy.authSecret.key | string | `""` | Secret key containing the proxy credential. | -| upstreamProxy.authSecret.name | string | `""` | Existing Secret in the sandbox namespace containing a user:pass value. | -| upstreamProxy.caBundle.configMapName | string | `""` | ConfigMap in the release namespace holding the corporate proxy CA bundle. Required for an https:// proxy with a private CA, and for a TLS-intercepting proxy that re-signs upstream certificates. The gateway reads it and stages it into each sandbox's immutable supervisor bootstrap Secret, so the anchor stays in the gateway's trust domain rather than the workload namespace. Supply only the CA that signs your proxy's certificate, or that the proxy re-signs intercepted upstream certificates with. Public roots already come from the supervisor image and its TLS stack, so a full merged trust bundle (for example an OpenShift config.openshift.io/inject-trusted-cabundle ConfigMap) adds hundreds of kilobytes of duplicated roots and can exceed the sandbox boundary's control-frame budget. | -| upstreamProxy.caBundle.key | string | `"ca.crt"` | Key inside that ConfigMap. Change this only to reuse an existing ConfigMap whose key is not ca.crt. | -| upstreamProxy.connectByHostname | bool | `false` | Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. | -| upstreamProxy.noProxy | string | `""` | Comma-separated destinations that bypass only the corporate proxy. | -| upstreamProxy.url | string | `""` | Proxy URL in http://host:port or https://host:port form. An https:// proxy whose certificate is not publicly trusted also needs caBundle below. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | | workspaceResources.enabled | bool | `true` | Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy from this chart. Disable for a gateway-only release. | diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index c015775efd..5b3092ae07 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,11 +3,11 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -upstreamProxy: - # The e2e wrapper replaces this endpoint with its dynamically allocated port. - # Keep the overlay valid when rendered independently as well. - url: http://host.openshell.internal:8080 - authSecret: - name: openshell-e2e-proxy-auth - key: proxy-auth - authAllowInsecure: true +gatewayConfig: + openshell.drivers.kubernetes: + # The e2e wrapper replaces this endpoint with its dynamically allocated + # port. Keep the overlay valid when rendered independently as well. + https_proxy: http://host.openshell.internal:8080 + proxy_auth_secret_name: openshell-e2e-proxy-auth + proxy_auth_secret_key: proxy-auth + proxy_auth_allow_insecure: true diff --git a/deploy/helm/openshell/ci/values-openshift-e2e.yaml b/deploy/helm/openshell/ci/values-openshift-e2e.yaml index bbe68c3e1c..a4e1e8fb2a 100644 --- a/deploy/helm/openshell/ci/values-openshift-e2e.yaml +++ b/deploy/helm/openshell/ci/values-openshift-e2e.yaml @@ -38,14 +38,9 @@ server: gatewayConfig: openshell.gateway.auth: allow_unauthenticated_users: true - -sandboxRuntime: - image: - pullPolicy: Always - -supervisor: - image: - pullPolicy: Always + openshell.drivers.kubernetes: + sandbox_runtime_image_pull_policy: always + supervisor_image_pull_policy: always openshiftRoute: enabled: true diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index da3e5aecee..018ee474cc 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -99,6 +99,14 @@ field must not require a Helm template change. {{- define "openshell.gatewayConfigToml" -}} {{- $root := . -}} {{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} +{{/* External credential drivers own their storage. Do not configure the +chart-managed encrypted database store when any driver is selected: its KEK +environment variable is intentionally not mounted in that mode. */}} +{{- $configuredGateway := get $config "openshell.gateway" | default dict -}} +{{- $configuredCredentialDrivers := get $configuredGateway "credential_drivers" | default list -}} +{{- if gt (len $configuredCredentialDrivers) 0 -}} +{{- $_ := unset $config "openshell.gateway.credential_storage" -}} +{{- end -}} {{/* Kubernetes packaging owns host aliases. Do not permit a second runtime source to make sandbox callback hostnames disagree with the pod spec. */}} {{- $kubernetes := get $config "openshell.drivers.kubernetes" | default dict -}} @@ -109,35 +117,6 @@ source to make sandbox callback hostnames disagree with the pod spec. */}} {{- end -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} -{{/* RFC 0012 packaging inputs are authoritative for paired runtime images, -the network-fence acknowledgement, and corporate proxy Secret wiring. */}} -{{- $runtime := .Values.sandboxRuntime | default dict -}} -{{- $runtimeImage := get $runtime "image" | default dict -}} -{{- $supervisor := .Values.supervisor | default dict -}} -{{- $supervisorImage := get $supervisor "image" | default dict -}} -{{- $runtimeTag := get $runtimeImage "tag" | default .Values.image.tag | default .Chart.AppVersion -}} -{{- $supervisorTag := get $supervisorImage "tag" | default .Values.image.tag | default .Chart.AppVersion -}} -{{- $_ := set $kubernetes "sandbox_runtime_image" (printf "%s:%s" (get $runtimeImage "repository" | default "ghcr.io/nvidia/openshell/sandbox") $runtimeTag) -}} -{{- $_ := set $kubernetes "supervisor_image" (printf "%s:%s" (get $supervisorImage "repository" | default "ghcr.io/nvidia/openshell/supervisor") $supervisorTag) -}} -{{- if get $runtimeImage "pullPolicy" -}} -{{- $_ := set $kubernetes "sandbox_runtime_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $runtimeImage "pullPolicy")) -}} -{{- else -}}{{- $_ := unset $kubernetes "sandbox_runtime_image_pull_policy" -}}{{- end -}} -{{- if get $supervisorImage "pullPolicy" -}} -{{- $_ := set $kubernetes "supervisor_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $supervisorImage "pullPolicy")) -}} -{{- else -}}{{- $_ := unset $kubernetes "supervisor_image_pull_policy" -}}{{- end -}} -{{- $runtimeConfig := get $supervisor "sandboxRuntime" | default dict -}} -{{- $_ := set $kubernetes "sandbox_runtime" (dict "network_policy_enforced" (get $runtimeConfig "networkPolicyEnforced") "boundary_port" (get $runtimeConfig "boundaryPort" | default 5500)) -}} -{{- $proxy := .Values.upstreamProxy | default dict -}} -{{- range $runtimeKey := list "https_proxy" "no_proxy" "proxy_auth_secret_name" "proxy_auth_secret_key" "proxy_auth_allow_insecure" "proxy_connect_by_hostname" -}}{{- $_ := unset $kubernetes $runtimeKey -}}{{- end -}} -{{- if get $proxy "url" -}}{{- $_ := set $kubernetes "https_proxy" (get $proxy "url") -}}{{- end -}} -{{- if get $proxy "noProxy" -}}{{- $_ := set $kubernetes "no_proxy" (get $proxy "noProxy") -}}{{- end -}} -{{- $proxySecret := get $proxy "authSecret" | default dict -}} -{{- if get $proxySecret "name" -}}{{- $_ := set $kubernetes "proxy_auth_secret_name" (get $proxySecret "name") -}}{{- end -}} -{{- if get $proxySecret "key" -}}{{- $_ := set $kubernetes "proxy_auth_secret_key" (get $proxySecret "key") -}}{{- end -}} -{{- if or (get $proxySecret "name") (get $proxySecret "key") -}}{{- $_ := set $kubernetes "proxy_auth_allow_insecure" (get $proxy "authAllowInsecure") -}}{{- end -}} -{{- if get $proxy "connectByHostname" -}}{{- $_ := set $kubernetes "proxy_connect_by_hostname" true -}}{{- end -}} -{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} - {{/* A Vault CA is a Kubernetes resource reference, not a free-form runtime path. Derive its mounted path only from the chart-owned ConfigMap reference. */}} {{- $credentialDrivers := .Values.credentialDrivers | default dict -}} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 08907c614a..d42c9732ab 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -126,7 +126,9 @@ tests: - it: grants managed sandbox-runtime companion permissions set: - server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed asserts: - contains: path: rules @@ -149,7 +151,9 @@ tests: - it: grants operator sandbox-runtime companion permissions set: - server.drivers.kubernetes.workspaceMode: operator + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator asserts: - contains: path: rules diff --git a/deploy/helm/openshell/tests/credential_drivers_test.yaml b/deploy/helm/openshell/tests/credential_drivers_test.yaml index d7f1907c0d..f06d47134f 100644 --- a/deploy/helm/openshell/tests/credential_drivers_test.yaml +++ b/deploy/helm/openshell/tests/credential_drivers_test.yaml @@ -97,6 +97,9 @@ tests: - notMatchRegex: path: data["gateway.toml"] pattern: 'transport\s*=\s*"in_tree"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.gateway\.credential_storage\]' - it: rejects multiple enabled credential drivers template: templates/statefulset.yaml diff --git a/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml index 12a00174b3..7465a3327f 100644 --- a/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml +++ b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml @@ -5,15 +5,8 @@ # intentionally opaque to the gateway loader, which makes it a safe way to # exercise every TOML shape supported by the chart serializer. gatewayConfig: - openshell.gateway: - name: '{{ .Release.Namespace }}/{{ .Release.Name }} "quoted" \\ path' - server_sans: - - localhost - - gateway.example.test - provider_profile_sources: - - type: builtin - - type: user openshell.drivers.parser-validation: + rendered_name: '{{ .Release.Namespace }}/{{ .Release.Name }} "quoted" \\ path' boolean_value: true integer_value: 42 float_value: 1.5 diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 93b25ef3c8..0c4268c6e1 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -22,6 +22,7 @@ render() { helm template parser-validation "${chart}" \ --namespace parser-namespace \ --set agentSandbox.preflight.enabled=false \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ "$@" >"${output}" } @@ -43,7 +44,7 @@ if helm template parser-validation "${chart}" --namespace parser-namespace \ echo "the chart must require explicit NetworkPolicy enforcement acknowledgement" >&2 exit 1 fi -grep -F 'supervisor.sandboxRuntime.networkPolicyEnforced must be true' "${work_dir}/unacknowledged-default.err" >/dev/null +grep -F 'gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced must be true' "${work_dir}/unacknowledged-default.err" >/dev/null # The runtime default is valid after the required infrastructure acknowledgement. render "${work_dir}/default.yaml" @@ -55,7 +56,7 @@ preflight "${work_dir}/default.toml" render "${work_dir}/shapes.yaml" --values "${fixture}" extract_toml "${work_dir}/shapes.yaml" "${work_dir}/shapes.toml" preflight "${work_dir}/shapes.toml" -grep -F 'name = "parser-namespace/parser-validation' "${work_dir}/shapes.toml" >/dev/null +grep -F 'rendered_name = "parser-namespace/parser-validation' "${work_dir}/shapes.toml" >/dev/null grep -F 'empty_value = ""' "${work_dir}/shapes.toml" >/dev/null if grep -Fq 'omitted_value' "${work_dir}/shapes.toml"; then echo "null gatewayConfig values must be omitted from gateway.toml" >&2 diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index 2f5a603935..e23f14c941 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -16,7 +16,7 @@ render() { helm template resource-coherence "${chart}" \ --namespace resource-namespace \ --set agentSandbox.preflight.enabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ "$@" >"${work_dir}/${name}.yaml" if ! awk 'BEGIN { RS="---" } diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx index fab2ea1258..741913bbcc 100644 --- a/docs/kubernetes/migrate-gateway-config.mdx +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -8,8 +8,8 @@ and mounts) remain chart values. ## Breaking upgrade Remove every application-only `server.*` setting before upgrading. There is no -compatibility translation. Retain chart-owned `upstreamProxy.*` settings; Helm -derives their runtime Kubernetes-driver fields. Kubernetes pull +compatibility translation. Runtime images, proxy settings, and NetworkPolicy +acknowledgement all live in `gatewayConfig`. Kubernetes pull policy spellings such as `Always` are not aliases: use `always`, `if_not_present`, or `never`. `gatewayConfig` is strictly a non-secret configuration contract: do not put passwords, tokens, private keys, @@ -33,11 +33,11 @@ runtime counterparts. | `server.otlp.{endpoint,serviceName}` | `openshell.gateway.otlp.{endpoint,service_name}` | | `server.auth.allowUnauthenticatedUsers` | `openshell.gateway.auth.allow_unauthenticated_users` | | `server.oidc.*` | `openshell.gateway.oidc` with snake_case keys | -| `server.sandboxImage*`, workspace storage/runtime and user namespace settings | `openshell.drivers.kubernetes`; isolated runtime images use `sandboxRuntime.image` and `supervisor.image`. Kubernetes AppArmor configuration is removed by RFC 0012. | +| `server.sandboxImage*`, workspace storage/runtime and user namespace settings | `openshell.drivers.kubernetes`. Kubernetes AppArmor configuration is removed by RFC 0012. | | `server.drivers.kubernetes.*` | `openshell.drivers.kubernetes` | | `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.gateway.gateway_jwt` or `openshell.drivers.kubernetes.sa_token_ttl_secs` | -| `supervisor.image.*` | Chart-owned `supervisor.image.*`, derived into the Kubernetes driver. `supervisor.topology` and `supervisor.sidecar.*` are removed by RFC 0012 with no replacement. | -| `upstreamProxy.*` | Retained as chart-owned packaging input; Helm derives `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}`. | +| `supervisor.image.*`, `supervisor.sandboxRuntime.*` | `openshell.drivers.kubernetes.{supervisor_image,supervisor_image_pull_policy,sandbox_runtime}`. `supervisor.topology` and `supervisor.sidecar.*` are removed by RFC 0012 with no replacement. | +| `upstreamProxy.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}`. | | `server.credentialDrivers.{kubernetesSecrets,vault}.*` | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | | `server.providerTokenGrants.spiffe.{enabled,workloadApiSocketPath}` | `openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path`; omit it to disable SPIFFE provider grants. | | `server.hostGatewayIP` | Retained as the chart-owned host-alias input; the chart derives `openshell.drivers.kubernetes.host_gateway_ip`. | @@ -56,6 +56,9 @@ gatewayConfig: openshell.drivers.kubernetes: default_image: registry.example/sandbox:latest image_pull_policy: if_not_present + supervisor_image: registry.example/openshell-supervisor:latest + sandbox_runtime: + network_policy_enforced: true workspace_mode: managed grpc_rate_limit_requests: null openshell.gateway: @@ -67,7 +70,7 @@ gatewayConfig: For Vault, set `openshell.gateway.credential_drivers: [vault]` and configure `openshell.credential_drivers.vault`; use `credentialDrivers.vault.caConfigMapName` -for a private Vault CA. For a proxy, use chart-owned `upstreamProxy.url`, -`noProxy`, and `authSecret` settings. TLS remains Secret-backed; set `server.disableTls: true` only for +for a private Vault CA. For a proxy, use the Kubernetes-driver fields in +`gatewayConfig`. TLS remains Secret-backed; set `server.disableTls: true` only for trusted external termination. A null map field omits it; null array elements are invalid. Strings evaluate Helm `tpl`; other values do not. From 48fce326625fbea1ce9870001ee1c19a3c3bfbc6 Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Fri, 18 Sep 2026 12:49:59 +0100 Subject: [PATCH 26/29] test(e2e): handle paginated provider lists Signed-off-by: Gaizka Menendez Hernandez --- e2e/rust/tests/credential_drivers.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/e2e/rust/tests/credential_drivers.rs b/e2e/rust/tests/credential_drivers.rs index 3901de1c15..fe59941528 100644 --- a/e2e/rust/tests/credential_drivers.rs +++ b/e2e/rust/tests/credential_drivers.rs @@ -184,9 +184,18 @@ async fn provider_identity(provider_name: &str) -> Result Result<(), String> { - let mut guard = SandboxGuard::create(&[ + let guard = SandboxGuard::create(&[ "--name", sandbox_name, "--provider", @@ -241,9 +250,6 @@ async fn assert_provider_placeholder_available_in_sandbox( ]) .await?; let clean = strip_ansi(&guard.create_output); - // Delete the sandbox before returning: the gateway refuses to delete a - // provider that is still attached to a sandbox. - guard.cleanup().await; if !contains_placeholder_for_env_key(&clean, CREDENTIAL_KEY) { return Err(format!( "sandbox {sandbox_name} did not receive provider credential placeholder:\n{clean}" @@ -379,12 +385,7 @@ async fn provider_credentials_are_stored_in_configured_backend() { let suffix = unique_suffix(); let driver_slug = driver.replace('-', ""); let provider_name = format!("cred-storage-{driver_slug}-{suffix}"); - // Sandbox names are DNS-routable and limited to 19 characters. - let sandbox_name = format!( - "cred-{}-{}", - &driver_slug[..1], - &suffix[suffix.len() - 10..] - ); + let sandbox_name = format!("cred-storage-sandbox-{driver_slug}-{suffix}"); let secret_value = format!("example-e2e-{driver_slug}-{suffix}"); delete_provider(&provider_name).await; From 5b602bd81c373fb45eabd6a31dfc2fbb6dda772e Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Mon, 21 Sep 2026 13:49:07 +0100 Subject: [PATCH 27/29] chore(helm): add SPDX header to TOML template --- deploy/helm/openshell/templates/_toml.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 018ee474cc..69cb073449 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 {{/* Render gatewayConfig as TOML. From b6c5b17775e7ab76ffe8eccf8571fc2ccd452f4c Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 23 Sep 2026 12:05:51 +0100 Subject: [PATCH 28/29] fix(helm): preserve legacy gateway configuration aliases Signed-off-by: Gaizka Menendez Hernandez --- .../openshell/templates/_gateway-workload.tpl | 13 ++++ deploy/helm/openshell/templates/_helpers.tpl | 75 ++++++++++++++++++- deploy/helm/openshell/templates/_toml.tpl | 2 + .../helm/openshell/templates/clusterrole.yaml | 5 +- deploy/helm/openshell/templates/role.yaml | 6 +- .../helm/openshell/templates/rolebinding.yaml | 3 +- .../tests/gateway_config_serializer_test.yaml | 67 +++++++++++++++++ .../tests/gateway_secret_boundary_test.yaml | 10 +++ .../tests/sandbox_namespace_test.yaml | 16 ++++ .../tests/statefulset_client_ca_test.yaml | 24 +++--- deploy/helm/openshell/values.yaml | 5 +- .../helm/test-gateway-resource-coherence.sh | 6 +- 12 files changed, 204 insertions(+), 28 deletions(-) diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 97569470b8..124789ac90 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -134,6 +134,11 @@ spec: mountPath: /etc/openshell-tls/vault readOnly: true {{- end }} + {{- if .Values.upstreamProxy.caBundle.configMapName }} + - name: upstream-proxy-ca + mountPath: /etc/openshell-tls/proxy-ca + readOnly: true + {{- end }} {{- if $spiffeSocketPath }} - name: spiffe-workload-api mountPath: {{ dir $spiffeSocketPath | quote }} @@ -222,6 +227,14 @@ spec: - key: ca.crt path: ca.crt {{- end }} + {{- if .Values.upstreamProxy.caBundle.configMapName }} + - name: upstream-proxy-ca + configMap: + name: {{ .Values.upstreamProxy.caBundle.configMapName | quote }} + items: + - key: {{ .Values.upstreamProxy.caBundle.key | default "ca.crt" | quote }} + path: ca.crt + {{- end }} {{- if $spiffeSocketPath }} - name: spiffe-workload-api csi: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 33120137c7..1736c9b974 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -206,7 +206,7 @@ defaults. {{- if .Values.server.disableTls -}} {{- else if not .Values.server.tls.enableMtls -}} {{- else if eq .Values.server.tls.clientCaSecretName "" -}} -{{- else if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) -}} +{{- else -}} true {{- end -}} {{- end -}} @@ -410,6 +410,75 @@ passes through Helm values into gateway.toml; only this reference is rendered. {{- end -}} {{- end }} +{{/* +Return the effective Kubernetes driver configuration as YAML. Schema-v2 +gatewayConfig fields take precedence; deprecated 0.1.x aliases fill only +absent fields so every chart consumer observes the same configuration. +*/}} +{{- define "openshell.effectiveKubernetesConfig" -}} +{{- $gatewayConfig := deepCopy (.Values.gatewayConfig | default dict) -}} +{{- $kubernetes := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $legacySandbox := .Values.sandboxRuntime.image | default dict -}} +{{- if and (include "openshell.sandboxRuntimeImageOverrideEnabled" .) (not (hasKey $kubernetes "sandbox_runtime_image")) -}} +{{- $_ := set $kubernetes "sandbox_runtime_image" (include "openshell.sandboxRuntimeImage" .) -}} +{{- end -}} +{{- if and (get $legacySandbox "pullPolicy") (not (hasKey $kubernetes "sandbox_runtime_image_pull_policy")) -}} +{{- $_ := set $kubernetes "sandbox_runtime_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $legacySandbox "pullPolicy")) -}} +{{- end -}} +{{- $legacySupervisor := .Values.supervisor.image | default dict -}} +{{- $legacySupervisorImage := include "openshell.supervisorImage" . -}} +{{- $defaultSupervisorImage := printf "ghcr.io/nvidia/openshell/supervisor:%s" .Chart.AppVersion -}} +{{- if and (ne $legacySupervisorImage $defaultSupervisorImage) (not (hasKey $kubernetes "supervisor_image")) -}} +{{- $_ := set $kubernetes "supervisor_image" $legacySupervisorImage -}} +{{- end -}} +{{- if and (get $legacySupervisor "pullPolicy") (not (hasKey $kubernetes "supervisor_image_pull_policy")) -}} +{{- $_ := set $kubernetes "supervisor_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $legacySupervisor "pullPolicy")) -}} +{{- end -}} +{{- $legacyRuntime := .Values.supervisor.sandboxRuntime | default dict -}} +{{- $runtimeConfig := get $kubernetes "sandbox_runtime" | default dict -}} +{{- if and (get $legacyRuntime "networkPolicyEnforced") (not (hasKey $runtimeConfig "network_policy_enforced")) -}} +{{- $_ := set $runtimeConfig "network_policy_enforced" true -}} +{{- end -}} +{{- if and (ne (int (get $legacyRuntime "boundaryPort" | default 5500)) 5500) (not (hasKey $runtimeConfig "boundary_port")) -}} +{{- $_ := set $runtimeConfig "boundary_port" (int (get $legacyRuntime "boundaryPort")) -}} +{{- end -}} +{{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} +{{- $legacyProxy := .Values.upstreamProxy | default dict -}} +{{- range $legacyKey, $runtimeKey := dict "url" "https_proxy" "noProxy" "no_proxy" "authAllowInsecure" "proxy_auth_allow_insecure" "connectByHostname" "proxy_connect_by_hostname" -}} +{{- if and (get $legacyProxy $legacyKey) (not (hasKey $kubernetes $runtimeKey)) -}} +{{- $_ := set $kubernetes $runtimeKey (get $legacyProxy $legacyKey) -}} +{{- end -}} +{{- end -}} +{{- $legacyProxyAuth := get $legacyProxy "authSecret" | default dict -}} +{{- if and (get $legacyProxyAuth "name") (not (hasKey $kubernetes "proxy_auth_secret_name")) -}}{{- $_ := set $kubernetes "proxy_auth_secret_name" (get $legacyProxyAuth "name") -}}{{- end -}} +{{- if and (get $legacyProxyAuth "key") (not (hasKey $kubernetes "proxy_auth_secret_key")) -}}{{- $_ := set $kubernetes "proxy_auth_secret_key" (get $legacyProxyAuth "key") -}}{{- end -}} +{{- $legacyProxyCa := get $legacyProxy "caBundle" | default dict -}} +{{- if and (get $legacyProxyCa "configMapName") (not (hasKey $kubernetes "proxy_ca_bundle")) -}} +{{- $_ := set $kubernetes "proxy_ca_bundle" "/etc/openshell-tls/proxy-ca/ca.crt" -}} +{{- end -}} +{{- $legacyKubernetes := .Values.server.drivers.kubernetes | default dict -}} +{{- if and (ne (get $legacyKubernetes "workspaceMode" | default "shared") "shared") (not (hasKey $kubernetes "workspace_mode")) -}} +{{- $_ := set $kubernetes "workspace_mode" (get $legacyKubernetes "workspaceMode") -}} +{{- end -}} +{{- range $legacyKey, $runtimeKey := dict "operatorNamespaceLabel" "operator_namespace_label" "operatorNamespaceFile" "operator_namespace_file" -}} +{{- if and (get $legacyKubernetes $legacyKey) (not (hasKey $kubernetes $runtimeKey)) -}} +{{- $_ := set $kubernetes $runtimeKey (get $legacyKubernetes $legacyKey) -}} +{{- end -}} +{{- end -}} +{{- if and (get $legacyKubernetes "allowDriverConfig") (not (hasKey $kubernetes "allow_driver_config")) -}} +{{- $_ := set $kubernetes "allow_driver_config" true -}} +{{- end -}} +{{/* Keep the rendered default configuration stable without making defaults look +like user-supplied schema-v2 fields during compatibility resolution. */}} +{{- if not (hasKey $kubernetes "workspace_mode") -}}{{- $_ := set $kubernetes "workspace_mode" "shared" -}}{{- end -}} +{{- if not (hasKey $kubernetes "sandbox_runtime_image") -}}{{- $_ := set $kubernetes "sandbox_runtime_image" (printf "ghcr.io/nvidia/openshell/sandbox:%s" .Chart.AppVersion) -}}{{- end -}} +{{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (printf "ghcr.io/nvidia/openshell/supervisor:%s" .Chart.AppVersion) -}}{{- end -}} +{{- if not (hasKey $runtimeConfig "network_policy_enforced") -}}{{- $_ := set $runtimeConfig "network_policy_enforced" false -}}{{- end -}} +{{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} +{{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} +{{- toYaml $kubernetes -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} @@ -440,8 +509,8 @@ Validate chart values that Helm would otherwise accept silently. {{- include "openshell.validateSecretReference" (list "server.sandboxJwt.signingSecretName" .Values.server.sandboxJwt.signingSecretName) -}} {{- include "openshell.validateSecretReference" (list "server.tls.certSecretName" .Values.server.tls.certSecretName) -}} {{- include "openshell.validateSecretReference" (list "upstreamProxy.authSecret.name" .Values.upstreamProxy.authSecret.name) -}} -{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} +{{- include "openshell.validateSecretReference" (list "gatewayConfig.openshell.drivers.kubernetes.proxy_auth_secret_name" (get $kubernetesConfig "proxy_auth_secret_name")) -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} {{- fail "gatewayConfig.openshell.drivers.kubernetes.workspace_mode must be one of: shared, managed, operator." -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 69cb073449..4205729b6a 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -101,6 +101,8 @@ field must not require a Helm template change. {{- define "openshell.gatewayConfigToml" -}} {{- $root := . -}} {{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} +{{- $kubernetesCompat := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetesCompat -}} {{/* External credential drivers own their storage. Do not configure the chart-managed encrypted database store when any driver is selected: its KEK environment variable is intentionally not mounted in that mode. */}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 92005b129b..4f2afa7368 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 {{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" }} +{{- $allowDriverConfig := get $kubernetesConfig "allow_driver_config" | default false -}} {{- $managedSshIngress := get $gatewayConfig "openshell.drivers.kubernetes.managed_ssh_ingress" | default dict -}} {{- $managedSshIngressEnabled := true -}} {{- if hasKey $managedSshIngress "enabled" -}} @@ -23,7 +24,7 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get"] - {{- if and (ne $workspaceMode "shared") .Values.server.drivers.kubernetes.allowDriverConfig }} + {{- if and (ne $workspaceMode "shared") $allowDriverConfig }} - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get"] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 1bb055d796..3bc7292da0 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,6 +1,6 @@ -{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} +{{- $allowDriverConfig := get $kubernetesConfig "allow_driver_config" | default false -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) (include "openshell.rbacCreate" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -12,7 +12,7 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: - {{- if .Values.server.drivers.kubernetes.allowDriverConfig }} + {{- if $allowDriverConfig }} # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] resources: ["persistentvolumeclaims"] diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index 886dafcf2f..a81d8c56c9 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,5 +1,4 @@ -{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) (include "openshell.rbacCreate" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. diff --git a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml index 776b5f0075..25731b53a9 100644 --- a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml @@ -9,6 +9,73 @@ release: namespace: serializer-namespace tests: + - it: gives gatewayConfig precedence over a deprecated proxy alias + set: + upstreamProxy.url: http://legacy-proxy.example:8080 + gatewayConfig: + openshell.drivers.kubernetes: + https_proxy: http://schema-v2-proxy.example:8443 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^https_proxy = "http://schema-v2-proxy\.example:8443"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: legacy-proxy + + - it: gives gatewayConfig precedence over deprecated image, runtime, and CA aliases + set: + sandboxRuntime.image.repository: legacy.example/sandbox + sandboxRuntime.image.tag: legacy + supervisor.image.repository: legacy.example/supervisor + supervisor.image.tag: legacy + supervisor.sandboxRuntime.networkPolicyEnforced: true + supervisor.sandboxRuntime.boundaryPort: 6600 + upstreamProxy.caBundle.configMapName: legacy-proxy-ca + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime_image: schema.example/sandbox:2 + supervisor_image: schema.example/supervisor:2 + sandbox_runtime: + network_policy_enforced: false + boundary_port: 7700 + proxy_ca_bundle: /schema-v2/ca.crt + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^sandbox_runtime_image = "schema\.example/sandbox:2"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^supervisor_image = "schema\.example/supervisor:2"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^proxy_ca_bundle = "/schema-v2/ca\.crt"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^sandbox_runtime = \{ boundary_port = 7700, network_policy_enforced = false \}$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: legacy\.example|6600|proxy-ca + + - it: translates a deprecated workspace mode when schema-v2 keeps its default + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^workspace_mode = "managed"$' + + - it: gives an explicit schema-v2 workspace mode precedence over the deprecated alias + set: + server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: shared + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^workspace_mode = "shared"$' + - it: renders all supported YAML shapes with deterministic TOML output set: gatewayConfig: diff --git a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml index 71d5ea1810..3c38002c83 100644 --- a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml +++ b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml @@ -78,6 +78,16 @@ tests: - failedTemplate: errorMessage: server.credentialStorage.existingSecret must be a valid Kubernetes Secret name + - it: rejects an invalid schema-v2 proxy authentication Secret reference + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + proxy_auth_secret_name: contains spaces + asserts: + - failedTemplate: + errorMessage: gatewayConfig.openshell.drivers.kubernetes.proxy_auth_secret_name must be a valid Kubernetes Secret name + - it: rejects Secret references longer than the Kubernetes limit template: templates/statefulset.yaml set: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 992d840922..596f29b307 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -105,6 +105,22 @@ tests: resources: ["persistentvolumeclaims"] verbs: ["get"] + - it: omits shared-workspace resources for the deprecated managed workspace mode + template: templates/role.yaml + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - hasDocuments: + count: 0 + + - it: omits shared-workspace resources for the deprecated operator workspace mode + template: templates/rolebinding.yaml + set: + server.drivers.kubernetes.workspaceMode: operator + asserts: + - hasDocuments: + count: 0 + - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml set: diff --git a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml index 6b3e320a8d..df6df15693 100644 --- a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml +++ b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml @@ -17,13 +17,13 @@ tests: certManager.enabled: false asserts: - equal: - path: spec.template.spec.volumes[5].name + path: spec.template.spec.volumes[3].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[5].secret.secretName + path: spec.template.spec.volumes[3].secret.secretName value: openshell-server-tls - equal: - path: spec.template.spec.volumes[5].secret.items[0].key + path: spec.template.spec.volumes[3].secret.items[0].key value: ca.crt - it: shares the cert-manager server TLS ca.crt when clientCaFromServerTlsSecret is true @@ -33,13 +33,13 @@ tests: certManager.clientCaFromServerTlsSecret: true asserts: - equal: - path: spec.template.spec.volumes[5].name + path: spec.template.spec.volumes[3].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[5].secret.secretName + path: spec.template.spec.volumes[3].secret.secretName value: openshell-server-tls - equal: - path: spec.template.spec.volumes[5].secret.items[0].key + path: spec.template.spec.volumes[3].secret.items[0].key value: ca.crt # Regression: with cert-manager enabled and pkiInitJob left at its default @@ -56,13 +56,13 @@ tests: server.tls.clientCaSecretName: openshell-ca-tls asserts: - equal: - path: spec.template.spec.volumes[5].name + path: spec.template.spec.volumes[3].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[5].secret.secretName + path: spec.template.spec.volumes[3].secret.secretName value: openshell-ca-tls - notExists: - path: spec.template.spec.volumes[5].secret.items + path: spec.template.spec.volumes[3].secret.items - it: omits client CA volume and mount when mTLS is disabled template: templates/statefulset.yaml @@ -96,7 +96,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 4 + count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -115,7 +115,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 4 + count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -132,7 +132,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 4 + count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 1c1b484976..578ed9d2ed 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -72,6 +72,8 @@ supervisor: sandboxRuntime: # -- Workload boundary TLS listener port. boundaryPort: 5500 + # -- Deprecated acknowledgement retained for 0.1.x values files. + networkPolicyEnforced: false sandbox: image: @@ -290,10 +292,7 @@ gatewayConfig: gateway_id: '{{ include "openshell.fullname" . }}' grpc_endpoint: '{{ include "openshell.grpcEndpoint" . }}' service_account_name: '{{ include "openshell.sandboxServiceAccountName" . }}' - supervisor_sideload_method: '{{ include "openshell.supervisorSideloadMethod" . }}' - topology: '{{ .Values.supervisor.topology }}' sa_token_ttl_secs: 3600 - app_armor_profile: Unconfined openshell.drivers.kubernetes.managed_ssh_ingress: enabled: true gateway_namespace: '{{ .Release.Namespace }}' diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index e23f14c941..d3f286f808 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -45,8 +45,8 @@ render default default_manifest="${work_dir}/default.yaml" default_toml="${work_dir}/default.toml" toml "${default_manifest}" "${default_toml}" -service_name="$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${default_manifest}")" -service_port="$(yq ea -e -r 'select(.kind == "Service") | .spec.ports[] | select(.name == "grpc") | .port' "${default_manifest}")" +service_name="$(yq ea -e -r 'select(.kind == "Service" and .metadata.name == "resource-coherence-openshell") | .metadata.name' "${default_manifest}")" +service_port="$(yq ea -e -r 'select(.kind == "Service" and .metadata.name == "resource-coherence-openshell") | .spec.ports[] | select(.name == "grpc") | .port' "${default_manifest}")" workload_port="$(workload_value "${default_manifest}" '.spec.template.spec.containers[] | select(.name == "openshell-gateway") | .ports[] | select(.name == "grpc") | .containerPort')" config_map="$(yq ea -e -r 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .metadata.name' "${default_manifest}")" mounted_config_map="$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "gateway-config") | .configMap.name')" @@ -74,7 +74,7 @@ route_toml="${work_dir}/openshift-route.toml" toml "${route_manifest}" "${route_toml}" route_service="$(yq ea -e -r 'select(.kind == "Route") | .spec.to.name' "${route_manifest}")" route_port="$(yq ea -e -r 'select(.kind == "Route") | .spec.port.targetPort' "${route_manifest}")" -[[ "${route_service}" == "$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${route_manifest}")" && "${route_port}" == "grpc" ]] +[[ "${route_service}" == "$(yq ea -e -r "select(.kind == \"Service\" and .metadata.name == \"${route_service}\") | .metadata.name" "${route_manifest}")" && "${route_port}" == "grpc" ]] [[ "$(workload_value "${route_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-external-cert") | .secret.secretName')" == "${route_service}-server-external-tls" ]] [[ "$(yq ea -e -r 'select(.kind == "Certificate") | .spec.secretName' "${route_manifest}" | grep -Fx "${route_service}-server-external-tls")" == "${route_service}-server-external-tls" ]] grep -F 'external_cert_path = "/etc/openshell-tls/server-external/tls.crt"' "${route_toml}" >/dev/null From de1451b01ab644c5ca2e8d8b2c806f264addba5e Mon Sep 17 00:00:00 2001 From: Gaizka Menendez Hernandez Date: Wed, 30 Sep 2026 18:51:10 +0100 Subject: [PATCH 29/29] fix(helm): complete legacy gateway config compatibility Signed-off-by: Gaizka Menendez Hernandez --- deploy/helm/openshell/README.md | 74 ++++++------ deploy/helm/openshell/README.md.gotmpl | 2 +- .../ci/values-credential-driver-vault.yaml | 4 +- deploy/helm/openshell/ci/values-keycloak.yaml | 7 +- .../openshell/templates/_gateway-workload.tpl | 79 +++++++++++-- deploy/helm/openshell/templates/_helpers.tpl | 74 +++++++++--- deploy/helm/openshell/templates/_toml.tpl | 108 +++++++++++++++++- .../helm/openshell/templates/clusterrole.yaml | 72 ++---------- .../templates/credential-secrets-role.yaml | 8 +- .../credential-secrets-rolebinding.yaml | 8 +- deploy/helm/openshell/templates/role.yaml | 6 +- .../openshell/tests/clusterrole_test.yaml | 15 +++ .../tests/credential_drivers_test.yaml | 43 ++++++- .../tests/gateway_config_serializer_test.yaml | 86 +++++++++++++- .../openshell/tests/gateway_config_test.yaml | 11 +- .../tests/statefulset_client_ca_test.yaml | 59 ++++------ deploy/helm/openshell/values.yaml | 41 ------- deploy/helm/test-gateway-config-parser.sh | 22 +--- .../helm/test-gateway-resource-coherence.sh | 8 +- docs/how-it-works/gateways/authentication.mdx | 5 +- docs/how-it-works/gateways/configuration.mdx | 2 +- docs/kubernetes/ingress.mdx | 1 + docs/kubernetes/migrate-gateway-config.mdx | 20 ++-- e2e/rust/tests/credential_drivers.rs | 26 ++++- e2e/with-kube-gateway.sh | 1 + tasks/scripts/test-e2e-image-overrides.sh | 21 ++++ 26 files changed, 554 insertions(+), 249 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index d426d16422..9c4d5232e1 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -149,12 +149,7 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell -<<<<<<< HEAD helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -======= -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) ``` ## Install on OpenShift @@ -162,18 +157,14 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ -<<<<<<< HEAD --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null -======= - --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) ``` On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the @@ -231,10 +222,6 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ -<<<<<<< HEAD -======= - --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -264,7 +251,7 @@ gatewayConfig: credential_drivers: - vault openshell.credential_drivers.vault: - address: http://vault.vault.svc.cluster.local:8200 + address: https://vault.vault.svc.cluster.local:8200 mount: secret kv_version: "2" auth_method: kubernetes @@ -289,10 +276,33 @@ namespace to limit access to OpenShell-managed Secrets. Append these flags to any of the PostgreSQL commands above for OpenShift: ``` +--set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` +### High availability + +Set `replicaCount` above `1` only with `server.externalDbSecret`; the default +SQLite database is per pod and cannot coordinate multiple gateway replicas. +The chart creates a headless peer Service for gateway-to-gateway relay traffic. +StatefulSet pods use stable pod DNS names through that headless Service. +Deployment pods advertise their pod IP with `OPENSHELL_PEER_ENDPOINT`, because +Kubernetes does not assign stable per-pod DNS names to Deployment replicas. + +Gateway peer traffic uses Kubernetes ServiceAccount identity. Each gateway pod +mounts a projected, pod-bound ServiceAccount token with audience +`openshell-gateway-peer`; receiving replicas validate that token with the +Kubernetes TokenReview API, verify the live pod UID and Helm selector labels, +and authorize only the internal `PeerRelay` RPC. The chart does not create or +accept a shared gateway peer Secret. + +With gateway TLS enabled, peer calls use the chart CA and client TLS Secret for +server verification and mTLS. The client verifies the stable gateway Service +DNS name while connecting directly to the owning pod. Custom TLS Secrets must +include that Service DNS name in the server certificate and provide the CA and +client credentials configured by `server.tls`. + ## Secret bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` @@ -343,18 +353,15 @@ discovery endpoint or its TLS CA. | certManager.serverIssuerRef | object | `{"group":"","kind":"","name":""}` | Override the issuerRef for the external server Certificate (e.g. a real LetsEncrypt/ACME ClusterIssuer for a publicly-trusted cert on an external hostname). When set, the chart creates a second server certificate from this issuer with only the hostnames in serverDnsNames; the internal server certificate is always signed by the chart's own CA. Leave name empty to use the chart CA for all server certificates (default). Requires certManager.enabled=true. | | credentialDrivers.vault.caConfigMapName | string | `""` | ConfigMap containing the private Vault/OpenBao CA certificate under the ca.crt key. Helm mounts it only when the Vault driver is selected. | | fullnameOverride | string | `""` | Override the full generated resource name. | -<<<<<<< HEAD | gateway.image.digest | string | `""` | Gateway image digest. When set, this takes precedence over tag. | | gateway.image.pullPolicy | string | `nil` | Gateway image pull policy. Empty uses global.image.pullPolicy. | | gateway.image.registry | string | `""` | Gateway image registry. Empty uses global.image.registry. | | gateway.image.repository | string | `"openshell/gateway"` | Gateway image repository. | | gateway.image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | +| gatewayConfig | object | `{"openshell":{"version":2}}` | Non-secret gateway application configuration. Top-level keys name TOML tables and are rendered into the mounted gateway.toml file. Kubernetes resource inputs remain outside this map; template expressions derive the corresponding runtime values from their resource owner. | | global.image.pullPolicy | string | `"IfNotPresent"` | Shared OpenShell image pull policy. Individual image pull policies take precedence. | | global.image.registry | string | `"ghcr.io/nvidia"` | Shared OpenShell image registry. Individual image registries take precedence. | | global.image.tag | string | `""` | Shared OpenShell image tag. Defaults to the chart appVersion when empty. | -======= -| gatewayConfig | object | `{"openshell":{"version":2},"openshell.drivers.kubernetes":{"client_tls_secret_name":"{{ .Values.server.tls.clientTlsSecretName }}","default_image":"ghcr.io/nvidia/openshell-community/sandboxes/base:latest","gateway_id":"{{ include \"openshell.fullname\" . }}","grpc_endpoint":"{{ include \"openshell.grpcEndpoint\" . }}","namespace":"{{ include \"openshell.sandboxNamespace\" . }}","sa_token_ttl_secs":3600,"sandbox_runtime":{"boundary_port":5500,"network_policy_enforced":false},"sandbox_runtime_image":"ghcr.io/nvidia/openshell/sandbox:{{ .Chart.AppVersion }}","service_account_name":"{{ include \"openshell.sandboxServiceAccountName\" . }}","supervisor_image":"ghcr.io/nvidia/openshell/supervisor:{{ .Chart.AppVersion }}","workspace_mode":"shared"},"openshell.drivers.kubernetes.managed_ssh_ingress":{"enabled":true,"gateway_namespace":"{{ .Release.Namespace }}","gateway_pod_selector":{"app.kubernetes.io/instance":"{{ .Release.Name }}","app.kubernetes.io/name":"{{ include \"openshell.name\" . }}"}},"openshell.gateway":{"bind_address":"0.0.0.0:{{ .Values.service.port }}","compute_driver":"kubernetes","enable_loopback_service_http":true,"health_bind_address":"0.0.0.0:{{ .Values.service.healthPort }}","log_level":"info","metrics_bind_address":"0.0.0.0:{{ .Values.service.metricsPort }}","name":"{{ include \"openshell.fullname\" . }}","policy_validation_failure_mode":"fail_closed"},"openshell.gateway.credential_storage":{"key_encryption_key_env":"{{ include \"openshell.credentialStorageKeyEncryptionKeyEnvName\" . }}"},"openshell.gateway.gateway_jwt":{"gateway_id":"{{ include \"openshell.fullname\" . }}","kid_path":"/etc/openshell-jwt/kid","public_key_path":"/etc/openshell-jwt/public.pem","signing_key_path":"/etc/openshell-jwt/signing.pem","ttl_secs":3600},"openshell.gateway.tls":{"cert_path":"/etc/openshell-tls/server/tls.crt","client_ca_path":"/etc/openshell-tls/client-ca/ca.crt","key_path":"/etc/openshell-tls/server/tls.key"}}` | Non-secret gateway application configuration. Top-level keys name TOML tables and are rendered into the mounted gateway.toml file. Kubernetes resource inputs remain outside this map; template expressions derive the corresponding runtime values from their resource owner. | ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | grpcRoute.backendTLSPolicy.caCertificateConfigMapName | string | `""` | Name of the ConfigMap containing the CA certificate (key: ca.crt) used to validate the gateway pod's TLS certificate. Defaults to `-backend-ca` when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds waiting for cert-manager to issue the server certificate, then creates the ConfigMap. A single install usually succeeds; if cert-manager takes longer, increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), the install fails if the timeout is reached; set failOnTimeout=false to allow the install to succeed and run `helm upgrade` after the certificate is issued. | | grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | | grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (`..svc.cluster.local`) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | @@ -372,7 +379,6 @@ discovery endpoint or its TLS CA. | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | | networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | -| oidc.caConfigMapName | string | `""` | | | openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | | openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | @@ -402,7 +408,6 @@ discovery endpoint or its TLS CA. | rbac.create | bool | `true` | Create the RBAC objects that grant the gateway ServiceAccount access. Disable to supply the namespaced sandbox and peer Role/RoleBinding and the cluster-scoped ClusterRole/ClusterRoleBinding out of band. The certgen hook and credential driver RBAC keep their own flags. | | replicaCount | int | `1` | Number of OpenShell gateway replicas. Values greater than 1 require server.externalDbSecret because the default SQLite backend is per pod. | | resources | object | `{}` | Gateway pod resource requests and limits. | -<<<<<<< HEAD | sandbox.image.digest | string | `""` | Sandbox image digest. When set, this takes precedence over tag. | | sandbox.image.pullPolicy | string | `nil` | Sandbox image pull policy. Leave unset to use the Kubernetes image default. | | sandbox.image.repository | string | `"nvcr.io/nvidia/base/ubuntu"` | Default standalone sandbox image repository. | @@ -412,8 +417,6 @@ discovery endpoint or its TLS CA. | sandboxRuntime.image.registry | string | `""` | Sandbox runtime image registry. Empty uses global.image.registry. | | sandboxRuntime.image.repository | string | `"openshell/sandbox"` | Sandbox runtime image repository. | | sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag. Defaults to the chart appVersion when empty. | -======= ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | @@ -421,7 +424,6 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | -<<<<<<< HEAD | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | | server.credentialDrivers.kubernetesSecrets.createNamespace | bool | `false` | Create the credential namespace. Requires a namespace other than the release namespace. The Namespace is retained on uninstall so stored credentials survive; an existing Namespace not owned by this release is left untouched. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | @@ -485,21 +487,17 @@ discovery endpoint or its TLS CA. | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | -======= -| server.credentialStorage.existingSecret | string | `""` | Name of a pre-existing Secret containing the key-encryption key. When set, the chart does NOT generate a new Secret; it references this one instead. The Secret must contain a key named "key-encryption-key" with a base64-encoded 32-byte value. Required for GitOps workflows that render manifests with `helm template` (where `lookup` is unavailable). | -| server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | -| server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | -| server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | -| server.hostGatewayIP | string | `""` | Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get hostAliases entries mapping host.docker.internal and host.openshell.internal to this IP, allowing them to reach services running on the Docker host. Auto-detected by the cluster entrypoint script. | ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | server.sandboxJwt.secretDefaultMode | string | `""` | File mode for the mounted JWT signing key Secret. Default 0400 (owner-read only). Override to 0440 or 0444 if the container UID does not match the volume file owner. | | server.sandboxJwt.signingSecretName | string | `""` | Name of the Opaque Secret holding the signing key material. Empty falls back to the chart fullname with "-jwt-keys" appended. | +| server.sandboxJwt.ttlSecs | int | `3600` | Token TTL in seconds. Defaults to 3600 (1h). | | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | | server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to disable client certificate verification for HTTPS-only mode. | | server.tls.clientTlsSecretName | string | `"openshell-client-tls"` | K8s secret mounted into sandbox pods for mTLS to the server. | | server.tls.enableMtls | bool | `true` | Enable mTLS client certificate authentication. When false, the gateway runs HTTPS-only without requiring client certificates (use OIDC for auth instead). Must be false when using BackendTLSPolicy because ingress proxies cannot present client certificates to the backend. | +| server.workspaceDefaultStorageSize | string | `""` | Default storage size for the workspace PVC in sandbox pods. Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). Empty = built-in default (2Gi). | +| server.workspaceStorageClass | string | `""` | Kubernetes StorageClass for the workspace PVC in sandbox pods. Empty (default) = omit storageClassName, using the cluster's default StorageClass. Set this on clusters with no default StorageClass, otherwise the workspace PVC stays Pending and the sandbox never starts. | | service.healthPort | int | `8081` | Gateway health service port. | | service.metricsPort | int | `9090` | Gateway metrics service port. | | service.port | int | `8080` | Gateway gRPC/HTTP service port. | @@ -507,16 +505,22 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -<<<<<<< HEAD | supervisor.image.digest | string | `""` | Supervisor image digest. When set, this takes precedence over tag. | | supervisor.image.pullPolicy | string | `nil` | Supervisor image pull policy. Empty uses global.image.pullPolicy. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.registry | string | `""` | Supervisor image registry. Empty uses global.image.registry. | | supervisor.image.repository | string | `"openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | -======= ->>>>>>> 1bb9ee2c9 (fix(helm): complete gateway config migration) | tolerations | list | `[]` | Tolerations for the gateway pod. | +| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"caBundle":{"configMapName":"","key":"ca.crt"},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | +| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | +| upstreamProxy.authSecret.key | string | `""` | Secret key containing the proxy credential. | +| upstreamProxy.authSecret.name | string | `""` | Existing Secret in the sandbox namespace containing a user:pass value. | +| upstreamProxy.caBundle.configMapName | string | `""` | ConfigMap in the release namespace holding the corporate proxy CA bundle. Required for an https:// proxy with a private CA, and for a TLS-intercepting proxy that re-signs upstream certificates. The gateway reads it and stages it into each sandbox's immutable supervisor bootstrap Secret, so the anchor stays in the gateway's trust domain rather than the workload namespace. Supply only the CA that signs your proxy's certificate, or that the proxy re-signs intercepted upstream certificates with. Public roots already come from the supervisor image and its TLS stack, so a full merged trust bundle (for example an OpenShift config.openshift.io/inject-trusted-cabundle ConfigMap) adds hundreds of kilobytes of duplicated roots and can exceed the sandbox boundary's control-frame budget. | +| upstreamProxy.caBundle.key | string | `"ca.crt"` | Key inside that ConfigMap. Change this only to reuse an existing ConfigMap whose key is not ca.crt. | +| upstreamProxy.connectByHostname | bool | `false` | Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. | +| upstreamProxy.noProxy | string | `""` | Comma-separated destinations that bypass only the corporate proxy. | +| upstreamProxy.url | string | `""` | Proxy URL in http://host:port or https://host:port form. An https:// proxy whose certificate is not publicly trusted also needs caBundle below. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | | workspaceResources.enabled | bool | `true` | Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy from this chart. Disable for a gateway-only release. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 127c824c9b..d0252a193e 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -252,7 +252,7 @@ gatewayConfig: credential_drivers: - vault openshell.credential_drivers.vault: - address: http://vault.vault.svc.cluster.local:8200 + address: https://vault.vault.svc.cluster.local:8200 mount: secret kv_version: "2" auth_method: kubernetes diff --git a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml index 83a891dd2a..b9fe45f69a 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml @@ -16,7 +16,9 @@ gatewayConfig: credential_drivers: - vault openshell.credential_drivers.vault: - address: https://openbao.openbao.svc.cluster.local:8200 + # The local E2E fixture exposes this alias in the gateway namespace. It + # also matches the DNS SAN in OpenBao's development TLS certificate. + address: https://openbao-0:8200 role: openshell-gateway credentialDrivers: diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index 4c76a8f841..39b56d2de8 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -25,7 +25,6 @@ gatewayConfig: openshell.gateway.oidc: # Must match KC_HOSTNAME set by keycloak:k8s:setup (in-cluster service hostname). issuer: "https://keycloak.keycloak.svc.cluster.local:443/realms/openshell" - caConfigMapName: "openshell-keycloak-ca" # Must match the client ID in the imported realm (openshell-cli). audience: "openshell-cli" # Short TTL for dev so JWKS key rotation is picked up quickly. @@ -36,3 +35,9 @@ gatewayConfig: # Leave both empty for authentication-only mode (any valid token is accepted). admin_role: "openshell-admin" user_role: "openshell-user" + +# This is a Kubernetes ConfigMap mounted by the chart, not a gateway TOML +# field, so it remains a chart-owned resource reference. +server: + oidc: + caConfigMapName: "openshell-keycloak-ca" diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 124789ac90..12120a89ce 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -6,15 +6,17 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. */}} {{- define "openshell.gatewayPodTemplate" -}} {{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $gatewayRuntimeConfig := get $gatewayConfig "openshell.gateway" | default dict -}} {{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} -{{- $kubernetesRuntimeConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- if not (get $oidcRuntimeConfig "issuer") -}}{{- $oidcRuntimeConfig = .Values.server.oidc | default dict -}}{{- end -}} +{{- $kubernetesRuntimeConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $spiffeSocketPath := get $kubernetesRuntimeConfig "provider_spiffe_workload_api_socket_path" -}} {{- $hasExternalCredentialDriver := or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") -}} {{- $vaultCredentialDriverEnabled := eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true" -}} {{- $credentialDrivers := .Values.credentialDrivers | default dict -}} {{- $vaultResources := get $credentialDrivers "vault" | default dict -}} -{{- $vaultCaConfigMapName := get $vaultResources "caConfigMapName" -}} +{{- $legacyCredentialDrivers := .Values.server.credentialDrivers | default dict -}} +{{- $legacyVaultResources := get $legacyCredentialDrivers "vault" | default dict -}} +{{- $vaultCaConfigMapName := get $vaultResources "caConfigMapName" | default (get $legacyVaultResources "caConfigMapName") -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -62,6 +64,50 @@ spec: - {{ .Values.server.dbUrl | quote }} {{- end }} env: + - name: OPENSHELL_REPLICA_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPENSHELL_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPENSHELL_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if eq (include "openshell.workloadKind" .) "deployment" }} + - name: OPENSHELL_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPENSHELL_PEER_ENDPOINT + value: {{ printf "%s://$(OPENSHELL_POD_IP):%d" (ternary "http" "https" (default false .Values.server.disableTls)) (int .Values.service.port) | quote }} + {{- end }} + - name: OPENSHELL_SERVICE_ACCOUNT_NAME + value: {{ include "openshell.serviceAccountName" . | quote }} + - name: OPENSHELL_PEER_SERVICE_NAME + value: {{ include "openshell.peerServiceName" . | quote }} + - name: OPENSHELL_PEER_TOKEN_AUDIENCE + value: "openshell-gateway-peer" + - name: OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE + value: /var/run/secrets/openshell-peer/token + - name: OPENSHELL_PEER_POD_LABELS + value: {{ printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "openshell.name" .) .Release.Name | quote }} + {{- if not .Values.server.disableTls }} + - name: OPENSHELL_PEER_TLS_SERVER_NAME + value: {{ printf "%s.%s.svc.cluster.local" (include "openshell.fullname" .) .Release.Namespace | quote }} + {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} + - name: OPENSHELL_PEER_TLS_CA_FILE + value: /etc/openshell-tls/server/ca.crt + {{- end }} + {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: OPENSHELL_PEER_TLS_CERT_FILE + value: /etc/openshell-tls/peer-client/tls.crt + - name: OPENSHELL_PEER_TLS_KEY_FILE + value: /etc/openshell-tls/peer-client/tls.key + {{- end }} + {{- end }} {{- if not $hasExternalCredentialDriver }} - name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }} valueFrom: @@ -80,7 +126,7 @@ spec: # mounted at /etc/openshell/gateway.toml. Secret-bearing settings use # env vars that the TOML references by name. Some process-level # settings consumed by libraries outside gateway code also remain here. - {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} # OIDC issuer custom-CA: rustls/reqwest read SSL_CERT_FILE for # outbound TLS verification. This is a process-level env var # consumed by the TLS stack itself, not by gateway code, so it @@ -109,6 +155,9 @@ spec: - name: sandbox-jwt mountPath: /etc/openshell-jwt readOnly: true + - name: gateway-peer-token + mountPath: /var/run/secrets/openshell-peer + readOnly: true {{- if not .Values.server.disableTls }} - name: tls-cert mountPath: /etc/openshell-tls/server @@ -119,19 +168,22 @@ spec: readOnly: true {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: peer-client-tls + mountPath: /etc/openshell-tls/peer-client + readOnly: true - name: tls-client-ca mountPath: /etc/openshell-tls/client-ca readOnly: true {{- end }} {{- end }} - {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} - name: oidc-ca mountPath: /etc/openshell-tls/oidc-ca readOnly: true {{- end }} {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca - mountPath: /etc/openshell-tls/vault + mountPath: /etc/openshell-tls/vault-ca readOnly: true {{- end }} {{- if .Values.upstreamProxy.caBundle.configMapName }} @@ -192,6 +244,14 @@ spec: secret: secretName: {{ include "openshell.sandboxJwtSecretName" . }} defaultMode: {{ .Values.server.sandboxJwt.secretDefaultMode | default 0400 }} + - name: gateway-peer-token + projected: + defaultMode: 0400 + sources: + - serviceAccountToken: + path: token + audience: openshell-gateway-peer + expirationSeconds: 3600 {{- if not .Values.server.disableTls }} - name: tls-cert secret: @@ -202,6 +262,9 @@ spec: secretName: {{ include "openshell.fullname" . }}-server-external-tls {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: peer-client-tls + secret: + secretName: {{ .Values.server.tls.clientTlsSecretName }} - name: tls-client-ca secret: {{- if or (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }} @@ -214,10 +277,10 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.server.oidc.caConfigMapName }} - name: oidc-ca configMap: - name: {{ .Values.oidc.caConfigMapName }} + name: {{ .Values.server.oidc.caConfigMapName }} {{- end }} {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 1736c9b974..75c08c6aa5 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -267,7 +267,8 @@ Namespace where Kubernetes Secret-backed provider credentials live. {{- define "openshell.credentialKubernetesSecretsNamespace" -}} {{- $gatewayConfig := .Values.gatewayConfig | default dict -}} {{- $config := get $gatewayConfig "openshell.credential_drivers.kubernetes-secrets" | default dict -}} -{{- get $config "namespace" | default .Release.Namespace -}} +{{- $legacy := .Values.server.credentialDrivers.kubernetesSecrets | default dict -}} +{{- get $config "namespace" | default (get $legacy "namespace") | default .Release.Namespace -}} {{- end }} {{/* Whether a credential driver is enabled in the generic gateway config. */}} @@ -276,7 +277,14 @@ Namespace where Kubernetes Secret-backed provider credentials live. {{- $driver := index . 1 -}} {{- $gatewayConfig := $root.Values.gatewayConfig | default dict -}} {{- $gateway := get $gatewayConfig "openshell.gateway" | default dict -}} -{{- if has $driver (get $gateway "credential_drivers" | default list) -}}true{{- end -}} +{{- $configuredDrivers := get $gateway "credential_drivers" -}} +{{- if and (hasKey $gateway "credential_drivers") (ne $configuredDrivers nil) -}} +{{- if has $driver ($configuredDrivers | default list) -}}true{{- end -}} +{{- else if eq $driver "kubernetes-secrets" -}} +{{- if $root.Values.server.credentialDrivers.kubernetesSecrets.enabled -}}true{{- end -}} +{{- else if eq $driver "vault" -}} +{{- if $root.Values.server.credentialDrivers.vault.enabled -}}true{{- end -}} +{{- end -}} {{- end }} {{/* @@ -422,8 +430,14 @@ absent fields so every chart consumer observes the same configuration. {{- if and (include "openshell.sandboxRuntimeImageOverrideEnabled" .) (not (hasKey $kubernetes "sandbox_runtime_image")) -}} {{- $_ := set $kubernetes "sandbox_runtime_image" (include "openshell.sandboxRuntimeImage" .) -}} {{- end -}} -{{- if and (get $legacySandbox "pullPolicy") (not (hasKey $kubernetes "sandbox_runtime_image_pull_policy")) -}} -{{- $_ := set $kubernetes "sandbox_runtime_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $legacySandbox "pullPolicy")) -}} +{{- $legacySandboxImage := .Values.sandbox.image | default dict -}} +{{- $legacySandboxPullPolicy := get $legacySandboxImage "pullPolicy" | default .Values.global.image.pullPolicy -}} +{{- if and $legacySandboxPullPolicy (not (hasKey $kubernetes "image_pull_policy")) -}} +{{- $_ := set $kubernetes "image_pull_policy" (include "openshell.canonicalImagePullPolicy" $legacySandboxPullPolicy) -}} +{{- end -}} +{{- $legacySandboxRuntimePullPolicy := get $legacySandbox "pullPolicy" | default .Values.global.image.pullPolicy -}} +{{- if and $legacySandboxRuntimePullPolicy (not (hasKey $kubernetes "sandbox_runtime_image_pull_policy")) -}} +{{- $_ := set $kubernetes "sandbox_runtime_image_pull_policy" (include "openshell.canonicalImagePullPolicy" $legacySandboxRuntimePullPolicy) -}} {{- end -}} {{- $legacySupervisor := .Values.supervisor.image | default dict -}} {{- $legacySupervisorImage := include "openshell.supervisorImage" . -}} @@ -431,14 +445,12 @@ absent fields so every chart consumer observes the same configuration. {{- if and (ne $legacySupervisorImage $defaultSupervisorImage) (not (hasKey $kubernetes "supervisor_image")) -}} {{- $_ := set $kubernetes "supervisor_image" $legacySupervisorImage -}} {{- end -}} -{{- if and (get $legacySupervisor "pullPolicy") (not (hasKey $kubernetes "supervisor_image_pull_policy")) -}} -{{- $_ := set $kubernetes "supervisor_image_pull_policy" (include "openshell.canonicalImagePullPolicy" (get $legacySupervisor "pullPolicy")) -}} +{{- $legacySupervisorPullPolicy := get $legacySupervisor "pullPolicy" | default .Values.global.image.pullPolicy -}} +{{- if and $legacySupervisorPullPolicy (not (hasKey $kubernetes "supervisor_image_pull_policy")) -}} +{{- $_ := set $kubernetes "supervisor_image_pull_policy" (include "openshell.canonicalImagePullPolicy" $legacySupervisorPullPolicy) -}} {{- end -}} {{- $legacyRuntime := .Values.supervisor.sandboxRuntime | default dict -}} {{- $runtimeConfig := get $kubernetes "sandbox_runtime" | default dict -}} -{{- if and (get $legacyRuntime "networkPolicyEnforced") (not (hasKey $runtimeConfig "network_policy_enforced")) -}} -{{- $_ := set $runtimeConfig "network_policy_enforced" true -}} -{{- end -}} {{- if and (ne (int (get $legacyRuntime "boundaryPort" | default 5500)) 5500) (not (hasKey $runtimeConfig "boundary_port")) -}} {{- $_ := set $runtimeConfig "boundary_port" (int (get $legacyRuntime "boundaryPort")) -}} {{- end -}} @@ -457,6 +469,8 @@ absent fields so every chart consumer observes the same configuration. {{- $_ := set $kubernetes "proxy_ca_bundle" "/etc/openshell-tls/proxy-ca/ca.crt" -}} {{- end -}} {{- $legacyKubernetes := .Values.server.drivers.kubernetes | default dict -}} +{{- $legacyServer := .Values.server | default dict -}} +{{- if not (hasKey $kubernetes "allow_driver_config") -}}{{- $_ := set $kubernetes "allow_driver_config" (get $legacyKubernetes "allowDriverConfig" | default false) -}}{{- end -}} {{- if and (ne (get $legacyKubernetes "workspaceMode" | default "shared") "shared") (not (hasKey $kubernetes "workspace_mode")) -}} {{- $_ := set $kubernetes "workspace_mode" (get $legacyKubernetes "workspaceMode") -}} {{- end -}} @@ -465,15 +479,36 @@ absent fields so every chart consumer observes the same configuration. {{- $_ := set $kubernetes $runtimeKey (get $legacyKubernetes $legacyKey) -}} {{- end -}} {{- end -}} -{{- if and (get $legacyKubernetes "allowDriverConfig") (not (hasKey $kubernetes "allow_driver_config")) -}} -{{- $_ := set $kubernetes "allow_driver_config" true -}} +{{- if not (hasKey $kubernetes "resource_admission") -}} +{{- $legacyAdmission := get $legacyKubernetes "resourceAdmission" | default dict -}} +{{- $admission := dict -}} +{{- if hasKey $legacyAdmission "enabled" -}}{{- $_ := set $admission "enabled" (get $legacyAdmission "enabled") -}}{{- end -}} +{{- if and (hasKey $legacyAdmission "requiredLabels") (ne (get $legacyAdmission "requiredLabels") nil) -}}{{- $_ := set $admission "required_labels" (deepCopy (get $legacyAdmission "requiredLabels")) -}}{{- end -}} +{{- $_ := set $kubernetes "resource_admission" $admission -}} +{{- end -}} +{{- if and (get $legacyServer "enableUserNamespaces") (not (hasKey $kubernetes "enable_user_namespaces")) -}}{{- $_ := set $kubernetes "enable_user_namespaces" true -}}{{- end -}} +{{- if and (get $legacyServer "hostGatewayIP") (not (hasKey $kubernetes "host_gateway_ip")) -}}{{- $_ := set $kubernetes "host_gateway_ip" (get $legacyServer "hostGatewayIP") -}}{{- end -}} +{{- if not (hasKey $kubernetes "namespace") -}}{{- $_ := set $kubernetes "namespace" (include "openshell.sandboxNamespace" .) -}}{{- end -}} +{{- if not (hasKey $kubernetes "default_image") -}}{{- $_ := set $kubernetes "default_image" (include "openshell.sandboxImage" .) -}}{{- end -}} +{{- if not (hasKey $kubernetes "gateway_id") -}}{{- $_ := set $kubernetes "gateway_id" (get (.Values.server.sandboxJwt | default dict) "gatewayId" | default (include "openshell.fullname" .)) -}}{{- end -}} +{{- if not (hasKey $kubernetes "grpc_endpoint") -}}{{- $_ := set $kubernetes "grpc_endpoint" (include "openshell.grpcEndpoint" .) -}}{{- end -}} +{{- if not (hasKey $kubernetes "service_account_name") -}}{{- $_ := set $kubernetes "service_account_name" (include "openshell.sandboxServiceAccountName" .) -}}{{- end -}} +{{- if not (hasKey $kubernetes "sa_token_ttl_secs") -}}{{- $_ := set $kubernetes "sa_token_ttl_secs" (get (.Values.server.sandboxJwt | default dict) "k8sSaTokenTtlSecs" | default 3600) -}}{{- end -}} +{{- if not (hasKey $kubernetes "image_pull_secrets") -}} +{{- $imagePullSecrets := list -}}{{- range (get $legacyServer "sandboxImagePullSecrets" | default list) }}{{- if .name }}{{- $imagePullSecrets = append $imagePullSecrets .name }}{{- end }}{{- end -}} +{{- if $imagePullSecrets }}{{- $_ := set $kubernetes "image_pull_secrets" $imagePullSecrets -}}{{- end -}} +{{- end -}} +{{- range $legacyKey, $runtimeKey := dict "workspaceDefaultStorageSize" "workspace_default_storage_size" "workspaceStorageClass" "workspace_storage_class" "defaultRuntimeClassName" "default_runtime_class_name" -}} +{{- if and (get $legacyServer $legacyKey) (not (hasKey $kubernetes $runtimeKey)) -}}{{- $_ := set $kubernetes $runtimeKey (get $legacyServer $legacyKey) -}}{{- end -}} +{{- end -}} +{{- $legacySpiffe := .Values.server.providerTokenGrants.spiffe | default dict -}} +{{- if and (get $legacySpiffe "enabled") (not (hasKey $kubernetes "provider_spiffe_workload_api_socket_path")) -}} +{{- $_ := set $kubernetes "provider_spiffe_workload_api_socket_path" (get $legacySpiffe "workloadApiSocketPath") -}} {{- end -}} {{/* Keep the rendered default configuration stable without making defaults look like user-supplied schema-v2 fields during compatibility resolution. */}} {{- if not (hasKey $kubernetes "workspace_mode") -}}{{- $_ := set $kubernetes "workspace_mode" "shared" -}}{{- end -}} -{{- if not (hasKey $kubernetes "sandbox_runtime_image") -}}{{- $_ := set $kubernetes "sandbox_runtime_image" (printf "ghcr.io/nvidia/openshell/sandbox:%s" .Chart.AppVersion) -}}{{- end -}} -{{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (printf "ghcr.io/nvidia/openshell/supervisor:%s" .Chart.AppVersion) -}}{{- end -}} -{{- if not (hasKey $runtimeConfig "network_policy_enforced") -}}{{- $_ := set $runtimeConfig "network_policy_enforced" false -}}{{- end -}} +{{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (include "openshell.supervisorImage" .) -}}{{- end -}} {{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} {{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} {{- toYaml $kubernetes -}} @@ -509,6 +544,17 @@ Validate chart values that Helm would otherwise accept silently. {{- include "openshell.validateSecretReference" (list "server.sandboxJwt.signingSecretName" .Values.server.sandboxJwt.signingSecretName) -}} {{- include "openshell.validateSecretReference" (list "server.tls.certSecretName" .Values.server.tls.certSecretName) -}} {{- include "openshell.validateSecretReference" (list "upstreamProxy.authSecret.name" .Values.upstreamProxy.authSecret.name) -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $gateway := get $gatewayConfig "openshell.gateway" | default dict -}} +{{- $credentialDrivers := get $gateway "credential_drivers" -}} +{{- if and (hasKey $gateway "credential_drivers") (ne $credentialDrivers nil) -}} +{{- if eq (len $credentialDrivers) 0 -}} +{{- fail "gatewayConfig.openshell.gateway.credential_drivers must select exactly one backend or be omitted/null to use the chart default" -}} +{{- end -}} +{{- if gt (len $credentialDrivers) 1 -}} +{{- fail "gatewayConfig.openshell.gateway.credential_drivers may select only one backend" -}} +{{- end -}} +{{- end -}} {{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- include "openshell.validateSecretReference" (list "gatewayConfig.openshell.drivers.kubernetes.proxy_auth_secret_name" (get $kubernetesConfig "proxy_auth_secret_name")) -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 4205729b6a..b581b3b0b3 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -101,8 +101,108 @@ field must not require a Helm template change. {{- define "openshell.gatewayConfigToml" -}} {{- $root := . -}} {{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} +{{- $legacyServer := .Values.server | default dict -}} +{{- $gateway := get $config "openshell.gateway" | default dict -}} +{{- if not (hasKey $gateway "name") -}}{{- $_ := set $gateway "name" (get $legacyServer "name" | default (include "openshell.fullname" .)) -}}{{- end -}} +{{- if not (hasKey $gateway "bind_address") -}}{{- $_ := set $gateway "bind_address" (printf "0.0.0.0:%v" .Values.service.port) -}}{{- end -}} +{{- if and .Values.service.healthPort (not (hasKey $gateway "health_bind_address")) -}}{{- $_ := set $gateway "health_bind_address" (printf "0.0.0.0:%v" .Values.service.healthPort) -}}{{- end -}} +{{- if and .Values.service.metricsPort (not (hasKey $gateway "metrics_bind_address")) -}}{{- $_ := set $gateway "metrics_bind_address" (printf "0.0.0.0:%v" .Values.service.metricsPort) -}}{{- end -}} +{{- range $legacyKey, $runtimeKey := dict "logLevel" "log_level" "enableLoopbackServiceHttp" "enable_loopback_service_http" "enableWebsocketTunnel" "enable_websocket_tunnel" "policyValidationFailureMode" "policy_validation_failure_mode" -}} +{{- if not (hasKey $gateway $runtimeKey) -}}{{- $_ := set $gateway $runtimeKey (get $legacyServer $legacyKey) -}}{{- end -}} +{{- end -}} +{{- if not (hasKey $gateway "compute_driver") -}}{{- $_ := set $gateway "compute_driver" "kubernetes" -}}{{- end -}} +{{- if and .Values.certManager.enabled .Values.certManager.serverDnsNames (not (hasKey $gateway "server_sans")) -}} +{{- $_ := set $gateway "server_sans" (deepCopy .Values.certManager.serverDnsNames) -}} +{{- end -}} +{{- $_ := set $config "openshell.gateway" $gateway -}} +{{- $gatewayJwt := get $config "openshell.gateway.gateway_jwt" | default dict -}} +{{- $legacyJwt := get $legacyServer "sandboxJwt" | default dict -}} +{{- range $key, $value := dict "signing_key_path" "/etc/openshell-jwt/signing.pem" "public_key_path" "/etc/openshell-jwt/public.pem" "kid_path" "/etc/openshell-jwt/kid" -}} +{{- if not (hasKey $gatewayJwt $key) -}}{{- $_ := set $gatewayJwt $key $value -}}{{- end -}} +{{- end -}} +{{- if not (hasKey $gatewayJwt "gateway_id") -}}{{- $_ := set $gatewayJwt "gateway_id" (get $legacyJwt "gatewayId" | default (include "openshell.fullname" .)) -}}{{- end -}} +{{- if not (hasKey $gatewayJwt "ttl_secs") -}}{{- $_ := set $gatewayJwt "ttl_secs" (get $legacyJwt "ttlSecs" | default 3600) -}}{{- end -}} +{{- $_ := set $config "openshell.gateway.gateway_jwt" $gatewayJwt -}} {{- $kubernetesCompat := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetesCompat -}} +{{- $legacyDrivers := get $legacyServer "drivers" | default dict -}} +{{- $legacyKubernetes := get $legacyDrivers "kubernetes" | default dict -}} +{{- if hasKey $kubernetesCompat "resource_admission" -}} +{{- $_ := set $config "openshell.drivers.kubernetes.resource_admission" (deepCopy (get $kubernetesCompat "resource_admission")) -}} +{{- else -}} +{{- $legacyAdmission := get $legacyKubernetes "resourceAdmission" | default dict -}} +{{- $admission := dict -}} +{{- if hasKey $legacyAdmission "enabled" -}}{{- $_ := set $admission "enabled" (get $legacyAdmission "enabled") -}}{{- end -}} +{{- if and (hasKey $legacyAdmission "requiredLabels") (ne (get $legacyAdmission "requiredLabels") nil) -}}{{- $_ := set $admission "required_labels" (deepCopy (get $legacyAdmission "requiredLabels")) -}}{{- end -}} +{{- $_ := set $config "openshell.drivers.kubernetes.resource_admission" $admission -}} +{{- end -}} +{{- $_ := unset $kubernetesCompat "resource_admission" -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetesCompat -}} +{{- if not (hasKey $config "openshell.drivers.kubernetes.managed_ssh_ingress") -}} +{{- $_ := set $config "openshell.drivers.kubernetes.managed_ssh_ingress" (dict "enabled" .Values.networkPolicy.enabled "gateway_namespace" .Release.Namespace "gateway_pod_selector" (dict "app.kubernetes.io/name" (include "openshell.name" .) "app.kubernetes.io/instance" .Release.Name)) -}} +{{- end -}} +{{- $legacyOidc := get $legacyServer "oidc" | default dict -}} +{{- if and (get $legacyOidc "issuer") (not (hasKey $config "openshell.gateway.oidc")) -}} +{{- $_ := set $config "openshell.gateway.oidc" (dict "issuer" (get $legacyOidc "issuer") "dangerously_allow_insecure_http" (get $legacyOidc "dangerouslyAllowInsecureHttp") "jwks_allowed_origins" (get $legacyOidc "jwksAllowedOrigins") "audience" (get $legacyOidc "audience") "jwks_ttl_secs" (get $legacyOidc "jwksTtl") "roles_claim" (get $legacyOidc "rolesClaim") "admin_role" (get $legacyOidc "adminRole") "user_role" (get $legacyOidc "userRole") "scopes_claim" (get $legacyOidc "scopesClaim")) -}} +{{- end -}} +{{- $legacyOtlp := get $legacyServer "otlp" | default dict -}} +{{- if and (get $legacyOtlp "endpoint") (not (hasKey $config "openshell.gateway.otlp")) -}}{{- $_ := set $config "openshell.gateway.otlp" (dict "endpoint" (get $legacyOtlp "endpoint") "service_name" (get $legacyOtlp "serviceName")) -}}{{- end -}} +{{- $legacyAuth := get $legacyServer "auth" | default dict -}} +{{- if and (get $legacyAuth "allowUnauthenticatedUsers") (not (hasKey $config "openshell.gateway.auth")) -}}{{- $_ := set $config "openshell.gateway.auth" (dict "allow_unauthenticated_users" true) -}}{{- end -}} +{{- $legacyOcsf := get $legacyServer "ocsfLog" | default dict -}} +{{- if and (get $legacyOcsf "enabled") (not (hasKey $config "openshell.gateway.ocsf_log")) -}} +{{- $rotation := get $legacyOcsf "rotation" | default "daily" -}} +{{- if not (has $rotation (list "daily" "never")) -}}{{- fail "server.ocsfLog.rotation must be daily or never" -}}{{- end -}} +{{- $path := get $legacyOcsf "path" -}}{{- if not $path -}}{{- fail "server.ocsfLog.path must be set when server.ocsfLog.enabled is true" -}}{{- end -}} +{{- $queueCapacity := 10000 -}}{{- if and (hasKey $legacyOcsf "queueCapacity") (ne (get $legacyOcsf "queueCapacity") nil) -}}{{- $queueCapacity = get $legacyOcsf "queueCapacity" -}}{{- end -}} +{{- $queueMaxBytes := 16777216 -}}{{- if and (hasKey $legacyOcsf "queueMaxBytes") (ne (get $legacyOcsf "queueMaxBytes") nil) -}}{{- $queueMaxBytes = get $legacyOcsf "queueMaxBytes" -}}{{- end -}} +{{- if or (lt (int $queueCapacity) 1) (lt (int $queueMaxBytes) 1) -}}{{- fail "server.ocsfLog.queueCapacity and queueMaxBytes must be positive" -}}{{- end -}} +{{- $schemaVersion := get $legacyOcsf "schemaVersion" | default "" -}} +{{- if not (has $schemaVersion (list "" "1.1" "1.3")) -}}{{- fail "server.ocsfLog.schemaVersion must be empty, 1.1, or 1.3" -}}{{- end -}} +{{- $ocsfConfig := dict "path" $path "rotation" $rotation "queue_capacity" (int $queueCapacity) "queue_max_bytes" (int $queueMaxBytes) -}} +{{- if $schemaVersion -}}{{- $_ := set $ocsfConfig "schema_version" $schemaVersion -}}{{- end -}} +{{- if eq $rotation "daily" -}}{{- $maxFiles := 7 -}}{{- if and (hasKey $legacyOcsf "maxFiles") (ne (get $legacyOcsf "maxFiles") nil) -}}{{- $maxFiles = get $legacyOcsf "maxFiles" -}}{{- end -}}{{- if lt (int $maxFiles) 1 -}}{{- fail "server.ocsfLog.maxFiles must be positive when rotation is daily" -}}{{- end -}}{{- $_ := set $ocsfConfig "max_files" (int $maxFiles) -}}{{- end -}} +{{- $_ := set $config "openshell.gateway.ocsf_log" $ocsfConfig -}} +{{- end -}} +{{- $legacyRateLimit := get $legacyServer "grpcRateLimit" | default dict -}} +{{- $rateRequests := int (get $legacyRateLimit "requests" | default 0) -}} +{{- $rateWindow := int (get $legacyRateLimit "windowSeconds" | default 0) -}} +{{- if or (lt $rateRequests 0) (lt $rateWindow 0) -}}{{- fail "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" -}}{{- end -}} +{{- if and (gt $rateRequests 0) (gt $rateWindow 0) -}} +{{- if not (hasKey $gateway "grpc_rate_limit_requests") -}}{{- $_ := set $gateway "grpc_rate_limit_requests" $rateRequests -}}{{- end -}} +{{- if not (hasKey $gateway "grpc_rate_limit_window_seconds") -}}{{- $_ := set $gateway "grpc_rate_limit_window_seconds" $rateWindow -}}{{- end -}} +{{- else if or (gt $rateRequests 0) (gt $rateWindow 0) -}}{{- fail "server.grpcRateLimit requires both requests and windowSeconds to be positive to enable rate limiting, or both 0/unset to disable it" -}}{{- end -}} +{{- $_ := set $config "openshell.gateway" $gateway -}} +{{- $legacyCredentialDrivers := .Values.server.credentialDrivers | default dict -}} +{{- $gatewayForCredentials := get $config "openshell.gateway" | default dict -}} +{{- $rawConfiguredCredentialDrivers := get $gatewayForCredentials "credential_drivers" -}} +{{- $hasConfiguredCredentialDrivers := and (hasKey $gatewayForCredentials "credential_drivers") (ne $rawConfiguredCredentialDrivers nil) -}} +{{- $configuredCredentialDrivers := $rawConfiguredCredentialDrivers | default list -}} +{{- if and $hasConfiguredCredentialDrivers (eq (len $configuredCredentialDrivers) 0) -}} +{{- fail "gatewayConfig.openshell.gateway.credential_drivers must select exactly one backend or be omitted/null to use the chart default" -}} +{{- end -}} +{{- if gt (len $configuredCredentialDrivers) 1 -}} +{{- fail "gatewayConfig.openshell.gateway.credential_drivers may select only one backend" -}} +{{- end -}} +{{- $legacyKubernetesSecrets := get $legacyCredentialDrivers "kubernetesSecrets" | default dict -}} +{{- $legacyVault := get $legacyCredentialDrivers "vault" | default dict -}} +{{- if and $legacyKubernetesSecrets.enabled $legacyVault.enabled -}} +{{- fail "only one external server.credentialDrivers backend can be enabled at a time" -}} +{{- end -}} +{{- if and (not $hasConfiguredCredentialDrivers) (eq (len $configuredCredentialDrivers) 0) -}} +{{- if $legacyKubernetesSecrets.enabled -}} +{{- $_ := set $gatewayForCredentials "credential_drivers" (list "kubernetes-secrets") -}} +{{- $_ := set $config "openshell.credential_drivers.kubernetes-secrets" (dict "namespace" (default .Release.Namespace $legacyKubernetesSecrets.namespace)) -}} +{{- else if $legacyVault.enabled -}} +{{- $_ := set $gatewayForCredentials "credential_drivers" (list "vault") -}} +{{- $vaultConfig := dict "address" $legacyVault.address "auth_method" $legacyVault.authMethod "role" $legacyVault.role -}} +{{- range $legacyKey, $runtimeKey := dict "mount" "mount" "kvVersion" "kv_version" "kubernetesAuthMount" "kubernetes_auth_mount" "serviceAccountTokenPath" "service_account_token_path" "tokenPath" "token_path" "timeoutSecs" "timeout_secs" -}} +{{- if get $legacyVault $legacyKey -}}{{- $_ := set $vaultConfig $runtimeKey (get $legacyVault $legacyKey) -}}{{- end -}} +{{- end -}} +{{- $_ := set $config "openshell.credential_drivers.vault" $vaultConfig -}} +{{- end -}} +{{- end -}} +{{- $_ := set $config "openshell.gateway" $gatewayForCredentials -}} {{/* External credential drivers own their storage. Do not configure the chart-managed encrypted database store when any driver is selected: its KEK environment variable is intentionally not mounted in that mode. */}} @@ -110,6 +210,8 @@ environment variable is intentionally not mounted in that mode. */}} {{- $configuredCredentialDrivers := get $configuredGateway "credential_drivers" | default list -}} {{- if gt (len $configuredCredentialDrivers) 0 -}} {{- $_ := unset $config "openshell.gateway.credential_storage" -}} +{{- else if not (hasKey $config "openshell.gateway.credential_storage") -}} +{{- $_ := set $config "openshell.gateway.credential_storage" (dict "key_encryption_key_env" (include "openshell.credentialStorageKeyEncryptionKeyEnvName" .)) -}} {{- end -}} {{/* Kubernetes packaging owns host aliases. Do not permit a second runtime source to make sandbox callback hostnames disagree with the pod spec. */}} @@ -125,11 +227,13 @@ source to make sandbox callback hostnames disagree with the pod spec. */}} path. Derive its mounted path only from the chart-owned ConfigMap reference. */}} {{- $credentialDrivers := .Values.credentialDrivers | default dict -}} {{- $vaultResources := get $credentialDrivers "vault" | default dict -}} +{{- $legacyVaultResources := get $legacyCredentialDrivers "vault" | default dict -}} +{{- $vaultCaConfigMapName := get $vaultResources "caConfigMapName" | default (get $legacyVaultResources "caConfigMapName") -}} {{- if hasKey $config "openshell.credential_drivers.vault" -}} {{- $vaultConfig := get $config "openshell.credential_drivers.vault" | default dict -}} {{- $_ := unset $vaultConfig "ca_bundle" -}} -{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") (get $vaultResources "caConfigMapName") -}} -{{- $_ := set $vaultConfig "ca_bundle" "/etc/openshell-tls/vault/ca.crt" -}} +{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") $vaultCaConfigMapName -}} +{{- $_ := set $vaultConfig "ca_bundle" "/etc/openshell-tls/vault-ca/ca.crt" -}} {{- end -}} {{- $_ := set $config "openshell.credential_drivers.vault" $vaultConfig -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 4f2afa7368..a8f2e6f57c 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -2,15 +2,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} -{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} +{{ $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" }} {{- $allowDriverConfig := get $kubernetesConfig "allow_driver_config" | default false -}} +{{- $managedSshIngressEnabled := .Values.networkPolicy.enabled -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} {{- $managedSshIngress := get $gatewayConfig "openshell.drivers.kubernetes.managed_ssh_ingress" | default dict -}} -{{- $managedSshIngressEnabled := true -}} {{- if hasKey $managedSshIngress "enabled" -}} {{- $managedSshIngressEnabled = get $managedSshIngress "enabled" -}} -{{- end }} +{{- end -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -98,73 +98,20 @@ rules: - patch - watch {{- end }} - {{- $copiedSecretNames := list }} - {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} - {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} - {{- end }} - {{- if eq $workspaceMode "managed" }} - {{- range (get $kubernetesConfig "image_pull_secrets" | default list) }} - {{- if . }} - {{- $copiedSecretNames = append $copiedSecretNames . }} - {{- end }} - {{- end }} -{{- end }} - {{- end }} {{- if ne $workspaceMode "shared" }} - apiGroups: [""] resources: ["services"] verbs: ["create", "get"] - - apiGroups: [""] - resources: ["secrets"] - # Bootstrap Secrets are generation-scoped. Recovery lists them by label and - # deletes stale generations with UID preconditions before retrying creation. - verbs: ["create", "list", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] {{- end }} - {{- $copiedSecretNames = uniq $copiedSecretNames }} - {{- if $copiedSecretNames }} - # Copy only explicitly configured TLS and image-pull Secrets into workspace - # namespaces. Server-side apply authorizes these requests as patch operations. - - apiGroups: - - "" - resources: - - secrets - resourceNames: - {{- range $copiedSecretNames }} - - {{ . | quote }} - {{- end }} - verbs: - - get - - patch - # Server-side apply uses PATCH for an existing Secret, but the API server - # additionally authorizes CREATE when the named Secret does not exist yet. - # Kubernetes RBAC cannot restrict CREATE by resourceNames because create - # authorization happens before the object name is available to RBAC. Keep - # reads and mutations name-restricted above; this broader grant is required - # only to create the initial copy in a gateway-owned managed namespace. - - apiGroups: - - "" - resources: - - secrets - verbs: - - create - {{- end }} - {{- if and (ne $workspaceMode "shared") (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") }} - # The kubernetes-secrets credential driver uses dynamic hashed names in - # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - patch - - delete - {{- end }} {{- if eq $workspaceMode "managed" }} + # Operator-mode namespaces receive Secret permissions from the + # openshell-workspace chart Role instead. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] # ServiceAccount creation in managed namespaces. - apiGroups: - "" @@ -186,3 +133,4 @@ rules: - update {{- end }} {{- end }} +{{- end }} diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index 1fcb97fd90..08dbd7949b 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,4 +1,10 @@ -{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} +{{- $credentialSecrets := .Values.server.credentialDrivers.kubernetesSecrets | default dict -}} +{{- $credentialSecretsRbac := get $credentialSecrets "rbac" | default dict -}} +{{- $createRbac := true -}} +{{- if hasKey $credentialSecretsRbac "create" -}} +{{- $createRbac = get $credentialSecretsRbac "create" -}} +{{- end -}} +{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") $createRbac }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 0df76a9ddc..234704ce40 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,4 +1,10 @@ -{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} +{{- $credentialSecrets := .Values.server.credentialDrivers.kubernetesSecrets | default dict -}} +{{- $credentialSecretsRbac := get $credentialSecrets "rbac" | default dict -}} +{{- $createRbac := true -}} +{{- if hasKey $credentialSecretsRbac "create" -}} +{{- $createRbac = get $credentialSecretsRbac "create" -}} +{{- end -}} +{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") $createRbac }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 3bc7292da0..4097976ce1 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -61,9 +61,9 @@ rules: verbs: ["create", "get"] - apiGroups: [""] resources: ["secrets"] - # Bootstrap Secrets are generation-scoped. Recovery lists them by label and - # deletes stale generations with UID preconditions before retrying creation. - verbs: ["create", "list", "delete"] + # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact + # name. + verbs: ["create", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index d42c9732ab..309b10fc0e 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -79,6 +79,21 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] + - it: gives schema-v2 managed SSH ingress precedence over the legacy alias + set: + server.drivers.kubernetes.workspaceMode: managed + networkPolicy.enabled: true + gatewayConfig: + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: false + asserts: + - notContains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create", "patch", "update"] + - it: omits credential Secret access in managed mode set: server.drivers.kubernetes.workspaceMode: managed diff --git a/deploy/helm/openshell/tests/credential_drivers_test.yaml b/deploy/helm/openshell/tests/credential_drivers_test.yaml index f06d47134f..8c3dec7d3f 100644 --- a/deploy/helm/openshell/tests/credential_drivers_test.yaml +++ b/deploy/helm/openshell/tests/credential_drivers_test.yaml @@ -93,7 +93,7 @@ tests: pattern: 'credential_drivers\s*=\s*\["vault"\]' - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.credential_drivers\.vault\].*?address\s*=\s*"https://vault\.vault\.svc\.cluster\.local:8200".*?ca_bundle\s*=\s*"/etc/openshell-tls/vault-ca/ca\.crt".*?auth_method\s*=\s*"kubernetes".*?role\s*=\s*"openshell-gateway"' + pattern: '(?ms)\[openshell\.credential_drivers\.vault\].*?address\s*=\s*"https://vault\.vault\.svc\.cluster\.local:8200".*?auth_method\s*=\s*"kubernetes".*?ca_bundle\s*=\s*"/etc/openshell-tls/vault-ca/ca\.crt".*?role\s*=\s*"openshell-gateway"' - notMatchRegex: path: data["gateway.toml"] pattern: 'transport\s*=\s*"in_tree"' @@ -112,6 +112,29 @@ tests: - failedTemplate: errorPattern: "only one external server.credentialDrivers backend can be enabled at a time" + - it: rejects multiple schema-v2 credential drivers + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.gateway: + credential_drivers: + - kubernetes-secrets + - vault + asserts: + - failedTemplate: + errorMessage: "gatewayConfig.openshell.gateway.credential_drivers may select only one backend" + + - it: rejects an explicit empty schema-v2 driver list + template: templates/statefulset.yaml + set: + server.credentialDrivers.vault.enabled: true + gatewayConfig: + openshell.gateway: + credential_drivers: [] + asserts: + - failedTemplate: + errorMessage: "gatewayConfig.openshell.gateway.credential_drivers must select exactly one backend or be omitted/null to use the chart default" + - it: mounts the Vault CA ConfigMap into the gateway template: templates/statefulset.yaml set: @@ -177,6 +200,24 @@ tests: path: subjects[0].namespace value: my-namespace + - it: honors the legacy Kubernetes Secrets RBAC opt-out + template: templates/credential-secrets-role.yaml + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.rbac.create: false + asserts: + - hasDocuments: + count: 0 + + - it: omits the Kubernetes Secrets RoleBinding when RBAC is opted out + template: templates/credential-secrets-rolebinding.yaml + set: + server.credentialDrivers.kubernetesSecrets.enabled: true + server.credentialDrivers.kubernetesSecrets.rbac.create: false + asserts: + - hasDocuments: + count: 0 + - it: allows default credential storage on a Deployment with an external database template: templates/deployment.yaml set: diff --git a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml index 25731b53a9..73bbb50033 100644 --- a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml @@ -29,7 +29,6 @@ tests: sandboxRuntime.image.tag: legacy supervisor.image.repository: legacy.example/supervisor supervisor.image.tag: legacy - supervisor.sandboxRuntime.networkPolicyEnforced: true supervisor.sandboxRuntime.boundaryPort: 6600 upstreamProxy.caBundle.configMapName: legacy-proxy-ca gatewayConfig: @@ -37,7 +36,6 @@ tests: sandbox_runtime_image: schema.example/sandbox:2 supervisor_image: schema.example/supervisor:2 sandbox_runtime: - network_policy_enforced: false boundary_port: 7700 proxy_ca_bundle: /schema-v2/ca.crt asserts: @@ -52,11 +50,23 @@ tests: pattern: '(?m)^proxy_ca_bundle = "/schema-v2/ca\.crt"$' - matchRegex: path: data["gateway.toml"] - pattern: '(?m)^sandbox_runtime = \{ boundary_port = 7700, network_policy_enforced = false \}$' + pattern: '(?m)^sandbox_runtime = \{ boundary_port = 7700 \}$' - notMatchRegex: path: data["gateway.toml"] pattern: legacy\.example|6600|proxy-ca + - it: does not serialize the removed NetworkPolicy acknowledgement alias + set: + supervisor.sandboxRuntime.networkPolicyEnforced: true + supervisor.sandboxRuntime.boundaryPort: 6600 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^sandbox_runtime = \{ boundary_port = 6600 \}$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: network_policy_enforced + - it: translates a deprecated workspace mode when schema-v2 keeps its default set: server.drivers.kubernetes.workspaceMode: managed @@ -76,6 +86,76 @@ tests: path: data["gateway.toml"] pattern: '(?m)^workspace_mode = "shared"$' + - it: gives explicit schema-v2 gateway settings precedence over legacy aliases + set: + server.name: legacy-gateway + server.enableWebsocketTunnel: false + gatewayConfig: + openshell.gateway: + name: schema-v2-gateway + enable_websocket_tunnel: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^name = "schema-v2-gateway"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^enable_websocket_tunnel = true$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: legacy-gateway + + - it: translates legacy OIDC roles and scopes into schema-v2 field names + set: + server.oidc.issuer: https://issuer.example/realms/openshell + server.oidc.audience: openshell-cli + server.oidc.rolesClaim: custom.roles + server.oidc.adminRole: platform-admin + server.oidc.userRole: platform-user + server.oidc.scopesClaim: permissions + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.oidc\].*?admin_role = "platform-admin"' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.oidc\].*?roles_claim = "custom\.roles"' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.oidc\].*?scopes_claim = "permissions"' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.oidc\].*?user_role = "platform-user"' + + - it: translates legacy resource admission labels into snake_case + set: + server.drivers.kubernetes.resourceAdmission.requiredLabels: + platform.example.com/approved: "true" + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^required_labels = \{ "platform\.example\.com/approved" = "true" \}$' + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/approved" = "true"' + + - it: gives schema-v2 resource admission precedence over the legacy alias + set: + server.drivers.kubernetes.resourceAdmission.requiredLabels: + platform.example.com/source: legacy + gatewayConfig: + openshell.drivers.kubernetes: + resource_admission: + required_labels: + platform.example.com/source: schema-v2 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/source" = "schema-v2"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/source" = "legacy"' + - it: renders all supported YAML shapes with deterministic TOML output set: gatewayConfig: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 16ccfaee6b..70966f4c15 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -359,7 +359,7 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?path\s*=\s*"/var/openshell/gateway-ocsf\.jsonl".*?schema_version\s*=\s*"1\.3".*?rotation\s*=\s*"daily".*?max_files\s*=\s*14.*?queue_capacity\s*=\s*20000.*?queue_max_bytes\s*=\s*33554432' + pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?max_files\s*=\s*14.*?path\s*=\s*"/var/openshell/gateway-ocsf\.jsonl".*?queue_capacity\s*=\s*20000.*?queue_max_bytes\s*=\s*33554432.*?rotation\s*=\s*"daily".*?schema_version\s*=\s*"1\.3"' - it: omits OCSF retention when rotation is disabled template: templates/gateway-config.yaml @@ -502,7 +502,10 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)supervisor_image_pull_policy\s*=\s*"if_not_present".*?sandbox_runtime_image_pull_policy\s*=\s*"if_not_present"' + pattern: '(?m)^supervisor_image_pull_policy\s*=\s*"if_not_present"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^sandbox_runtime_image_pull_policy\s*=\s*"if_not_present"$' - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml @@ -549,7 +552,7 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?client_tls_secret_name\s*=.*?default_image\s*=.*?enable_user_namespaces\s*=\s*true.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?namespace\s*=\s*"my-namespace".*?sa_token_ttl_secs\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?supervisor_image\s*=' - notMatchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' @@ -1320,7 +1323,7 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: "(?ms)name\\s*=\\s*\\\"openshell\\\".*?\\[openshell\\.drivers\\.kubernetes\\].*?namespace\\s*=\\s*\\\"my-namespace\\\".*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\".*?namespace\\s*=\\s*\\\"my-namespace\\\"" - it: uses an explicit server grpc endpoint verbatim template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml index df6df15693..63b10b53e4 100644 --- a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml +++ b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml @@ -16,15 +16,15 @@ tests: pkiInitJob.enabled: true certManager.enabled: false asserts: - - equal: - path: spec.template.spec.volumes[3].name - value: tls-client-ca - - equal: - path: spec.template.spec.volumes[3].secret.secretName - value: openshell-server-tls - - equal: - path: spec.template.spec.volumes[3].secret.items[0].key - value: ca.crt + - contains: + path: spec.template.spec.volumes + content: + name: tls-client-ca + secret: + secretName: openshell-server-tls + items: + - key: ca.crt + path: ca.crt - it: shares the cert-manager server TLS ca.crt when clientCaFromServerTlsSecret is true template: templates/statefulset.yaml @@ -32,15 +32,15 @@ tests: certManager.enabled: true certManager.clientCaFromServerTlsSecret: true asserts: - - equal: - path: spec.template.spec.volumes[3].name - value: tls-client-ca - - equal: - path: spec.template.spec.volumes[3].secret.secretName - value: openshell-server-tls - - equal: - path: spec.template.spec.volumes[3].secret.items[0].key - value: ca.crt + - contains: + path: spec.template.spec.volumes + content: + name: tls-client-ca + secret: + secretName: openshell-server-tls + items: + - key: ca.crt + path: ca.crt # Regression: with cert-manager enabled and pkiInitJob left at its default # `true`, the client CA condition must honor certManager precedence and NOT @@ -55,14 +55,12 @@ tests: pkiInitJob.enabled: true server.tls.clientCaSecretName: openshell-ca-tls asserts: - - equal: - path: spec.template.spec.volumes[3].name - value: tls-client-ca - - equal: - path: spec.template.spec.volumes[3].secret.secretName - value: openshell-ca-tls - - notExists: - path: spec.template.spec.volumes[3].secret.items + - contains: + path: spec.template.spec.volumes + content: + name: tls-client-ca + secret: + secretName: openshell-ca-tls - it: omits client CA volume and mount when mTLS is disabled template: templates/statefulset.yaml @@ -94,9 +92,6 @@ tests: pkiInitJob.enabled: true server.tls.clientCaSecretName: "" asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -113,9 +108,6 @@ tests: certManager.enabled: false server.tls.clientCaSecretName: "" asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -130,9 +122,6 @@ tests: certManager.clientCaFromServerTlsSecret: true server.tls.clientCaSecretName: "" asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 3 - notContains: path: spec.template.spec.containers[0].volumeMounts content: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 578ed9d2ed..82853337c9 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -72,8 +72,6 @@ supervisor: sandboxRuntime: # -- Workload boundary TLS listener port. boundaryPort: 5500 - # -- Deprecated acknowledgement retained for 0.1.x values files. - networkPolicyEnforced: false sandbox: image: @@ -265,45 +263,6 @@ affinity: {} gatewayConfig: openshell: version: 2 - openshell.gateway: - name: '{{ include "openshell.fullname" . }}' - bind_address: '0.0.0.0:{{ .Values.service.port }}' - health_bind_address: '0.0.0.0:{{ .Values.service.healthPort }}' - metrics_bind_address: '0.0.0.0:{{ .Values.service.metricsPort }}' - log_level: info - compute_driver: kubernetes - enable_loopback_service_http: true - policy_validation_failure_mode: fail_closed - openshell.gateway.gateway_jwt: - signing_key_path: /etc/openshell-jwt/signing.pem - public_key_path: /etc/openshell-jwt/public.pem - kid_path: /etc/openshell-jwt/kid - gateway_id: '{{ include "openshell.fullname" . }}' - ttl_secs: 3600 - openshell.gateway.tls: - cert_path: /etc/openshell-tls/server/tls.crt - key_path: /etc/openshell-tls/server/tls.key - client_ca_path: /etc/openshell-tls/client-ca/ca.crt - openshell.drivers.kubernetes: - namespace: '{{ include "openshell.sandboxNamespace" . }}' - default_image: ghcr.io/nvidia/openshell-community/sandboxes/base:latest - client_tls_secret_name: '{{ .Values.server.tls.clientTlsSecretName }}' - workspace_mode: shared - gateway_id: '{{ include "openshell.fullname" . }}' - grpc_endpoint: '{{ include "openshell.grpcEndpoint" . }}' - service_account_name: '{{ include "openshell.sandboxServiceAccountName" . }}' - sa_token_ttl_secs: 3600 - openshell.drivers.kubernetes.managed_ssh_ingress: - enabled: true - gateway_namespace: '{{ .Release.Namespace }}' - gateway_pod_selector: - app.kubernetes.io/name: '{{ include "openshell.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - openshell.drivers.kubernetes.sidecar: - proxy_uid: 1337 - process_binary_aware_network_policy: true - openshell.gateway.credential_storage: - key_encryption_key_env: '{{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }}' # Server configuration server: diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 0c4268c6e1..fa8cdf4f0f 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -22,7 +22,6 @@ render() { helm template parser-validation "${chart}" \ --namespace parser-namespace \ --set agentSandbox.preflight.enabled=false \ - --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ "$@" >"${output}" } @@ -39,14 +38,7 @@ preflight() { "${gateway_bin}" config preflight --path "${toml}" } -if helm template parser-validation "${chart}" --namespace parser-namespace \ - --set agentSandbox.preflight.enabled=false >"${work_dir}/unacknowledged-default.yaml" 2>"${work_dir}/unacknowledged-default.err"; then - echo "the chart must require explicit NetworkPolicy enforcement acknowledgement" >&2 - exit 1 -fi -grep -F 'gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced must be true' "${work_dir}/unacknowledged-default.err" >/dev/null - -# The runtime default is valid after the required infrastructure acknowledgement. +# The runtime default must render and pass the Rust loader validation. render "${work_dir}/default.yaml" extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" preflight "${work_dir}/default.toml" @@ -63,9 +55,9 @@ if grep -Fq 'omitted_value' "${work_dir}/shapes.toml"; then exit 1 fi -# The loader gives absent and empty credential-driver selection deliberately +# The chart gives absent and empty credential-driver selection deliberately # different meanings: absent retains encrypted storage, while an empty list is -# an invalid and ambiguous external-driver selection. +# invalid before a manifest is rendered. render "${work_dir}/credential-drivers-absent.yaml" \ --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=null' extract_toml "${work_dir}/credential-drivers-absent.yaml" "${work_dir}/credential-drivers-absent.toml" @@ -74,11 +66,9 @@ if grep -Fq 'credential_drivers' "${work_dir}/credential-drivers-absent.toml"; t echo "null credential_drivers must be absent from gateway.toml" >&2 exit 1 fi -render "${work_dir}/credential-drivers-empty.yaml" \ - --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=[]' -extract_toml "${work_dir}/credential-drivers-empty.yaml" "${work_dir}/credential-drivers-empty.toml" -if preflight "${work_dir}/credential-drivers-empty.toml" >"${work_dir}/credential-drivers-empty.err" 2>&1; then - echo "the gateway loader accepted empty credential_drivers" >&2 +if render "${work_dir}/credential-drivers-empty.yaml" \ + --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=[]' >"${work_dir}/credential-drivers-empty.err" 2>&1; then + echo "the chart accepted empty credential_drivers" >&2 exit 1 fi diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index d3f286f808..df68025831 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -16,14 +16,10 @@ render() { helm template resource-coherence "${chart}" \ --namespace resource-namespace \ --set agentSandbox.preflight.enabled=false \ - --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ "$@" >"${work_dir}/${name}.yaml" - if ! awk 'BEGIN { RS="---" } - /^\n?# Source:/ && $0 !~ /\napiVersion:/ && $0 !~ /network-policy-ack\.yaml/ { - print "rendered an empty or invalid Kubernetes document:" $0 > "/dev/stderr" - exit 1 - }' "${work_dir}/${name}.yaml"; then + if invalid_documents="$(yq ea -r 'select(. != null and ((has("apiVersion") | not) or (has("kind") | not))) | .kind // ""' "${work_dir}/${name}.yaml")" && [[ -n "${invalid_documents}" ]]; then + printf 'rendered Kubernetes documents without apiVersion or kind:\n%s\n' "${invalid_documents}" >&2 echo "${name}: rendered an invalid Kubernetes document" >&2 exit 1 fi diff --git a/docs/how-it-works/gateways/authentication.mdx b/docs/how-it-works/gateways/authentication.mdx index 3594a3e82f..6cd1c4f4e3 100644 --- a/docs/how-it-works/gateways/authentication.mdx +++ b/docs/how-it-works/gateways/authentication.mdx @@ -269,7 +269,10 @@ This is transparent to the user. All CLI commands work the same regardless of wh ### Plaintext -When a gateway is deployed without a TLS table in `gatewayConfig`, TLS is disabled entirely. The CLI connects over plain HTTP/2. This mode is intended for local port-forwarding or gateways behind a trusted reverse proxy or tunnel that handles TLS termination externally. +When a gateway is deployed with `disable_tls = true` (set through Helm as +`server.disableTls=true`), TLS is disabled entirely. The CLI connects over plain +HTTP/2. This mode is intended for local port-forwarding or gateways behind a +trusted reverse proxy or tunnel that handles TLS termination externally. Register a plaintext gateway with an explicit `http://` endpoint: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 9c6e1dc976..75d274db4e 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -527,7 +527,7 @@ credential_drivers = ["vault"] [openshell.credential_drivers.vault] address = "https://vault.vault.svc.cluster.local:8200" -ca_bundle = "/etc/openshell-tls/vault/ca.crt" +ca_bundle = "/etc/openshell-tls/vault-ca/ca.crt" mount = "secret" kv_version = "2" auth_method = "kubernetes" diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 54ec3e07eb..c913b2bc21 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -176,6 +176,7 @@ helm upgrade --install openshell \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ --set grpcRoute.backendTLSPolicy.enabled=true \ --values oidc-values.yaml \ + --set server.disableTls=false \ --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx index 741913bbcc..610927ab7d 100644 --- a/docs/kubernetes/migrate-gateway-config.mdx +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -5,13 +5,14 @@ Chart schema v2 replaces application-specific Helm values with one non-secret inline tables. Kubernetes resource references (Secrets, certificates, Services, and mounts) remain chart values. -## Breaking upgrade +## Upgrade compatibility -Remove every application-only `server.*` setting before upgrading. There is no -compatibility translation. Runtime images, proxy settings, and NetworkPolicy -acknowledgement all live in `gatewayConfig`. Kubernetes pull -policy spellings such as `Always` are not aliases: use `always`, -`if_not_present`, or `never`. `gatewayConfig` is strictly a non-secret +The chart accepts the 0.1.x application values as deprecated aliases while +moving to `gatewayConfig`. When both forms set the same runtime field, +`gatewayConfig` wins. Runtime images and proxy settings may therefore be +migrated incrementally. Kubernetes pull policy spellings such as `Always` are +accepted aliases and normalize to `always`, `if_not_present`, or `never`. +`gatewayConfig` is strictly a non-secret configuration contract: do not put passwords, tokens, private keys, `database_url`, or other secret material in it; use Secret-backed environment, file, or volume configuration instead. Helm serializes unknown fields @@ -20,13 +21,13 @@ generically and cannot determine whether an arbitrary string such as and PEM private keys, but is not a general secret scanner. Keep `server.disableTls`, TLS Secret names, `server.hostGatewayIP`, `server.externalDbSecret`, -`server.sandboxNamespace`, `oidc.caConfigMapName`, certificate settings and +`server.sandboxNamespace`, `server.oidc.caConfigMapName`, certificate settings and Service settings: these own Kubernetes resources. The chart derives their runtime counterparts. ## Mapping -| Removed value | New location | +| Deprecated alias | Schema-v2 location | | --- | --- | | `server.name`, `server.logLevel`, `server.enableLoopbackServiceHttp`, `server.policyValidationFailureMode` | `openshell.gateway.{name,log_level,enable_loopback_service_http,policy_validation_failure_mode}` | | `server.grpcRateLimit.{requests,windowSeconds}` | `openshell.gateway.{grpc_rate_limit_requests,grpc_rate_limit_window_seconds}` | @@ -58,9 +59,8 @@ gatewayConfig: image_pull_policy: if_not_present supervisor_image: registry.example/openshell-supervisor:latest sandbox_runtime: - network_policy_enforced: true + boundary_port: 5500 workspace_mode: managed - grpc_rate_limit_requests: null openshell.gateway: grpc_rate_limit_requests: 120 grpc_rate_limit_window_seconds: 60 diff --git a/e2e/rust/tests/credential_drivers.rs b/e2e/rust/tests/credential_drivers.rs index fe59941528..b6de64d973 100644 --- a/e2e/rust/tests/credential_drivers.rs +++ b/e2e/rust/tests/credential_drivers.rs @@ -23,6 +23,17 @@ fn unique_suffix() -> String { format!("{}-{millis}", std::process::id()) } +fn credential_sandbox_name(suffix: &str) -> String { + // The Kubernetes driver prefixes sandbox names when it creates companion + // resources, and therefore limits the original sandbox name to 19 bytes. + // `unique_suffix` is ASCII (PID and epoch milliseconds), so retaining its + // final eight bytes keeps names distinct across normal e2e runs while + // leaving ample room for the driver's prefix. + let timestamp = suffix.rsplit_once('-').map_or(suffix, |(_, timestamp)| timestamp); + let start = timestamp.len().saturating_sub(8); + format!("cs-{}", ×tamp[start..]) +} + fn namespace() -> String { std::env::var("OPENSHELL_E2E_SANDBOX_NAMESPACE").unwrap_or_else(|_| "openshell".to_string()) } @@ -383,9 +394,13 @@ async fn provider_credentials_are_stored_in_configured_backend() { let driver = credential_driver(); let suffix = unique_suffix(); - let driver_slug = driver.replace('-', ""); + let driver_slug = match driver.as_str() { + "kubernetes-secrets" => "k8s", + "vault" => "vault", + _ => "driver", + }; let provider_name = format!("cred-storage-{driver_slug}-{suffix}"); - let sandbox_name = format!("cred-storage-sandbox-{driver_slug}-{suffix}"); + let sandbox_name = credential_sandbox_name(&suffix); let secret_value = format!("example-e2e-{driver_slug}-{suffix}"); delete_provider(&provider_name).await; @@ -411,3 +426,10 @@ async fn provider_credentials_are_stored_in_configured_backend() { .await .expect("credential backend object should be deleted with provider"); } + +#[test] +fn credential_sandbox_name_fits_the_kubernetes_driver_limit() { + let name = credential_sandbox_name("12345-1234567890123456789"); + assert_eq!(name, "cs-23456789"); + assert!(name.len() <= 19); +} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index b6219879ff..471f5123e8 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1241,6 +1241,7 @@ if [ -n "${HOST_GATEWAY_IP}" ]; then fi helm_values_args=(--values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml") +helm_values_args+=(--set-string "global.image.registry=$(e2e_image_reference_registry "${GATEWAY_IMAGE}")") if [ "${OPENSHIFT_DETECTED}" = "1" ]; then echo "OpenShift detected — applying SCC-compatible security context overrides." helm_values_args+=(--values "${ROOT}/deploy/helm/openshell/ci/values-openshift-scc.yaml") diff --git a/tasks/scripts/test-e2e-image-overrides.sh b/tasks/scripts/test-e2e-image-overrides.sh index 83c92d0012..2918e001ea 100755 --- a/tasks/scripts/test-e2e-image-overrides.sh +++ b/tasks/scripts/test-e2e-image-overrides.sh @@ -111,4 +111,25 @@ assert_helm_image_translation "sandbox" \ "registry.example/sandbox@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" \ "registry.example" "sandbox" "" "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +# The certgen hook uses the chart's shared registry fallback. The Kubernetes +# harness must propagate the gateway registry there too, including an empty +# registry for k3d-imported local images. +if grep -Fq 'helm_values_args+=(--set-string "global.image.registry=$(e2e_image_reference_registry "${GATEWAY_IMAGE}")")' "${ROOT}/e2e/with-kube-gateway.sh"; then + : +else + echo "FAIL: Kubernetes Helm installs must propagate the gateway registry to certgen" >&2 + exit 1 +fi + +# The Vault credential-driver overlay must use the fixture alias created in +# the gateway namespace. The alias is covered by OpenBao's development TLS +# certificate; the chart's namespace service is not. +if grep -Fq 'address: https://openbao-0:8200' \ + "${ROOT}/deploy/helm/openshell/ci/values-credential-driver-vault.yaml"; then + : +else + echo "FAIL: Vault credential-driver overlay must use the OpenBao TLS alias" >&2 + exit 1 +fi + echo "E2E image override tests passed."