feat(observability): configure opt-in OpenTelemetry export

charts/grid-operator/templates/crds/gridnetwork.yaml

@@ -287,6 +287,63 @@ spec:
                             maximum: 65535.0
                             minimum: 0.0
                             type: integer
+                          telemetry:
+                            description: |-
+                              Optional OpenTelemetry exporter and sampling settings for this gateway.
+
+                              When present, the generated Praxis configuration opts into telemetry and
+                              adds the `trace_context` filter for outbound W3C header propagation.
+                              Collector authentication must be provided to the gateway Deployment via
+                              `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied
+                              into this `ConfigMap`.
+                            nullable: true
+                            properties:
+                              batchIntervalSecs:
+                                description: Batch exporter interval in seconds. Must be from 1 through 300 when set.
+                                format: uint64
+                                maximum: 300.0
+                                minimum: 1.0
+                                nullable: true
+                                type: integer
+                              batchSize:
+                                description: Maximum spans per export batch. Must be from 1 through 65,536 when set.
+                                format: uint
+                                maximum: 65536.0
+                                minimum: 1.0
+                                nullable: true
+                                type: integer
+                              environment:
+                                description: Deployment environment resource attribute.
+                                nullable: true
+                                pattern: ^.*\S.*$
+                                type: string
+                              otlpEndpoint:
+                                description: |-
+                                  OTLP collector endpoint, for example `http://otel-collector:4317`.
+
+                                  If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the
+                                  gateway Deployment. An empty string has the same meaning as omission.
+                                nullable: true
+                                pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$
+                                type: string
+                              samplingRate:
+                                description: Root trace sampling probability from `0.0` through `1.0`.
+                                format: double
+                                maximum: 1.0
+                                minimum: 0.0
+                                nullable: true
+                                type: number
+                              serviceName:
+                                description: OpenTelemetry `service.name` resource attribute.
+                                nullable: true
+                                pattern: ^.*\S.*$
+                                type: string
+                              serviceVersion:
+                                description: OpenTelemetry `service.version` resource attribute.
+                                nullable: true
+                                pattern: ^.*\S.*$
+                                type: string
+                            type: object
                           tlsCertMountPath:
                             default: /etc/praxis/tls
                             description: |-

charts/praxis-gateway/README.md

@@ -186,7 +186,7 @@ Praxis AI image; these values may advance independently.
 | `image.repository` | string | `ghcr.io/praxis-proxy/ai` | Image repository. |
 | `image.tag` | string | `0.4.0` | Image tag (ignored when `image.digest` is set). |
 | `image.digest` | string | `""` | Immutable digest (sha256:…). When set, tag is ignored. |
-| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider` and `gridServing` need. A repository ending in `/grid-gateway` sets it. |
+| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider`, `gridServing`, and telemetry need. A repository ending in `/grid-gateway` sets it. |
 | `image.pullPolicy` | string | `IfNotPresent` | Image pull policy. |
 | `imagePullSecrets` | list | `[]` | Pull secrets for private registries. |
 | `log.level` | string | `""` | Level for every module, rendered as RUST_LOG on the gateway and overlay-sync: off, error, warn, info, debug, or trace, in any case. Empty leaves RUST_LOG unset, so the binaries use their info default. An `env` entry named RUST_LOG takes precedence on the gateway. |
@@ -206,6 +206,11 @@ Praxis AI image; these values may advance independently.
 | `config.key` | string | `praxis.yaml` | Key in the ConfigMap. |
 | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. |
 | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). |
+| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP/gRPC export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.3, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). |
+| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If configured or generic/trace-specific exporter headers are present, the endpoint must explicitly use `https://`; scheme-less endpoints are rejected. |
+| `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. |
+| `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. |
+| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1 through 300 seconds and maximum batch size from 1 through 65,536 spans. |
 | `gatewayConfig.model` | string | **required** for a consumer without `gridServing` | Model advertised on the routing candidates. |
 | `gatewayConfig.backends` | map | **required** when rendered | Backends keyed by site, each with `endpoint` and optional `healthCheck` and `transport`. A consumer's key is the site it reaches over mutual TLS. A provider's `local` key is its one plaintext backend. The older list of `cluster`, `endpoints` entries still renders. |
 | `gatewayConfig.backends[].site` | string | `localSite` | Grid site the backend serves. A consumer's remote `mutual_tls` backend must name it, and it must differ from `localSite`. Its `transport.sni` defaults to `<site>.grid.internal`. |

charts/praxis-gateway/templates/_gateway-config.tpl

@@ -5,11 +5,44 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf
 {{- $cfg := .Values.gatewayConfig }}
 {{- $provider := eq ($cfg.role | default "consumer") "provider" }}
 {{- $apiKey := and (not $provider) (eq ($cfg.auth.mode | default "none") "api-key") }}
+{{- $telemetry := $cfg.telemetry | default dict }}
+{{- $hasSamplingRate := and (hasKey $telemetry "samplingRate") (ne $telemetry.samplingRate nil) }}
+{{- $hasBatchInterval := and (hasKey $telemetry "batchIntervalSecs") (ne $telemetry.batchIntervalSecs nil) }}
+{{- $hasBatchSize := and (hasKey $telemetry "batchSize") (ne $telemetry.batchSize nil) }}
+{{- $hasTelemetryValues := or $telemetry.otlpEndpoint $hasSamplingRate $telemetry.serviceName $telemetry.serviceVersion $telemetry.environment $hasBatchInterval $hasBatchSize }}
   praxis.yaml: |
     {{- with $cfg.upstreamCA.secretName }}
     runtime:
       upstream_ca_file: {{ printf "%s/%s" $cfg.upstreamCA.mountPath ($cfg.upstreamCA.key | default "ca.crt") | quote }}
     {{- end }}
+    {{- if $telemetry.enabled }}
+    telemetry:
+      {{- if not $hasTelemetryValues }}
+      {}
+      {{- else }}
+      {{- with $telemetry.otlpEndpoint }}
+      otlp_endpoint: {{ . | quote }}
+      {{- end }}
+      {{- if $hasSamplingRate }}
+      sampling_rate: {{ $telemetry.samplingRate }}
+      {{- end }}
+      {{- with $telemetry.serviceName }}
+      service_name: {{ . | quote }}
+      {{- end }}
+      {{- with $telemetry.serviceVersion }}
+      service_version: {{ . | quote }}
+      {{- end }}
+      {{- with $telemetry.environment }}
+      environment: {{ . | quote }}
+      {{- end }}
+      {{- if $hasBatchInterval }}
+      batch_interval_secs: {{ $telemetry.batchIntervalSecs }}
+      {{- end }}
+      {{- if $hasBatchSize }}
+      batch_size: {{ $telemetry.batchSize }}
+      {{- end }}
+      {{- end }}
+    {{- end }}
     admin:
       address: {{ include "praxis-gateway.renderedAdminAddress" . | quote }}
     listeners:
@@ -43,6 +76,9 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf
     filter_chains:
       - name: main
         filters:
+          {{- if $telemetry.enabled }}
+          - filter: trace_context
+          {{- end }}
           {{- if $provider }}
           {{- if ne (($cfg.peerTrust).mode | default "pin") "spiffe" }}
           - filter: peer_identity_trust

charts/praxis-gateway/templates/_helpers.tpl

@@ -132,6 +132,15 @@ Validate image digest format when provided.
 Validate required config ConfigMap name.
 */}}
 {{- define "praxis-gateway.validateConfig" -}}
+{{- $telemetry := .Values.gatewayConfig.telemetry | default dict }}
+{{- if $telemetry.enabled }}
+{{- if ne .Values.image.flavor "grid-gateway" }}
+{{- fail "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway, whose Grid gateway build includes the OTLP and AI routing span features" }}
+{{- end }}
+{{- if not .Values.gatewayConfig.render }}
+{{- fail "gatewayConfig.telemetry.enabled needs gatewayConfig.render true so the exporter settings are written to praxis.yaml" }}
+{{- end }}
+{{- end }}
 {{- if .Values.gatewayConfig.render }}
 {{- $consumer := ne (.Values.gatewayConfig.role | default "consumer") "provider" }}
 {{- if and $consumer (not (.Values.gridServing).enabled) (not (trim (toString .Values.gatewayConfig.model))) }}

charts/praxis-gateway/templates/deployment.yaml

@@ -33,6 +33,10 @@ spec:
       {{- else if eq $configSource "render" }}
       {{- $_ := set $podAnnotations "checksum/config" (include "praxis-gateway.gatewayConfigData" . | sha256sum) }}
       {{- end }}
+      {{- if (.Values.gatewayConfig.telemetry).enabled }}
+      {{- /* Exporter initialization is process-scoped, so changed settings need a restart. */}}
+      {{- $_ := set $podAnnotations "checksum/telemetry" (toJson .Values.gatewayConfig.telemetry | sha256sum) }}
+      {{- end }}
       {{- with $podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}

charts/praxis-gateway/tests/telemetry_test.yaml

@@ -0,0 +1,156 @@
+suite: gateway telemetry
+templates:
+  - templates/gateway-config.yaml
+  - templates/deployment.yaml
+tests:
+  - it: emits telemetry and W3C propagation from explicit settings
+    template: templates/gateway-config.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+      gatewayConfig.telemetry.otlpEndpoint: https://otel-collector.observability:4317
+      gatewayConfig.telemetry.samplingRate: 0.25
+      gatewayConfig.telemetry.serviceName: grid-edge
+      gatewayConfig.telemetry.batchIntervalSecs: 3
+      gatewayConfig.telemetry.batchSize: 32
+      env:
+        - name: OTEL_EXPORTER_OTLP_HEADERS
+          valueFrom:
+            secretKeyRef:
+              name: collector-credentials
+              key: headers
+    asserts:
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "(?s)telemetry:\\n\\s+otlp_endpoint: \\\"https://otel-collector\\.observability:4317\\\".*sampling_rate: 0.25.*service_name: \\\"grid-edge\\\".*batch_interval_secs: 3.*batch_size: 32"
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "(?s)filters:\\n\\s+- filter: trace_context"
+      - equal:
+          path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.name
+          value: collector-credentials
+        template: templates/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.key
+          value: headers
+        template: templates/deployment.yaml
+      - exists:
+          path: spec.template.metadata.annotations["checksum/telemetry"]
+        template: templates/deployment.yaml
+
+  - it: accepts the lower sampling boundary
+    template: templates/gateway-config.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+      gatewayConfig.telemetry.samplingRate: 0.0
+    asserts:
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "sampling_rate: 0"
+
+  - it: accepts the upper sampling boundary
+    template: templates/gateway-config.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+      gatewayConfig.telemetry.samplingRate: 1.0
+    asserts:
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "sampling_rate: 1"
+
+  - it: renders an empty telemetry mapping for environment-only settings
+    template: templates/gateway-config.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+    asserts:
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "(?s)telemetry:\\n\\s+\\{\\}"
+
+  - it: treats an empty endpoint as environment fallback
+    template: templates/gateway-config.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+      gatewayConfig.telemetry.otlpEndpoint: ""
+    asserts:
+      - matchRegex:
+          path: data["praxis.yaml"]
+          pattern: "(?s)telemetry:\\n\\s+\\{\\}"
+      - notMatchRegex:
+          path: data["praxis.yaml"]
+          pattern: "otlp_endpoint:"
+
+  - it: requires a Grid gateway image when telemetry is enabled
+    template: templates/deployment.yaml
+    set:
+      gatewayConfig.render: true
+      gatewayConfig.localSite: edge
+      gatewayConfig.model: test-model
+      gatewayConfig.auth.mode: none
+      gatewayConfig.backends:
+        site-a:
+          endpoint: 203.0.113.20:8080
+          transport:
+            mode: plaintext
+      gatewayConfig.telemetry.enabled: true
+    asserts:
+      - failedTemplate:
+          errorPattern: "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway"
+
+  - it: requires generated Praxis config before enabling telemetry
+    template: templates/deployment.yaml
+    set:
+      image.flavor: grid-gateway
+      gatewayConfig.telemetry.enabled: true
+    asserts:
+      - failedTemplate:
+          errorPattern: "gatewayConfig.telemetry.enabled needs gatewayConfig.render true"

charts/praxis-gateway/values.schema.json

@@ -176,6 +176,21 @@
             "existingSecret": { "type": "string" },
             "mountPath": { "type": "string" }
           }
+        },
+        "telemetry": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Optional exporter settings for generated praxis.yaml. Collector credentials must come from deployment-managed Secret environment references.",
+          "properties": {
+            "enabled": { "type": "boolean", "description": "Enable OTLP export and the trace_context propagation filter." },
+            "otlpEndpoint": { "type": "string", "pattern": "^(|https?://[^/?#@\\s]+(:[0-9]+)?(/[^\\s?#]*)?)$", "description": "OTLP/gRPC endpoint without userinfo, query, or fragment credentials. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT." },
+            "samplingRate": { "type": ["number", "null"], "minimum": 0, "maximum": 1, "description": "Root trace sampling probability from 0.0 through 1.0." },
+            "serviceName": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.name resource attribute." },
+            "serviceVersion": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.version resource attribute." },
+            "environment": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "deployment.environment resource attribute." },
+            "batchIntervalSecs": { "type": ["integer", "null"], "minimum": 1, "maximum": 300, "description": "Positive batch exporter interval in seconds." },
+            "batchSize": { "type": ["integer", "null"], "minimum": 1, "maximum": 65536, "description": "Positive maximum spans per export batch." }
+          }
         }
       }
     },

charts/praxis-gateway/values.yaml

@@ -132,6 +132,26 @@ gatewayConfig:
   # when config.existingConfigMap is empty and backends, role provider, or gridServing is set.
   # Changing the rendered config rolls the pods.
   render: false
+  # -- Opt in to OTLP tracing in the generated praxis.yaml. Requires
+  # image.flavor: grid-gateway, whose Grid image build includes the Praxis OTLP
+  # exporter and Praxis AI routing-span features. Credentials belong in env as
+  # Secret-backed OTEL_EXPORTER_OTLP_HEADERS, never here.
+  telemetry:
+    enabled: false
+    # -- OTLP/gRPC endpoint. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT from env.
+    otlpEndpoint: ""
+    # -- Root sampling probability in the inclusive range 0.0..1.0.
+    samplingRate: null
+    # -- OpenTelemetry service.name resource attribute.
+    serviceName: ""
+    # -- OpenTelemetry service.version resource attribute.
+    serviceVersion: ""
+    # -- deployment.environment resource attribute.
+    environment: ""
+    # -- Batch export interval in seconds; must be positive.
+    batchIntervalSecs: null
+    # -- Maximum spans per export batch; must be positive.
+    batchSize: null
   # -- consumer routes callers. provider serves grid peers on the grid identity and forwards to one backend.
   role: consumer
   # -- Grid peers a provider accepts, each with a Grid-CA client certificate.