From e1b50ddb8ca9e60bad77b4bc2f495c145de04022 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 03:56:34 +0000 Subject: [PATCH 1/9] feat(observability): configure gateway OpenTelemetry tracing Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 55 ++++ charts/praxis-gateway/README.md | 7 +- .../templates/_gateway-config.tpl | 36 +++ charts/praxis-gateway/templates/_helpers.tpl | 9 + .../praxis-gateway/templates/deployment.yaml | 4 + .../praxis-gateway/tests/telemetry_test.yaml | 133 ++++++++++ charts/praxis-gateway/values.schema.json | 15 ++ charts/praxis-gateway/values.yaml | 20 ++ deploy/crds/gridnetwork.yaml | 55 ++++ docs/README.md | 2 + docs/architecture/consumer-config.md | 8 + docs/architecture/opentelemetry.md | 135 ++++++++++ gateway/Cargo.toml | 4 +- gateway/src/main.rs | 8 +- gateway/tests/startup_log.rs | 31 +++ operator/src/controller/grid_network.rs | 3 +- operator/src/crd/grid_network.rs | 179 ++++++++++++- operator/src/resources/consumer_config.rs | 242 +++++++++++++++++- scripts/verify-helm-chart.sh | 8 + 19 files changed, 944 insertions(+), 10 deletions(-) create mode 100644 charts/praxis-gateway/tests/telemetry_test.yaml create mode 100644 docs/architecture/opentelemetry.md diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 90d4900bf..e89b5310f 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -287,6 +287,61 @@ spec: maximum: 65535.0 minimum: 0.0 type: integer + telemetry: + description: |- + Optional OpenTelemetry exporter and sampling settings for this gateway. + + When present, the generated Praxis configuration opts into telemetry and + adds the `trace_context` filter for outbound W3C header propagation. + Collector authentication must be provided to the gateway Deployment via + `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + into this `ConfigMap`. + nullable: true + properties: + batchIntervalSecs: + description: Batch exporter interval in seconds. Must be positive when set. + format: uint64 + minimum: 1.0 + nullable: true + type: integer + batchSize: + description: Maximum spans per export batch. Must be positive when set. + format: uint + minimum: 1.0 + nullable: true + type: integer + environment: + description: Deployment environment resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + otlpEndpoint: + description: |- + OTLP collector endpoint, for example `http://otel-collector:4317`. + + If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + gateway Deployment. + nullable: true + pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ + type: string + samplingRate: + description: Root trace sampling probability from `0.0` through `1.0`. + format: double + maximum: 1.0 + minimum: 0.0 + nullable: true + type: number + serviceName: + description: OpenTelemetry `service.name` resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + serviceVersion: + description: OpenTelemetry `service.version` resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + type: object tlsCertMountPath: default: /etc/praxis/tls description: |- diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index f86c8f822..b0ae5f04b 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -186,7 +186,7 @@ Praxis AI image; these values may advance independently. | `image.repository` | string | `ghcr.io/praxis-proxy/ai` | Image repository. | | `image.tag` | string | `0.4.0` | Image tag (ignored when `image.digest` is set). | | `image.digest` | string | `""` | Immutable digest (sha256:…). When set, tag is ignored. | -| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider` and `gridServing` need. A repository ending in `/grid-gateway` sets it. | +| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider`, `gridServing`, and telemetry need. A repository ending in `/grid-gateway` sets it. | | `image.pullPolicy` | string | `IfNotPresent` | Image pull policy. | | `imagePullSecrets` | list | `[]` | Pull secrets for private registries. | | `log.level` | string | `""` | Level for every module, rendered as RUST_LOG on the gateway and overlay-sync: off, error, warn, info, debug, or trace, in any case. Empty leaves RUST_LOG unset, so the binaries use their info default. An `env` entry named RUST_LOG takes precedence on the gateway. | @@ -206,6 +206,11 @@ Praxis AI image; these values may advance independently. | `config.key` | string | `praxis.yaml` | Key in the ConfigMap. | | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | +| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. Praxis 0.7.1 exports HTTP server and upstream exchange spans, plus configured AI routing spans; its W3C headers do not link exported edge and provider spans. See [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | +| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP/gRPC endpoint without URL userinfo, query, or fragment credentials. Empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. | +| `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | +| `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | +| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | Positive OTLP batch export interval and maximum batch size. | | `gatewayConfig.model` | string | **required** for a consumer without `gridServing` | Model advertised on the routing candidates. | | `gatewayConfig.backends` | map | **required** when rendered | Backends keyed by site, each with `endpoint` and optional `healthCheck` and `transport`. A consumer's key is the site it reaches over mutual TLS. A provider's `local` key is its one plaintext backend. The older list of `cluster`, `endpoints` entries still renders. | | `gatewayConfig.backends[].site` | string | `localSite` | Grid site the backend serves. A consumer's remote `mutual_tls` backend must name it, and it must differ from `localSite`. Its `transport.sni` defaults to `.grid.internal`. | diff --git a/charts/praxis-gateway/templates/_gateway-config.tpl b/charts/praxis-gateway/templates/_gateway-config.tpl index c85681492..ac589686e 100644 --- a/charts/praxis-gateway/templates/_gateway-config.tpl +++ b/charts/praxis-gateway/templates/_gateway-config.tpl @@ -5,11 +5,44 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf {{- $cfg := .Values.gatewayConfig }} {{- $provider := eq ($cfg.role | default "consumer") "provider" }} {{- $apiKey := and (not $provider) (eq ($cfg.auth.mode | default "none") "api-key") }} +{{- $telemetry := $cfg.telemetry | default dict }} +{{- $hasSamplingRate := and (hasKey $telemetry "samplingRate") (ne $telemetry.samplingRate nil) }} +{{- $hasBatchInterval := and (hasKey $telemetry "batchIntervalSecs") (ne $telemetry.batchIntervalSecs nil) }} +{{- $hasBatchSize := and (hasKey $telemetry "batchSize") (ne $telemetry.batchSize nil) }} +{{- $hasTelemetryValues := or $telemetry.otlpEndpoint $hasSamplingRate $telemetry.serviceName $telemetry.serviceVersion $telemetry.environment $hasBatchInterval $hasBatchSize }} praxis.yaml: | {{- with $cfg.upstreamCA.secretName }} runtime: upstream_ca_file: {{ printf "%s/%s" $cfg.upstreamCA.mountPath ($cfg.upstreamCA.key | default "ca.crt") | quote }} {{- end }} + {{- if $telemetry.enabled }} + telemetry: + {{- if not $hasTelemetryValues }} + {} + {{- else }} + {{- with $telemetry.otlpEndpoint }} + otlp_endpoint: {{ . | quote }} + {{- end }} + {{- if $hasSamplingRate }} + sampling_rate: {{ $telemetry.samplingRate }} + {{- end }} + {{- with $telemetry.serviceName }} + service_name: {{ . | quote }} + {{- end }} + {{- with $telemetry.serviceVersion }} + service_version: {{ . | quote }} + {{- end }} + {{- with $telemetry.environment }} + environment: {{ . | quote }} + {{- end }} + {{- if $hasBatchInterval }} + batch_interval_secs: {{ $telemetry.batchIntervalSecs }} + {{- end }} + {{- if $hasBatchSize }} + batch_size: {{ $telemetry.batchSize }} + {{- end }} + {{- end }} + {{- end }} admin: address: {{ include "praxis-gateway.renderedAdminAddress" . | quote }} listeners: @@ -43,6 +76,9 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf filter_chains: - name: main filters: + {{- if $telemetry.enabled }} + - filter: trace_context + {{- end }} {{- if $provider }} {{- if ne (($cfg.peerTrust).mode | default "pin") "spiffe" }} - filter: peer_identity_trust diff --git a/charts/praxis-gateway/templates/_helpers.tpl b/charts/praxis-gateway/templates/_helpers.tpl index b09ec3bad..a08505fba 100644 --- a/charts/praxis-gateway/templates/_helpers.tpl +++ b/charts/praxis-gateway/templates/_helpers.tpl @@ -132,6 +132,15 @@ Validate image digest format when provided. Validate required config ConfigMap name. */}} {{- define "praxis-gateway.validateConfig" -}} +{{- $telemetry := .Values.gatewayConfig.telemetry | default dict }} +{{- if $telemetry.enabled }} +{{- if ne .Values.image.flavor "grid-gateway" }} +{{- fail "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway, whose Grid gateway build includes the OTLP and AI routing span features" }} +{{- end }} +{{- if not .Values.gatewayConfig.render }} +{{- fail "gatewayConfig.telemetry.enabled needs gatewayConfig.render true so the exporter settings are written to praxis.yaml" }} +{{- end }} +{{- end }} {{- if .Values.gatewayConfig.render }} {{- $consumer := ne (.Values.gatewayConfig.role | default "consumer") "provider" }} {{- if and $consumer (not (.Values.gridServing).enabled) (not (trim (toString .Values.gatewayConfig.model))) }} diff --git a/charts/praxis-gateway/templates/deployment.yaml b/charts/praxis-gateway/templates/deployment.yaml index 4952793e6..97c88d9d1 100644 --- a/charts/praxis-gateway/templates/deployment.yaml +++ b/charts/praxis-gateway/templates/deployment.yaml @@ -33,6 +33,10 @@ spec: {{- else if eq $configSource "render" }} {{- $_ := set $podAnnotations "checksum/config" (include "praxis-gateway.gatewayConfigData" . | sha256sum) }} {{- end }} + {{- if (.Values.gatewayConfig.telemetry).enabled }} + {{- /* Exporter initialization is process-scoped, so changed settings need a restart. */}} + {{- $_ := set $podAnnotations "checksum/telemetry" (toJson .Values.gatewayConfig.telemetry | sha256sum) }} + {{- end }} {{- with $podAnnotations }} annotations: {{- toYaml . | nindent 8 }} diff --git a/charts/praxis-gateway/tests/telemetry_test.yaml b/charts/praxis-gateway/tests/telemetry_test.yaml new file mode 100644 index 000000000..14a328b68 --- /dev/null +++ b/charts/praxis-gateway/tests/telemetry_test.yaml @@ -0,0 +1,133 @@ +suite: gateway telemetry +templates: + - templates/gateway-config.yaml + - templates/deployment.yaml +tests: + - it: emits telemetry and W3C propagation from explicit settings + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.otlpEndpoint: http://otel-collector.observability:4317 + gatewayConfig.telemetry.samplingRate: 0.25 + gatewayConfig.telemetry.serviceName: grid-edge + gatewayConfig.telemetry.batchIntervalSecs: 3 + gatewayConfig.telemetry.batchSize: 32 + env: + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: collector-credentials + key: headers + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+otlp_endpoint: \\\"http://otel-collector\\.observability:4317\\\".*sampling_rate: 0.25.*service_name: \\\"grid-edge\\\".*batch_interval_secs: 3.*batch_size: 32" + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)filters:\\n\\s+- filter: trace_context" + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.name + value: collector-credentials + template: templates/deployment.yaml + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.key + value: headers + template: templates/deployment.yaml + - exists: + path: spec.template.metadata.annotations["checksum/telemetry"] + template: templates/deployment.yaml + + - it: accepts the lower sampling boundary + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.samplingRate: 0.0 + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "sampling_rate: 0" + + - it: accepts the upper sampling boundary + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.samplingRate: 1.0 + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "sampling_rate: 1" + + - it: renders an empty telemetry mapping for environment-only settings + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+\\{\\}" + + - it: requires a Grid gateway image when telemetry is enabled + template: templates/deployment.yaml + set: + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + asserts: + - failedTemplate: + errorPattern: "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway" + + - it: requires generated Praxis config before enabling telemetry + template: templates/deployment.yaml + set: + image.flavor: grid-gateway + gatewayConfig.telemetry.enabled: true + asserts: + - failedTemplate: + errorPattern: "gatewayConfig.telemetry.enabled needs gatewayConfig.render true" diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index aa48826d3..c44ad8444 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -176,6 +176,21 @@ "existingSecret": { "type": "string" }, "mountPath": { "type": "string" } } + }, + "telemetry": { + "type": "object", + "additionalProperties": false, + "description": "Optional exporter settings for generated praxis.yaml. Collector credentials must come from deployment-managed Secret environment references.", + "properties": { + "enabled": { "type": "boolean", "description": "Enable OTLP export and the trace_context propagation filter." }, + "otlpEndpoint": { "type": "string", "pattern": "^(|https?://[^/?#@\\s]+(:[0-9]+)?(/[^\\s?#]*)?)$", "description": "OTLP/gRPC endpoint without userinfo, query, or fragment credentials. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT." }, + "samplingRate": { "type": ["number", "null"], "minimum": 0, "maximum": 1, "description": "Root trace sampling probability from 0.0 through 1.0." }, + "serviceName": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.name resource attribute." }, + "serviceVersion": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.version resource attribute." }, + "environment": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "deployment.environment resource attribute." }, + "batchIntervalSecs": { "type": ["integer", "null"], "minimum": 1, "description": "Positive batch exporter interval in seconds." }, + "batchSize": { "type": ["integer", "null"], "minimum": 1, "description": "Positive maximum spans per export batch." } + } } } }, diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index 57c56fc1e..834d6b460 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -132,6 +132,26 @@ gatewayConfig: # when config.existingConfigMap is empty and backends, role provider, or gridServing is set. # Changing the rendered config rolls the pods. render: false + # -- Opt in to OTLP tracing in the generated praxis.yaml. Requires + # image.flavor: grid-gateway, whose Grid image build includes the Praxis OTLP + # exporter and Praxis AI routing-span features. Credentials belong in env as + # Secret-backed OTEL_EXPORTER_OTLP_HEADERS, never here. + telemetry: + enabled: false + # -- OTLP/gRPC endpoint. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT from env. + otlpEndpoint: "" + # -- Root sampling probability in the inclusive range 0.0..1.0. + samplingRate: null + # -- OpenTelemetry service.name resource attribute. + serviceName: "" + # -- OpenTelemetry service.version resource attribute. + serviceVersion: "" + # -- deployment.environment resource attribute. + environment: "" + # -- Batch export interval in seconds; must be positive. + batchIntervalSecs: null + # -- Maximum spans per export batch; must be positive. + batchSize: null # -- consumer routes callers. provider serves grid peers on the grid identity and forwards to one backend. role: consumer # -- Grid peers a provider accepts, each with a Grid-CA client certificate. diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index f8106023d..2213ff0f5 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -279,6 +279,61 @@ spec: maximum: 65535.0 minimum: 0.0 type: integer + telemetry: + description: |- + Optional OpenTelemetry exporter and sampling settings for this gateway. + + When present, the generated Praxis configuration opts into telemetry and + adds the `trace_context` filter for outbound W3C header propagation. + Collector authentication must be provided to the gateway Deployment via + `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + into this `ConfigMap`. + nullable: true + properties: + batchIntervalSecs: + description: Batch exporter interval in seconds. Must be positive when set. + format: uint64 + minimum: 1.0 + nullable: true + type: integer + batchSize: + description: Maximum spans per export batch. Must be positive when set. + format: uint + minimum: 1.0 + nullable: true + type: integer + environment: + description: Deployment environment resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + otlpEndpoint: + description: |- + OTLP collector endpoint, for example `http://otel-collector:4317`. + + If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + gateway Deployment. + nullable: true + pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ + type: string + samplingRate: + description: Root trace sampling probability from `0.0` through `1.0`. + format: double + maximum: 1.0 + minimum: 0.0 + nullable: true + type: number + serviceName: + description: OpenTelemetry `service.name` resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + serviceVersion: + description: OpenTelemetry `service.version` resource attribute. + nullable: true + pattern: ^(|.*\S.*)$ + type: string + type: object tlsCertMountPath: default: /etc/praxis/tls description: |- diff --git a/docs/README.md b/docs/README.md index 1247e558b..ba75c5a7e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,8 @@ access policy, and trust model. - [Consumer Config](architecture/consumer-config.md) — operator-generated consumer Praxis `ConfigMap` and the `GatewayRef.consumerConfig` API. +- [OpenTelemetry](architecture/opentelemetry.md) — exporter configuration, + secret handling, image requirements, and the Praxis 0.7.1 trace-linkage limit. - [External Client Ingress](architecture/external-ingress.md) — GTM/GLB edge selection, AGN provider routing, trust boundaries, affinity, snapshot delivery, and provider-boundary ownership. diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index a416e6646..6c4baff5e 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -72,6 +72,14 @@ The generated config is a complete, runnable Praxis config containing: - `admin:` — admin listener at `127.0.0.1:9901` - `shutdown_timeout_secs: 5` +Set `consumerConfig.telemetry` to add process-level OTLP settings and the +`trace_context` propagation filter. These settings are written at the Praxis +config root and never enter the routing overlay. The consumer ConfigMap does +not contain collector headers; configure `OTEL_EXPORTER_OTLP_HEADERS` on the +gateway Deployment with a Secret-backed environment reference. See +[OpenTelemetry for Grid gateways](opentelemetry.md) for examples and the +Praxis 0.7.1 trace-linkage limitation. + This generated config covers the direct API-provider path where the consumer gateway is often also the final-hop gateway for the provider API call. Remote provider sites follow the same SecretRef contract, but the provider credential diff --git a/docs/architecture/opentelemetry.md b/docs/architecture/opentelemetry.md new file mode 100644 index 000000000..46ff03cd5 --- /dev/null +++ b/docs/architecture/opentelemetry.md @@ -0,0 +1,135 @@ +# OpenTelemetry for Grid gateways + +Grid gateway builds can export traces to an OTLP/gRPC collector. Export is +configured on the gateway process; routing overlays remain limited to routing +state. + +## Enable export + +The default configuration has no collector endpoint, so it starts without an +OTLP exporter and does not require a collector. To opt in, use either +`GatewayRef.consumerConfig.telemetry` for operator-generated consumer config or +`gatewayConfig.telemetry` for the Helm-generated Praxis config. + +An operator example: + +```yaml +spec: + gatewayRefs: + - name: edge + namespace: grid + consumerConfig: + enabled: true + telemetry: + otlpEndpoint: http://otel-collector.observability:4317 + samplingRate: 0.1 + serviceName: grid-edge + environment: production +``` + +For Helm, enable `gatewayConfig.telemetry` and use the Grid gateway image: + +```yaml +image: + repository: ghcr.io/praxis-proxy/grid-gateway + flavor: grid-gateway +gatewayConfig: + render: true + telemetry: + enabled: true + otlpEndpoint: http://otel-collector.observability:4317 + samplingRate: 0.1 + serviceName: grid-edge +``` + +Both paths render a top-level Praxis `telemetry` block. The Helm chart adds the +`trace_context` filter when telemetry is enabled. The operator-generated +consumer config does the same. Neither path places exporter settings in the +routing overlay. + +## Credentials and lifecycle + +Do not put credentials in `otlpEndpoint`, telemetry fields, routing overlays, +or filter configuration. Praxis reads OTLP headers from +`OTEL_EXPORTER_OTLP_HEADERS`; provide that variable from a Deployment-managed +Secret. Helm exposes the container `env` list for a Secret reference: + +```yaml +env: + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: collector-credentials + key: headers +``` + +The Secret value uses the OpenTelemetry `key=value` header format. The operator +only creates the consumer `ConfigMap`; the deployment manager must add the same +Secret-backed environment reference to its gateway Deployment. Praxis redacts +OTLP header values from its config debug representation. + +Exporter setup runs once at process startup. Helm rolls gateway pods when its +telemetry values change. An externally managed consumer Deployment must be +restarted after its generated ConfigMap changes, and a gateway must be restarted +after the collector endpoint or Secret-backed environment changes. On normal +server return, Grid drops Praxis's `TracingGuard`, which shuts down the OTLP +provider and flushes queued spans. + +## Build and spans + +The `grid-gateway` binary built by `deploy/gateway/Containerfile` includes the +Praxis 0.7.1 `otel` feature and the pinned Praxis AI v0.4.1 +`opentelemetry` feature. Use an image built from that target, published as +`ghcr.io/praxis-proxy/grid-gateway`, for this configuration. The chart's default +`ghcr.io/praxis-proxy/ai:0.4.0` image does not include the Grid build features. + +With the AI feature enabled, these short semantic spans are supported when the +corresponding filters run: + +- `http_request` server spans and `upstream_exchange` internal spans from + Praxis 0.7.1's HTTP proxy protocol. +- `routing.select` from `intelligent_route`, including the serving overlay + semantic revision when that revision is available to the filter. +- `provider.route` from `provider_route`, including a validated edge overlay + revision when present. + +The pinned AI implementation projects bounded routing fields into those spans; +it does not add prompt or body contents, credential values, authorization +headers, cookies, session keys, or raw request IDs. These spans describe route +selection and resolution; `provider.route` does not prove the model backend +served the request. + +## Trace linkage status in Praxis 0.7.1 + +Praxis 0.7.1 exports each gateway's HTTP server span and an internal +`upstream_exchange` span. The AI `routing.select` and `provider.route` spans +also appear as children of their local gateway's server span. The proxy's +upstream span is named `upstream_exchange` with `INTERNAL` kind; Praxis 0.7.1 +does not export it as an HTTP client span. + +The `trace_context` filter separately validates inbound W3C `traceparent`, +stores that value in a Praxis request extension, and forwards its trace ID and +flags with a newly generated hop span ID. The filter's own source documentation +states that this hop ID does not name an exported span. Praxis 0.7.1 also does +not extract that extension into the OpenTelemetry context used by the HTTP +server span. This leaves two parallel contexts: exported spans use the local +OpenTelemetry request context, while forwarded headers use the filter's W3C +context. + +A collector-backed test used an incoming trace ID of +`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`. The edge exported a `POST` server span in +trace `89e767a01d701b4f04d1b7b12c022c8a`, and the provider exported a `POST` +server span in trace `abea135120a177a97b76273f37477f99`; both have no parent ID. +The edge `routing.select` span is a child of the edge server span, and the +provider `provider.route` span is a child of the provider server span. The +test backend received `traceparent` with the original `aaaa...` trace ID and a +fresh hop ID. The collector received spans only after normal SIGTERM shutdown, +confirming provider flush. Full request details and collector output are kept +outside the source tree in the PR evidence directory. + +This proves header propagation and local span export. It also proves that the +exact pinned Praxis version does not create exported cross-gateway parent/child +relationships. Issue #260 remains incomplete until Praxis provides a shared +W3C/OpenTelemetry context for inbound server spans and outbound client spans, +then a collector test confirms the resulting parent IDs across both gateways. +See the accompanying Praxis issue draft for the proposed dependency change. diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index e36c2b44a..848e0ba01 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -308,10 +308,10 @@ name = "grid-gateway" path = "src/main.rs" [dependencies] -praxis-proxy = { version = "0.7.3", features = ["spiffe"] } +praxis-proxy = { version = "0.7.3", features = ["otel", "spiffe"] } praxis-proxy-filter = "0.7.3" praxis-proxy-core = "0.7.3" -praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false } +praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false, features = ["opentelemetry"] } ai-grid-filters = { path = "ai-grid-filters" } tracing = "0.1.44" diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 629a2da2a..b88439233 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -59,10 +59,14 @@ fn main() -> ExitCode { None => None, }; - // Returning instead of exiting drops the guard, flushing queued log lines. + // Use the returning server path so both the routing runtime and tracing + // provider can shut down cleanly after the listeners stop. let result = praxis::try_run_server_with_registry(config, registry, config_file, log_level); drop(grid_runtime); - result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS) + let exit_code = result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS); + // The Praxis guard shuts down the OTLP provider and flushes queued spans. + drop(tracing_guard); + exit_code } /// Start the cross-site pollers and register `grid_site_route` over their snapshot. diff --git a/gateway/tests/startup_log.rs b/gateway/tests/startup_log.rs index 88a8a7d52..818f3f94d 100644 --- a/gateway/tests/startup_log.rs +++ b/gateway/tests/startup_log.rs @@ -109,4 +109,35 @@ mod tests { ); Ok(()) } + + #[test] + fn server_pipeline_error_is_reported_before_tracing_guard_flush() -> TestResult { + let path = write_config("invalid-filter.yaml")?; + let yaml = std::fs::read_to_string(&path)? + .replace("filter: static_response", "filter: not_registered_for_startup_test"); + std::fs::write(&path, yaml)?; + let (mut child, lines) = spawn(path, None)?; + + // Both streams close on exit, which disconnects the channel. + let deadline = Instant::now() + TIMEOUT; + let mut output = Vec::new(); + loop { + match lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) => output.push(line), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => { + child.kill()?; + child.wait()?; + return Err(format!("gateway still running after {TIMEOUT:?}").into()); + }, + } + } + let status = child.wait()?; + assert!(!status.success(), "an invalid pipeline filter must fail startup"); + assert!( + output.iter().any(|line| line.contains("fatal error; exiting")), + "the server startup error was not logged before the tracing guard dropped: {output:#?}" + ); + Ok(()) + } } diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 9438805cd..4137f666e 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2085,12 +2085,13 @@ fn consumer_config_map( gw_ref: &GatewayRef, cc: &ConsumerConfig, ) -> Result { - let config_yaml = consumer_config::generate_consumer_praxis_config( + let config_yaml = consumer_config::generate_consumer_praxis_config_with_telemetry( overlay, &cc.credential_mount_base, &cc.cluster_endpoints, &cc.tls_cert_mount_path, cc.listener_port, + cc.telemetry.as_ref(), )?; Ok(consumer_config::build_consumer_config_map( &config_yaml, diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 9ef3523c1..7a66ab9b2 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -778,7 +778,7 @@ pub struct GatewayRef { /// The generated `ConfigMap` never contains credential token bytes. Credential /// entries use a `file:` source under `credentialMountBase`; the mounted /// Kubernetes Secret provides the token at runtime. -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct ConsumerConfig { /// Enable operator-managed consumer Praxis config generation for this gateway. @@ -840,6 +840,116 @@ pub struct ConsumerConfig { /// Default: `8080`. #[serde(default = "default_listener_port")] pub listener_port: u16, + + /// Optional OpenTelemetry exporter and sampling settings for this gateway. + /// + /// When present, the generated Praxis configuration opts into telemetry and + /// adds the `trace_context` filter for outbound W3C header propagation. + /// Collector authentication must be provided to the gateway Deployment via + /// `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + /// into this `ConfigMap`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub telemetry: Option, +} + +/// Validated OpenTelemetry settings rendered into operator-generated Praxis YAML. +/// +/// Secret material is intentionally not part of this type. Supply collector +/// credentials through the gateway Deployment's `OTEL_EXPORTER_OTLP_HEADERS` +/// environment variable using a Secret-backed `valueFrom` reference. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct GatewayTelemetryConfig { + /// OTLP collector endpoint, for example `http://otel-collector:4317`. + /// + /// If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + /// gateway Deployment. + #[schemars(regex(pattern = r"^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub otlp_endpoint: Option, + + /// Root trace sampling probability from `0.0` through `1.0`. + #[schemars(range(min = 0.0, max = 1.0))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sampling_rate: Option, + + /// OpenTelemetry `service.name` resource attribute. + #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_name: Option, + + /// OpenTelemetry `service.version` resource attribute. + #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_version: Option, + + /// Deployment environment resource attribute. + #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option, + + /// Batch exporter interval in seconds. Must be positive when set. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_interval_secs: Option, + + /// Maximum spans per export batch. Must be positive when set. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_size: Option, +} + +impl GatewayTelemetryConfig { + /// Validate values before rendering them into the consumer configuration. + /// + /// # Errors + /// + /// Returns an explanatory message for empty strings, invalid sampling + /// rates, or zero-valued batch settings. + #[expect( + clippy::too_many_lines, + reason = "validates all telemetry settings before CRD rendering" + )] + pub fn validate(&self) -> Result<(), String> { + if self.otlp_endpoint.as_ref().is_some_and(|endpoint| { + let endpoint = endpoint.trim(); + let Some((scheme, rest)) = endpoint.split_once("://") else { + return true; + }; + let authority = rest.split('/').next().unwrap_or_default(); + endpoint.is_empty() + || !matches!(scheme, "http" | "https") + || authority.is_empty() + || authority.contains('@') + || endpoint.contains('?') + || endpoint.contains('#') + || endpoint.chars().any(char::is_whitespace) + }) { + return Err("telemetry.otlpEndpoint must be an http(s) URL without credentials".to_owned()); + } + if let Some(rate) = self.sampling_rate + && (!rate.is_finite() || !(0.0..=1.0).contains(&rate)) + { + return Err("telemetry.samplingRate must be between 0.0 and 1.0".to_owned()); + } + if self.batch_interval_secs == Some(0) { + return Err("telemetry.batchIntervalSecs must be greater than zero".to_owned()); + } + if self.batch_size == Some(0) { + return Err("telemetry.batchSize must be greater than zero".to_owned()); + } + for (field, value) in [ + ("serviceName", self.service_name.as_deref()), + ("serviceVersion", self.service_version.as_deref()), + ("environment", self.environment.as_deref()), + ] { + if value.is_some_and(|value| value.trim().is_empty()) { + return Err(format!("telemetry.{field} must not be blank")); + } + } + Ok(()) + } } impl Default for ConsumerConfig { @@ -851,6 +961,7 @@ impl Default for ConsumerConfig { cluster_endpoints: Vec::new(), tls_cert_mount_path: default_tls_cert_mount_path(), listener_port: default_listener_port(), + telemetry: None, } } } @@ -1505,7 +1616,12 @@ mod tests { "mode": "mutual_tls", "sni": "site-a.grid.internal" } - }] + }], + "telemetry": { + "otlpEndpoint": "http://otel-collector:4317", + "samplingRate": 0.125, + "serviceName": "grid-edge" + } } }); let gw: GatewayRef = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); @@ -1538,6 +1654,14 @@ mod tests { Some("site-a.grid.internal"), "transport SNI must round-trip" ); + let telemetry = cc.telemetry.as_ref().unwrap_or_else(|| std::process::abort()); + assert_eq!(telemetry.otlp_endpoint.as_deref(), Some("http://otel-collector:4317")); + assert_eq!(telemetry.sampling_rate, Some(0.125)); + assert_eq!(telemetry.service_name.as_deref(), Some("grid-edge")); + assert!( + telemetry.validate().is_ok(), + "valid telemetry settings must pass validation" + ); } #[test] @@ -1592,12 +1716,41 @@ mod tests { cc.cluster_endpoints.is_empty(), "clusterEndpoints must default to empty" ); + assert!(cc.telemetry.is_none(), "telemetry must remain disabled by default"); assert_eq!( cc.tls_cert_mount_path, "/etc/praxis/tls", "tlsCertMountPath must use default" ); } + #[test] + fn telemetry_config_rejects_embedded_headers_and_malformed_endpoints() { + let with_headers = serde_json::json!({ + "otlpEndpoint": "http://collector:4317", + "otlpHeaders": {"authorization": "test-value"} + }); + assert!( + serde_json::from_value::(with_headers).is_err(), + "collector credentials must not be accepted as config fields" + ); + + for endpoint in [ + "https://user:password@collector:4317", + "https://collector:4317?api_key=secret", + "https://collector:4317/otlp#token=secret", + ] { + let telemetry = serde_json::from_value::(serde_json::json!({ + "otlpEndpoint": endpoint, + "samplingRate": 0.5 + })) + .unwrap_or_else(|_| std::process::abort()); + assert!( + telemetry.validate().is_err(), + "credentials in endpoint userinfo, query, or fragment must be rejected: {endpoint}" + ); + } + } + #[test] fn consumer_config_absent_not_serialized() { let gw = GatewayRef { @@ -1614,6 +1767,10 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "asserts generated telemetry fields and all opt-in security boundaries" + )] fn grid_network_crd_has_consumer_config_field_on_gateway_ref() { let crd = crd_json(); let gateway_ref_properties = crd @@ -1637,6 +1794,24 @@ mod tests { consumer_config_properties.contains_key("tlsCertMountPath"), "CRD schema must include consumerConfig.tlsCertMountPath" ); + let telemetry_properties = consumer_config_properties + .get("telemetry") + .and_then(|v| v.pointer("/properties")) + .and_then(serde_json::Value::as_object) + .unwrap_or_else(|| std::process::abort()); + let sampling_rate = telemetry_properties + .get("samplingRate") + .unwrap_or_else(|| std::process::abort()); + assert_eq!( + sampling_rate.pointer("/minimum").and_then(serde_json::Value::as_f64), + Some(0.0), + "CRD schema must reject sampling rates below zero" + ); + assert_eq!( + sampling_rate.pointer("/maximum").and_then(serde_json::Value::as_f64), + Some(1.0), + "CRD schema must reject sampling rates above one" + ); } #[test] diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index c4012f463..505f28634 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -23,7 +23,7 @@ use std::collections::{BTreeMap, BTreeSet}; use k8s_openapi::api::core::v1::ConfigMap; use crate::{ - crd::grid_network::{ClusterEndpointConfig, SelectionMode, TransportMode}, + crd::grid_network::{ClusterEndpointConfig, GatewayTelemetryConfig, SelectionMode, TransportMode}, resources::routing_overlay::{RoutingCandidate, RoutingOverlay}, }; @@ -86,6 +86,10 @@ pub enum ConsumerConfigError { cluster: String, }, + /// Telemetry configuration failed validation. + #[error("invalid telemetry configuration: {0}")] + InvalidTelemetry(String), + /// JSON serialization failed. #[error("json error: {0}")] Json(#[from] serde_json::Error), @@ -123,16 +127,48 @@ pub enum ConsumerConfigError { /// - Any candidate cluster has no matching endpoint in `cluster_endpoints`. /// - Any cluster endpoint has no `transport` configuration. /// - Any `mutual_tls` endpoint has no (or blank) `sni`. +#[cfg(test)] +pub(crate) fn generate_consumer_praxis_config( + overlay: &RoutingOverlay, + credential_mount_base: &str, + cluster_endpoints: &[ClusterEndpointConfig], + tls_cert_mount_path: &str, + listener_port: u16, +) -> Result { + generate_consumer_praxis_config_with_telemetry( + overlay, + credential_mount_base, + cluster_endpoints, + tls_cert_mount_path, + listener_port, + None, + ) +} + +/// Generate the consumer config with optional process-level telemetry settings. +/// +/// The telemetry block is emitted at the Praxis config root, beside listeners +/// and admin settings. It is never copied into the routing overlay. +/// +/// # Errors +/// +/// Returns [`ConsumerConfigError`] when the overlay, endpoint topology, or +/// optional telemetry settings are invalid. #[expect( clippy::too_many_lines, reason = "sequential validation + three rendering passes; splitting would obscure the overall config shape" )] -pub(crate) fn generate_consumer_praxis_config( +#[expect( + clippy::too_many_arguments, + reason = "preserves the renderer's established inputs and adds optional telemetry" +)] +pub(crate) fn generate_consumer_praxis_config_with_telemetry( overlay: &RoutingOverlay, credential_mount_base: &str, cluster_endpoints: &[ClusterEndpointConfig], tls_cert_mount_path: &str, listener_port: u16, + telemetry: Option<&GatewayTelemetryConfig>, ) -> Result { if overlay.local_site.trim().is_empty() { return Err(ConsumerConfigError::BlankLocalSite); @@ -148,10 +184,18 @@ pub(crate) fn generate_consumer_praxis_config( }); } } + if let Some(telemetry) = telemetry { + telemetry.validate().map_err(ConsumerConfigError::InvalidTelemetry)?; + } let candidates_yaml = render_candidates(&overlay.candidates); let selection_policy_yaml = render_selection_policy(overlay.selection_policy.as_ref()); let local_site = yaml_scalar(&overlay.local_site)?; + let trace_context_filter = if telemetry.is_some() { + " - filter: trace_context\n" + } else { + "" + }; let credential_inject_section = render_credential_inject(&overlay.candidates, credential_mount_base); let load_balancer_section = render_load_balancer(&overlay.candidates, cluster_endpoints, tls_cert_mount_path)?; @@ -166,6 +210,7 @@ pub(crate) fn generate_consumer_praxis_config( filter_chains:\n\ \x20 - name: consumer-chain\n\ \x20 filters:\n\ + {trace_context_filter}\ \x20 - filter: json_body_field\n\ \x20 field: model\n\ \x20 header: X-Model\n\ @@ -183,12 +228,69 @@ pub(crate) fn generate_consumer_praxis_config( config.push_str(&load_balancer_section); + if let Some(telemetry) = telemetry { + config.push_str(&render_telemetry(telemetry)?); + } + // Admin interface and graceful shutdown — standard constants for consumer gateways. config.push_str("\nadmin:\n address: \"127.0.0.1:9901\"\nshutdown_timeout_secs: 5\n"); Ok(config) } +/// Render validated exporter settings without secret material. +#[expect( + clippy::too_many_lines, + reason = "serializes every optional telemetry value in a stable YAML order" +)] +fn render_telemetry(telemetry: &GatewayTelemetryConfig) -> Result { + if telemetry.otlp_endpoint.is_none() + && telemetry.sampling_rate.is_none() + && telemetry.service_name.is_none() + && telemetry.service_version.is_none() + && telemetry.environment.is_none() + && telemetry.batch_interval_secs.is_none() + && telemetry.batch_size.is_none() + { + return Ok("\ntelemetry: {}\n".to_owned()); + } + let mut output = String::from("\ntelemetry:\n"); + if let Some(endpoint) = telemetry.otlp_endpoint.as_deref() { + output.push_str(" otlp_endpoint: "); + output.push_str(&yaml_scalar(endpoint)?); + output.push('\n'); + } + if let Some(rate) = telemetry.sampling_rate { + output.push_str(" sampling_rate: "); + output.push_str(&rate.to_string()); + output.push('\n'); + } + for (name, value) in [ + ("service_name", telemetry.service_name.as_deref()), + ("service_version", telemetry.service_version.as_deref()), + ("environment", telemetry.environment.as_deref()), + ] { + if let Some(value) = value { + output.push_str(" "); + output.push_str(name); + output.push_str(": "); + output.push_str(&yaml_scalar(value)?); + output.push('\n'); + } + } + if let Some(interval) = telemetry.batch_interval_secs { + output.push_str(" batch_interval_secs: "); + output.push_str(&interval.to_string()); + output.push('\n'); + } + if let Some(size) = telemetry.batch_size { + output.push_str(" batch_size: "); + output.push_str(&size.to_string()); + output.push('\n'); + } + Ok(output) +} + /// Build the Kubernetes `ConfigMap` for the generated consumer Praxis config. /// /// The `ConfigMap` contains a single `praxis.yaml` key with the rendered YAML. @@ -650,6 +752,142 @@ mod tests { // Renderer: basic structure // ----------------------------------------------------------------------- + #[test] + #[expect( + clippy::too_many_lines, + reason = "checks opt-in output, secret exclusion, and overlay separation" + )] + fn telemetry_is_opt_in_and_rendered_outside_routing_filters() { + let overlay = simple_overlay(vec![plain_candidate( + "inference_model", + "model-a", + "site-a", + "gateway-site-a", + true, + )]); + let endpoints = endpoint_coverage(&overlay); + let disabled = generate_consumer_praxis_config(&overlay, MOUNT_BASE, &endpoints, "/etc/praxis/tls", 8080) + .unwrap_or_else(|_| std::process::abort()); + assert!( + !disabled.contains("telemetry:"), + "telemetry must remain absent by default" + ); + assert!( + !disabled.contains("filter: trace_context"), + "propagation must remain opt-in" + ); + + let telemetry = GatewayTelemetryConfig { + otlp_endpoint: Some("http://collector.observability:4317".to_owned()), + sampling_rate: Some(0.25), + service_name: Some("grid-edge-site-a".to_owned()), + service_version: Some("0.1.4".to_owned()), + environment: Some("test".to_owned()), + batch_interval_secs: Some(3), + batch_size: Some(32), + }; + let enabled = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &endpoints, + "/etc/praxis/tls", + 8080, + Some(&telemetry), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + enabled.contains("filter: trace_context"), + "telemetry must enable W3C propagation" + ); + assert!(enabled.contains("telemetry:\n otlp_endpoint: \"http://collector.observability:4317\"")); + assert!(enabled.contains(" sampling_rate: 0.25")); + assert!(enabled.contains(" service_name: \"grid-edge-site-a\"")); + assert!(enabled.contains(" batch_interval_secs: 3")); + assert!( + !enabled.contains("otlp_headers"), + "credentials must not be rendered into the ConfigMap" + ); + assert!( + !enabled.contains("collector-secret-value"), + "secret values must not appear in generated YAML" + ); + assert!( + !serde_json::to_string(&overlay) + .unwrap_or_else(|_| std::process::abort()) + .contains("telemetry"), + "exporter settings must stay out of the routing overlay" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "covers environment fallback, sampling boundaries, and credential rejection" + )] + fn telemetry_rejects_invalid_sampling_rates_and_credentials_in_endpoint() { + let overlay = simple_overlay(Vec::new()); + let env_only = GatewayTelemetryConfig { + otlp_endpoint: None, + sampling_rate: None, + service_name: None, + service_version: None, + environment: None, + batch_interval_secs: None, + batch_size: None, + }; + let env_config = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + Some(&env_only), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + env_config.contains("telemetry: {}"), + "an environment-only exporter must render as an empty mapping, not null" + ); + + for rate in [-0.01, 1.01, f64::NAN, f64::INFINITY] { + let telemetry = GatewayTelemetryConfig { + sampling_rate: Some(rate), + ..valid_telemetry() + }; + let result = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + Some(&telemetry), + ); + assert!(result.is_err(), "invalid sampling rate {rate:?} must be rejected"); + } + + let telemetry = GatewayTelemetryConfig { + otlp_endpoint: Some("https://user:password@collector:4317".to_owned()), + ..valid_telemetry() + }; + assert!( + telemetry.validate().is_err(), + "collector authentication must use Secret-backed environment references" + ); + } + + /// Return a minimal valid telemetry setting for validation tests. + fn valid_telemetry() -> GatewayTelemetryConfig { + GatewayTelemetryConfig { + otlp_endpoint: Some("http://collector:4317".to_owned()), + sampling_rate: Some(0.5), + service_name: None, + service_version: None, + environment: None, + batch_interval_secs: None, + batch_size: None, + } + } + #[test] fn plain_candidates_produce_intelligent_route_and_load_balancer() { let overlay = simple_overlay(vec![plain_candidate( diff --git a/scripts/verify-helm-chart.sh b/scripts/verify-helm-chart.sh index 4e734d95d..38767c955 100755 --- a/scripts/verify-helm-chart.sh +++ b/scripts/verify-helm-chart.sh @@ -538,6 +538,14 @@ try_template "$GW_DIR" "subchart keys (gw)" "${GW_REQ[@]}" --set enabled=true -- try_reject "$GW_DIR" "runAsNonRoot override" "${GW_REQ[@]}" --set podSecurityContext.runAsNonRoot=false try_reject "$GW_DIR" "overlay enabled no name" "${GW_REQ[@]}" --set overlay.enabled=true try_reject "$GW_DIR" "tls enabled no secret" "${GW_REQ[@]}" --set tls.enabled=true --set tls.existingSecret="" +try_reject_msg "$GW_DIR" "telemetry sampling rate above one" \ + "gatewayConfig[./]telemetry[./]samplingRate.*(less than or equal to 1|maximum: got 1\\.01)" \ + --set gatewayConfig.telemetry.enabled=true --set-json gatewayConfig.telemetry.samplingRate=1.01 +try_reject_msg "$GW_DIR" "telemetry sampling rate below zero" \ + "gatewayConfig[./]telemetry[./]samplingRate.*(greater than or equal to 0|minimum: got -0\\.01)" \ + --set gatewayConfig.telemetry.enabled=true --set-json gatewayConfig.telemetry.samplingRate=-0.01 +try_reject "$GW_DIR" "telemetry endpoint query credentials" "${GW_REQ[@]}" \ + --set-string 'gatewayConfig.telemetry.otlpEndpoint=http://collector:4317?api_key=secret' # ── Secure gateway config (render) ────────────────────────────────── GW_RENDER=(--set gatewayConfig.render=true --set gatewayConfig.localSite=hub --set gatewayConfig.model=q --set gatewayConfig.auth.mode=none From 832d027e26267bc415111bf4c8766fb06e0d365f Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 12:39:00 +0000 Subject: [PATCH 2/9] fix(observability): align telemetry config and validation Signed-off-by: Brent Salisbury --- charts/praxis-gateway/README.md | 2 +- .../praxis-gateway/tests/telemetry_test.yaml | 47 ++++++++++++++ docs/architecture/opentelemetry.md | 63 +++++++------------ operator/src/crd/grid_network.rs | 48 ++++++++++++-- operator/src/resources/consumer_config.rs | 28 ++++++++- 5 files changed, 139 insertions(+), 49 deletions(-) diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index b0ae5f04b..e5b7ab368 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -207,7 +207,7 @@ Praxis AI image; these values may advance independently. | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | | `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. Praxis 0.7.1 exports HTTP server and upstream exchange spans, plus configured AI routing spans; its W3C headers do not link exported edge and provider spans. See [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | -| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP/gRPC endpoint without URL userinfo, query, or fragment credentials. Empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. | +| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP/gRPC endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. | | `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | | `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | | `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | Positive OTLP batch export interval and maximum batch size. | diff --git a/charts/praxis-gateway/tests/telemetry_test.yaml b/charts/praxis-gateway/tests/telemetry_test.yaml index 14a328b68..3082981de 100644 --- a/charts/praxis-gateway/tests/telemetry_test.yaml +++ b/charts/praxis-gateway/tests/telemetry_test.yaml @@ -106,6 +106,53 @@ tests: path: data["praxis.yaml"] pattern: "(?s)telemetry:\\n\\s+\\{\\}" + - it: treats an empty endpoint as environment fallback + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.otlpEndpoint: "" + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+\\{\\}" + - notMatchRegex: + path: data["praxis.yaml"] + pattern: "otlp_endpoint:" + + - it: rejects credentials in the endpoint userinfo + template: templates/gateway-config.yaml + set: + gatewayConfig.telemetry.otlpEndpoint: https://user:password@collector:4317 + asserts: + - failedTemplate: + errorPattern: "Does not match pattern" + + - it: rejects query strings on the endpoint + template: templates/gateway-config.yaml + set: + gatewayConfig.telemetry.otlpEndpoint: "https://collector:4317?api_key=sentinel" + asserts: + - failedTemplate: + errorPattern: "Does not match pattern" + + - it: rejects fragments on the endpoint + template: templates/gateway-config.yaml + set: + gatewayConfig.telemetry.otlpEndpoint: "https://collector:4317/otlp#token=sentinel" + asserts: + - failedTemplate: + errorPattern: "Does not match pattern" + - it: requires a Grid gateway image when telemetry is enabled template: templates/deployment.yaml set: diff --git a/docs/architecture/opentelemetry.md b/docs/architecture/opentelemetry.md index 46ff03cd5..b20f0504a 100644 --- a/docs/architecture/opentelemetry.md +++ b/docs/architecture/opentelemetry.md @@ -77,59 +77,38 @@ provider and flushes queued spans. ## Build and spans -The `grid-gateway` binary built by `deploy/gateway/Containerfile` includes the -Praxis 0.7.1 `otel` feature and the pinned Praxis AI v0.4.1 -`opentelemetry` feature. Use an image built from that target, published as -`ghcr.io/praxis-proxy/grid-gateway`, for this configuration. The chart's default -`ghcr.io/praxis-proxy/ai:0.4.0` image does not include the Grid build features. +The `grid-gateway` binary built by `deploy/gateway/Containerfile` enables the +Praxis `otel` feature and the Praxis AI v0.4.1 `opentelemetry` feature. The +tracked gateway lockfile currently resolves Praxis 0.7.1. Use an image built +from this Grid target, published as `ghcr.io/praxis-proxy/grid-gateway`, for +this configuration. The chart's default `ghcr.io/praxis-proxy/ai:0.4.0` image +does not include the Grid build features. With the AI feature enabled, these short semantic spans are supported when the corresponding filters run: -- `http_request` server spans and `upstream_exchange` internal spans from - Praxis 0.7.1's HTTP proxy protocol. +- HTTP server spans and `upstream_exchange` internal spans from Praxis 0.7.1. - `routing.select` from `intelligent_route`, including the serving overlay semantic revision when that revision is available to the filter. - `provider.route` from `provider_route`, including a validated edge overlay revision when present. -The pinned AI implementation projects bounded routing fields into those spans; +The AI implementation projects bounded routing fields into those spans; it does not add prompt or body contents, credential values, authorization headers, cookies, session keys, or raw request IDs. These spans describe route selection and resolution; `provider.route` does not prove the model backend served the request. -## Trace linkage status in Praxis 0.7.1 - -Praxis 0.7.1 exports each gateway's HTTP server span and an internal -`upstream_exchange` span. The AI `routing.select` and `provider.route` spans -also appear as children of their local gateway's server span. The proxy's -upstream span is named `upstream_exchange` with `INTERNAL` kind; Praxis 0.7.1 -does not export it as an HTTP client span. - -The `trace_context` filter separately validates inbound W3C `traceparent`, -stores that value in a Praxis request extension, and forwards its trace ID and -flags with a newly generated hop span ID. The filter's own source documentation -states that this hop ID does not name an exported span. Praxis 0.7.1 also does -not extract that extension into the OpenTelemetry context used by the HTTP -server span. This leaves two parallel contexts: exported spans use the local -OpenTelemetry request context, while forwarded headers use the filter's W3C -context. - -A collector-backed test used an incoming trace ID of -`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`. The edge exported a `POST` server span in -trace `89e767a01d701b4f04d1b7b12c022c8a`, and the provider exported a `POST` -server span in trace `abea135120a177a97b76273f37477f99`; both have no parent ID. -The edge `routing.select` span is a child of the edge server span, and the -provider `provider.route` span is a child of the provider server span. The -test backend received `traceparent` with the original `aaaa...` trace ID and a -fresh hop ID. The collector received spans only after normal SIGTERM shutdown, -confirming provider flush. Full request details and collector output are kept -outside the source tree in the PR evidence directory. - -This proves header propagation and local span export. It also proves that the -exact pinned Praxis version does not create exported cross-gateway parent/child -relationships. Issue #260 remains incomplete until Praxis provides a shared -W3C/OpenTelemetry context for inbound server spans and outbound client spans, -then a collector test confirms the resulting parent IDs across both gateways. -See the accompanying Praxis issue draft for the proposed dependency change. +## Cross-gateway trace linkage + +Praxis 0.7.1 forwards W3C trace headers but does not connect them to the +exported HTTP server span or emit an exported HTTP client span for the upstream +attempt. Edge and provider gateways therefore export separate local traces, +even when the backend receives a forwarded `traceparent`. The routing spans +remain local children of their gateway's HTTP server span. + +Cross-gateway exported parentage requires a Praxis framework release containing +the inbound context and upstream-attempt span changes. Once Grid updates its +gateway dependency to that release, rebuild the tracked image and verify the +collector's parent IDs across both gateways before claiming linked traces. +The Praxis AI routing-span feature alone does not supply that linkage. diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 7a66ab9b2..9dea1fd50 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -864,7 +864,7 @@ pub struct GatewayTelemetryConfig { /// OTLP collector endpoint, for example `http://otel-collector:4317`. /// /// If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the - /// gateway Deployment. + /// gateway Deployment. An empty string has the same meaning as omission. #[schemars(regex(pattern = r"^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$"))] #[serde(default, skip_serializing_if = "Option::is_none")] pub otlp_endpoint: Option, @@ -913,13 +913,15 @@ impl GatewayTelemetryConfig { )] pub fn validate(&self) -> Result<(), String> { if self.otlp_endpoint.as_ref().is_some_and(|endpoint| { + if endpoint.is_empty() { + return false; + } let endpoint = endpoint.trim(); let Some((scheme, rest)) = endpoint.split_once("://") else { return true; }; let authority = rest.split('/').next().unwrap_or_default(); - endpoint.is_empty() - || !matches!(scheme, "http" | "https") + !matches!(scheme, "http" | "https") || authority.is_empty() || authority.contains('@') || endpoint.contains('?') @@ -1724,7 +1726,7 @@ mod tests { } #[test] - fn telemetry_config_rejects_embedded_headers_and_malformed_endpoints() { + fn telemetry_config_rejects_embedded_headers() { let with_headers = serde_json::json!({ "otlpEndpoint": "http://collector:4317", "otlpHeaders": {"authorization": "test-value"} @@ -1733,7 +1735,22 @@ mod tests { serde_json::from_value::(with_headers).is_err(), "collector credentials must not be accepted as config fields" ); + } + + #[test] + fn telemetry_config_accepts_empty_endpoint_fallback() { + let empty_endpoint = serde_json::from_value::(serde_json::json!({ + "otlpEndpoint": "" + })) + .unwrap_or_else(|_| std::process::abort()); + assert!( + empty_endpoint.validate().is_ok(), + "an empty endpoint must use the deployment environment fallback" + ); + } + #[test] + fn telemetry_config_rejects_credentials_in_endpoint() { for endpoint in [ "https://user:password@collector:4317", "https://collector:4317?api_key=secret", @@ -1812,6 +1829,29 @@ mod tests { Some(1.0), "CRD schema must reject sampling rates above one" ); + let endpoint_schema = telemetry_properties + .get("otlpEndpoint") + .unwrap_or_else(|| std::process::abort()); + assert_eq!( + endpoint_schema.get("type").and_then(serde_json::Value::as_str), + Some("string"), + "OTLP endpoint remains an optional string" + ); + assert!( + !telemetry_properties + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and(|required| required.iter().any(|field| field.as_str() == Some("otlpEndpoint"))), + "omitted endpoint must remain valid for environment fallback" + ); + let endpoint_pattern = endpoint_schema + .get("pattern") + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| std::process::abort()); + assert!( + endpoint_pattern.starts_with("^(|") && endpoint_pattern.contains("https?://"), + "CRD schema must accept empty environment fallback and explicit HTTP(S) URLs" + ); } #[test] diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index 505f28634..d7da6b1ef 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -244,7 +244,7 @@ pub(crate) fn generate_consumer_praxis_config_with_telemetry( reason = "serializes every optional telemetry value in a stable YAML order" )] fn render_telemetry(telemetry: &GatewayTelemetryConfig) -> Result { - if telemetry.otlp_endpoint.is_none() + if telemetry.otlp_endpoint.as_deref().is_none_or(str::is_empty) && telemetry.sampling_rate.is_none() && telemetry.service_name.is_none() && telemetry.service_version.is_none() @@ -255,7 +255,11 @@ fn render_telemetry(telemetry: &GatewayTelemetryConfig) -> Result Date: Sat, 3 Oct 2026 18:22:55 +0000 Subject: [PATCH 3/9] test(helm): verify telemetry URL rejection across Helm versions Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 2 +- .../praxis-gateway/tests/telemetry_test.yaml | 24 ------------------- deploy/crds/gridnetwork.yaml | 2 +- scripts/verify-helm-chart.sh | 11 +++++++-- 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index e89b5310f..b8a6db84c 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -320,7 +320,7 @@ spec: OTLP collector endpoint, for example `http://otel-collector:4317`. If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the - gateway Deployment. + gateway Deployment. An empty string has the same meaning as omission. nullable: true pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ type: string diff --git a/charts/praxis-gateway/tests/telemetry_test.yaml b/charts/praxis-gateway/tests/telemetry_test.yaml index 3082981de..4dbf41d4d 100644 --- a/charts/praxis-gateway/tests/telemetry_test.yaml +++ b/charts/praxis-gateway/tests/telemetry_test.yaml @@ -129,30 +129,6 @@ tests: path: data["praxis.yaml"] pattern: "otlp_endpoint:" - - it: rejects credentials in the endpoint userinfo - template: templates/gateway-config.yaml - set: - gatewayConfig.telemetry.otlpEndpoint: https://user:password@collector:4317 - asserts: - - failedTemplate: - errorPattern: "Does not match pattern" - - - it: rejects query strings on the endpoint - template: templates/gateway-config.yaml - set: - gatewayConfig.telemetry.otlpEndpoint: "https://collector:4317?api_key=sentinel" - asserts: - - failedTemplate: - errorPattern: "Does not match pattern" - - - it: rejects fragments on the endpoint - template: templates/gateway-config.yaml - set: - gatewayConfig.telemetry.otlpEndpoint: "https://collector:4317/otlp#token=sentinel" - asserts: - - failedTemplate: - errorPattern: "Does not match pattern" - - it: requires a Grid gateway image when telemetry is enabled template: templates/deployment.yaml set: diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 2213ff0f5..79e314693 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -312,7 +312,7 @@ spec: OTLP collector endpoint, for example `http://otel-collector:4317`. If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the - gateway Deployment. + gateway Deployment. An empty string has the same meaning as omission. nullable: true pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ type: string diff --git a/scripts/verify-helm-chart.sh b/scripts/verify-helm-chart.sh index 38767c955..54e8c8f92 100755 --- a/scripts/verify-helm-chart.sh +++ b/scripts/verify-helm-chart.sh @@ -544,8 +544,15 @@ try_reject_msg "$GW_DIR" "telemetry sampling rate above one" \ try_reject_msg "$GW_DIR" "telemetry sampling rate below zero" \ "gatewayConfig[./]telemetry[./]samplingRate.*(greater than or equal to 0|minimum: got -0\\.01)" \ --set gatewayConfig.telemetry.enabled=true --set-json gatewayConfig.telemetry.samplingRate=-0.01 -try_reject "$GW_DIR" "telemetry endpoint query credentials" "${GW_REQ[@]}" \ - --set-string 'gatewayConfig.telemetry.otlpEndpoint=http://collector:4317?api_key=secret' +try_reject_msg "$GW_DIR" "telemetry endpoint userinfo credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://user:password@collector:4317' +try_reject_msg "$GW_DIR" "telemetry endpoint query credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://collector:4317?api_key=sentinel' +try_reject_msg "$GW_DIR" "telemetry endpoint fragment credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://collector:4317/otlp#token=sentinel' # ── Secure gateway config (render) ────────────────────────────────── GW_RENDER=(--set gatewayConfig.render=true --set gatewayConfig.localSite=hub --set gatewayConfig.model=q --set gatewayConfig.auth.mode=none From 7ec16316b8d51946461602efee954bfcbadd70ce Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 20:11:22 +0000 Subject: [PATCH 4/9] docs(otel): align trace claims and telemetry schema Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 6 ++-- charts/praxis-gateway/README.md | 2 +- deploy/crds/gridnetwork.yaml | 6 ++-- operator/src/crd/grid_network.rs | 33 +++++++++++++++++-- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index b8a6db84c..d0e090e09 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -313,7 +313,7 @@ spec: environment: description: Deployment environment resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string otlpEndpoint: description: |- @@ -334,12 +334,12 @@ spec: serviceName: description: OpenTelemetry `service.name` resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string serviceVersion: description: OpenTelemetry `service.version` resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string type: object tlsCertMountPath: diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index e5b7ab368..ab7a5ac3b 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -206,7 +206,7 @@ Praxis AI image; these values may advance independently. | `config.key` | string | `praxis.yaml` | Key in the ConfigMap. | | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | -| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. Praxis 0.7.1 exports HTTP server and upstream exchange spans, plus configured AI routing spans; its W3C headers do not link exported edge and provider spans. See [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | +| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.1, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | | `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP/gRPC endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. | | `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | | `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 79e314693..8016fd5cc 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -305,7 +305,7 @@ spec: environment: description: Deployment environment resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string otlpEndpoint: description: |- @@ -326,12 +326,12 @@ spec: serviceName: description: OpenTelemetry `service.name` resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string serviceVersion: description: OpenTelemetry `service.version` resource attribute. nullable: true - pattern: ^(|.*\S.*)$ + pattern: ^.*\S.*$ type: string type: object tlsCertMountPath: diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 9dea1fd50..a84474eb3 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -875,17 +875,17 @@ pub struct GatewayTelemetryConfig { pub sampling_rate: Option, /// OpenTelemetry `service.name` resource attribute. - #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[schemars(regex(pattern = r"^.*\S.*$"))] #[serde(default, skip_serializing_if = "Option::is_none")] pub service_name: Option, /// OpenTelemetry `service.version` resource attribute. - #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[schemars(regex(pattern = r"^.*\S.*$"))] #[serde(default, skip_serializing_if = "Option::is_none")] pub service_version: Option, /// Deployment environment resource attribute. - #[schemars(regex(pattern = r"^(|.*\S.*)$"))] + #[schemars(regex(pattern = r"^.*\S.*$"))] #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option, @@ -1749,6 +1749,33 @@ mod tests { ); } + #[test] + fn telemetry_resource_attributes_reject_blank_values_at_admission_and_runtime() { + let crd = crd_json(); + let telemetry_properties = crd + .pointer( + "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/gatewayRefs/items/properties/consumerConfig/properties/telemetry/properties", + ) + .unwrap_or_else(|| std::process::abort()); + for field in ["serviceName", "serviceVersion", "environment"] { + let pattern = telemetry_properties + .get(field) + .and_then(|property| property.get("pattern")) + .and_then(serde_json::Value::as_str); + assert_eq!( + pattern, + Some(r"^.*\S.*$"), + "{field} must reject blank values at admission" + ); + let telemetry: GatewayTelemetryConfig = + serde_json::from_value(serde_json::json!({(field): ""})).unwrap_or_else(|_| std::process::abort()); + assert!( + telemetry.validate().is_err(), + "{field} must reject blank values at runtime" + ); + } + } + #[test] fn telemetry_config_rejects_credentials_in_endpoint() { for endpoint in [ From 2e51473ba10f7691bf05cedd8a2269a7aa64dc2a Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Mon, 5 Oct 2026 01:58:44 +0000 Subject: [PATCH 5/9] fix(gateway): lock OpenTelemetry dependencies Signed-off-by: Brent Salisbury --- gateway/Cargo.lock | 384 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 9bf30a3b4..77c3cbcde 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -310,6 +310,49 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -1643,9 +1686,11 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1653,12 +1698,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ + "base64 0.23.1", "bytes", "futures-channel", "futures-util", @@ -1666,7 +1725,9 @@ dependencies = [ "http-body", "httparse", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1851,6 +1912,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2176,6 +2243,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" @@ -2256,6 +2329,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2525,6 +2604,83 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-http" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +dependencies = [ + "http", + "httpdate", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ouroboros" version = "0.18.5" @@ -2653,6 +2809,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2951,6 +3127,9 @@ dependencies = [ "dashmap", "http", "metrics", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "percent-encoding", "praxis-proxy-tls", "quixotic-plecostomus-core", @@ -2960,8 +3139,10 @@ dependencies = [ "serde", "thiserror", "tokio", + "tonic", "tracing", "tracing-appender", + "tracing-opentelemetry", "tracing-subscriber", "yaml_serde", ] @@ -2980,6 +3161,7 @@ dependencies = [ "http", "metrics", "openssl", + "opentelemetry", "parking_lot", "percent-encoding", "praxis-policy", @@ -2996,6 +3178,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-opentelemetry", "yaml_serde", "zeroize", ] @@ -3087,6 +3270,38 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "psm" version = "0.1.32" @@ -3569,6 +3784,35 @@ dependencies = [ "yaml_serde", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -4156,6 +4400,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4371,6 +4624,100 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.2", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4433,6 +4780,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -4716,6 +5079,17 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.129" @@ -4758,6 +5132,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wildmatch" version = "2.6.1" From 29326b782a55ac1d206ee3bcecbd2dd5f3e23788 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Mon, 5 Oct 2026 03:29:01 +0000 Subject: [PATCH 6/9] fix(otel): enforce telemetry bounds and endpoint security Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 6 +- charts/praxis-gateway/README.md | 4 +- charts/praxis-gateway/values.schema.json | 4 +- deploy/crds/gridnetwork.yaml | 6 +- gateway/src/main.rs | 104 +++++++++++++++++- operator/src/crd/grid_network.rs | 74 +++++++++++-- 6 files changed, 179 insertions(+), 19 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index d0e090e09..ed194ce94 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -299,14 +299,16 @@ spec: nullable: true properties: batchIntervalSecs: - description: Batch exporter interval in seconds. Must be positive when set. + description: Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. format: uint64 + maximum: 300.0 minimum: 1.0 nullable: true type: integer batchSize: - description: Maximum spans per export batch. Must be positive when set. + description: Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. format: uint + maximum: 65536.0 minimum: 1.0 nullable: true type: integer diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index ab7a5ac3b..13acae8ca 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -207,10 +207,10 @@ Praxis AI image; these values may advance independently. | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | | `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.1, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | -| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP/gRPC endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. | +| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If `OTEL_EXPORTER_OTLP_HEADERS` is set, the resolved endpoint must use HTTPS; the gateway refuses to send exporter headers over HTTP. | | `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | | `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | -| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | Positive OTLP batch export interval and maximum batch size. | +| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1–300 seconds and maximum batch size from 1–65,536 spans. | | `gatewayConfig.model` | string | **required** for a consumer without `gridServing` | Model advertised on the routing candidates. | | `gatewayConfig.backends` | map | **required** when rendered | Backends keyed by site, each with `endpoint` and optional `healthCheck` and `transport`. A consumer's key is the site it reaches over mutual TLS. A provider's `local` key is its one plaintext backend. The older list of `cluster`, `endpoints` entries still renders. | | `gatewayConfig.backends[].site` | string | `localSite` | Grid site the backend serves. A consumer's remote `mutual_tls` backend must name it, and it must differ from `localSite`. Its `transport.sni` defaults to `.grid.internal`. | diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index c44ad8444..aa14c3cc0 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -188,8 +188,8 @@ "serviceName": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.name resource attribute." }, "serviceVersion": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.version resource attribute." }, "environment": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "deployment.environment resource attribute." }, - "batchIntervalSecs": { "type": ["integer", "null"], "minimum": 1, "description": "Positive batch exporter interval in seconds." }, - "batchSize": { "type": ["integer", "null"], "minimum": 1, "description": "Positive maximum spans per export batch." } + "batchIntervalSecs": { "type": ["integer", "null"], "minimum": 1, "maximum": 300, "description": "Positive batch exporter interval in seconds." }, + "batchSize": { "type": ["integer", "null"], "minimum": 1, "maximum": 65536, "description": "Positive maximum spans per export batch." } } } } diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 8016fd5cc..0b8a88e2a 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -291,14 +291,16 @@ spec: nullable: true properties: batchIntervalSecs: - description: Batch exporter interval in seconds. Must be positive when set. + description: Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. format: uint64 + maximum: 300.0 minimum: 1.0 nullable: true type: integer batchSize: - description: Maximum spans per export batch. Must be positive when set. + description: Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. format: uint + maximum: 65536.0 minimum: 1.0 nullable: true type: integer diff --git a/gateway/src/main.rs b/gateway/src/main.rs index b88439233..36fb4d6b2 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -19,6 +19,10 @@ use tracing::info; /// Log line emitted once tracing is up; the startup test waits for it. const STARTUP_MESSAGE: &str = "starting grid-gateway"; +/// OTel-standard environment variable used as the OTLP endpoint fallback. +const OTLP_ENDPOINT_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; +/// OTel-standard environment variable containing exporter headers. +const OTLP_HEADERS_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_HEADERS"; fn main() -> ExitCode { // Install the crypto provider before anything builds a TLS config. @@ -36,6 +40,8 @@ fn main() -> ExitCode { let config = praxis::with_bootstrap_logging(|| Config::from_config_file_or(config_file.as_ref(), DEFAULT_CONFIG)) .unwrap_or_else(|err| praxis::fatal(&err)); + validate_otlp_endpoint_transport(&config).unwrap_or_else(|err| praxis::fatal(&err)); + // Without a subscriber every log line, including reload results, is dropped. let tracing_guard = praxis::init_tracing(&config).unwrap_or_else(|err| praxis::fatal(&err)); let log_level = Some(tracing_guard.log_level_state()); @@ -69,6 +75,45 @@ fn main() -> ExitCode { exit_code } +/// Refuse to send configured OTLP headers to an unencrypted HTTP endpoint. +/// +/// Praxis resolves the endpoint and headers from config first, then environment +/// variables. Validate the same effective values before initializing its +/// exporter so Secret-backed `OTEL_EXPORTER_OTLP_HEADERS` cannot be sent in +/// cleartext through either endpoint source. +fn validate_otlp_endpoint_transport(config: &Config) -> Result<(), &'static str> { + let environment_endpoint = std::env::var(OTLP_ENDPOINT_ENV_VAR).ok(); + let environment_headers_present = std::env::var_os(OTLP_HEADERS_ENV_VAR).is_some_and(|value| !value.is_empty()); + let configured_headers_present = config.telemetry.otlp_headers.as_ref().map(|headers| !headers.is_empty()); + + validate_otlp_endpoint_transport_values( + config.telemetry.otlp_endpoint.as_deref(), + environment_endpoint.as_deref(), + configured_headers_present, + environment_headers_present, + ) +} + +/// Validate endpoint/header pairs using Praxis's config-before-environment precedence. +fn validate_otlp_endpoint_transport_values( + configured_endpoint: Option<&str>, + environment_endpoint: Option<&str>, + configured_headers_present: Option, + environment_headers_present: bool, +) -> Result<(), &'static str> { + let endpoint = configured_endpoint.or_else(|| environment_endpoint.filter(|value| !value.trim().is_empty())); + let headers_present = configured_headers_present.unwrap_or(environment_headers_present); + let endpoint_uses_http = endpoint + .and_then(|value| value.trim().split_once("://")) + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("http")); + + if headers_present && endpoint_uses_http { + return Err("OTLP exporter headers require HTTPS; refusing to send OTLP credentials over HTTP"); + } + + Ok(()) +} + /// Start the cross-site pollers and register `grid_site_route` over their snapshot. /// /// # Errors @@ -120,7 +165,7 @@ fn config_arg>(args: I) -> Result, #[cfg(test)] mod tests { - use super::{USAGE, config_arg}; + use super::{USAGE, config_arg, validate_otlp_endpoint_transport_values}; fn parse(args: &[&str]) -> Result, String> { config_arg(args.iter().map(|arg| (*arg).to_owned())) @@ -157,4 +202,61 @@ mod tests { assert_eq!(parse(args), Err(USAGE.to_owned()), "{args:?}"); } } + + #[test] + fn rejects_http_endpoint_when_otlp_headers_are_configured() { + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, Some(true), false).is_err(), + "an explicitly configured HTTP endpoint must not receive headers" + ); + } + + #[test] + fn rejects_http_environment_fallback_when_otlp_headers_are_configured() { + assert!( + validate_otlp_endpoint_transport_values(None, Some("http://collector:4317"), None, true).is_err(), + "the OTEL_EXPORTER_OTLP_ENDPOINT fallback must not receive headers over HTTP" + ); + } + + #[test] + fn accepts_https_endpoints_with_otlp_headers() { + for (configured, fallback, configured_headers, environment_headers) in [ + (Some("https://collector:4317"), None, Some(true), false), + (None, Some("https://collector:4317"), None, true), + ] { + assert!( + validate_otlp_endpoint_transport_values( + configured, + fallback, + configured_headers, + environment_headers, + ) + .is_ok(), + "HTTPS endpoints must remain usable with headers" + ); + } + } + + #[test] + fn preserves_http_behavior_when_otlp_headers_are_absent() { + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, None, false).is_ok(), + "HTTP without exporter headers remains supported" + ); + } + + #[test] + fn configured_endpoint_takes_precedence_over_environment_fallback() { + assert!( + validate_otlp_endpoint_transport_values( + Some("https://configured:4317"), + Some("http://fallback:4317"), + None, + true, + ) + .is_ok(), + "an unused HTTP fallback must not reject the configured HTTPS endpoint" + ); + } } diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index a84474eb3..804a7c64e 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -852,6 +852,11 @@ pub struct ConsumerConfig { pub telemetry: Option, } +/// Maximum batch interval accepted by Praxis telemetry configuration. +const MAX_TELEMETRY_BATCH_INTERVAL_SECS: u64 = 300; +/// Maximum batch size accepted by Praxis telemetry configuration. +const MAX_TELEMETRY_BATCH_SIZE: usize = 65_536; + /// Validated OpenTelemetry settings rendered into operator-generated Praxis YAML. /// /// Secret material is intentionally not part of this type. Supply collector @@ -889,13 +894,13 @@ pub struct GatewayTelemetryConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option, - /// Batch exporter interval in seconds. Must be positive when set. - #[schemars(range(min = 1))] + /// Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. + #[schemars(range(min = 1, max = 300))] #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_interval_secs: Option, - /// Maximum spans per export batch. Must be positive when set. - #[schemars(range(min = 1))] + /// Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. + #[schemars(range(min = 1, max = 65_536))] #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_size: Option, } @@ -905,8 +910,8 @@ impl GatewayTelemetryConfig { /// /// # Errors /// - /// Returns an explanatory message for empty strings, invalid sampling - /// rates, or zero-valued batch settings. + /// Returns an explanatory message for invalid strings, sampling rates, or + /// batch settings outside their supported ranges. #[expect( clippy::too_many_lines, reason = "validates all telemetry settings before CRD rendering" @@ -935,11 +940,21 @@ impl GatewayTelemetryConfig { { return Err("telemetry.samplingRate must be between 0.0 and 1.0".to_owned()); } - if self.batch_interval_secs == Some(0) { - return Err("telemetry.batchIntervalSecs must be greater than zero".to_owned()); + if self + .batch_interval_secs + .is_some_and(|seconds| !(1..=MAX_TELEMETRY_BATCH_INTERVAL_SECS).contains(&seconds)) + { + return Err(format!( + "telemetry.batchIntervalSecs must be between 1 and {MAX_TELEMETRY_BATCH_INTERVAL_SECS}" + )); } - if self.batch_size == Some(0) { - return Err("telemetry.batchSize must be greater than zero".to_owned()); + if self + .batch_size + .is_some_and(|size| !(1..=MAX_TELEMETRY_BATCH_SIZE).contains(&size)) + { + return Err(format!( + "telemetry.batchSize must be between 1 and {MAX_TELEMETRY_BATCH_SIZE}" + )); } for (field, value) in [ ("serviceName", self.service_name.as_deref()), @@ -1749,6 +1764,45 @@ mod tests { ); } + #[test] + fn telemetry_batch_bounds_appear_in_crd_schema() { + let crd = crd_json(); + let properties = crd + .pointer( + "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/gatewayRefs/items/properties/consumerConfig/properties/telemetry/properties", + ) + .unwrap_or_else(|| std::process::abort()); + + for (field, maximum) in [("batchIntervalSecs", 300.0), ("batchSize", 65_536.0)] { + let schema = properties.get(field).unwrap_or_else(|| std::process::abort()); + assert_eq!( + schema.get("minimum").and_then(serde_json::Value::as_f64), + Some(1.0), + "{field} CRD schema must enforce the inclusive lower bound" + ); + assert_eq!( + schema.get("maximum").and_then(serde_json::Value::as_f64), + Some(maximum), + "{field} CRD schema must enforce the inclusive upper bound" + ); + } + } + + #[test] + fn telemetry_batch_bounds_are_enforced_at_runtime() { + for (field, maximum) in [("batchIntervalSecs", 300_u64), ("batchSize", 65_536_u64)] { + for (value, expected_valid) in [(0, false), (1, true), (maximum, true), (maximum + 1, false)] { + let telemetry = serde_json::from_value::(serde_json::json!({(field): value})) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + telemetry.validate().is_ok(), + expected_valid, + "{field}={value} runtime validation mismatch" + ); + } + } + } + #[test] fn telemetry_resource_attributes_reject_blank_values_at_admission_and_runtime() { let crd = crd_json(); From eedb73c485612b521f1cb9a1bc46b9c79389598f Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Mon, 5 Oct 2026 03:38:29 +0000 Subject: [PATCH 7/9] style(gateway): satisfy pinned rustfmt Signed-off-by: Brent Salisbury --- .../grid-operator/templates/crds/gridnetwork.yaml | 4 ++-- charts/praxis-gateway/README.md | 2 +- deploy/crds/gridnetwork.yaml | 4 ++-- docs/README.md | 2 +- gateway/src/main.rs | 15 +++++++-------- operator/src/crd/grid_network.rs | 4 ++-- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index ed194ce94..46ed6bdb2 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -299,14 +299,14 @@ spec: nullable: true properties: batchIntervalSecs: - description: Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. + description: Batch exporter interval in seconds. Must be from 1 through 300 when set. format: uint64 maximum: 300.0 minimum: 1.0 nullable: true type: integer batchSize: - description: Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. + description: Maximum spans per export batch. Must be from 1 through 65,536 when set. format: uint maximum: 65536.0 minimum: 1.0 diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index 13acae8ca..6c714bba3 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -210,7 +210,7 @@ Praxis AI image; these values may advance independently. | `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If `OTEL_EXPORTER_OTLP_HEADERS` is set, the resolved endpoint must use HTTPS; the gateway refuses to send exporter headers over HTTP. | | `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | | `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | -| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1–300 seconds and maximum batch size from 1–65,536 spans. | +| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1 through 300 seconds and maximum batch size from 1 through 65,536 spans. | | `gatewayConfig.model` | string | **required** for a consumer without `gridServing` | Model advertised on the routing candidates. | | `gatewayConfig.backends` | map | **required** when rendered | Backends keyed by site, each with `endpoint` and optional `healthCheck` and `transport`. A consumer's key is the site it reaches over mutual TLS. A provider's `local` key is its one plaintext backend. The older list of `cluster`, `endpoints` entries still renders. | | `gatewayConfig.backends[].site` | string | `localSite` | Grid site the backend serves. A consumer's remote `mutual_tls` backend must name it, and it must differ from `localSite`. Its `transport.sni` defaults to `.grid.internal`. | diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 0b8a88e2a..d6b082a68 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -291,14 +291,14 @@ spec: nullable: true properties: batchIntervalSecs: - description: Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. + description: Batch exporter interval in seconds. Must be from 1 through 300 when set. format: uint64 maximum: 300.0 minimum: 1.0 nullable: true type: integer batchSize: - description: Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. + description: Maximum spans per export batch. Must be from 1 through 65,536 when set. format: uint maximum: 65536.0 minimum: 1.0 diff --git a/docs/README.md b/docs/README.md index ba75c5a7e..6dc5458ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,7 +19,7 @@ access policy, and trust model. - [Consumer Config](architecture/consumer-config.md) — operator-generated consumer Praxis `ConfigMap` and the `GatewayRef.consumerConfig` API. -- [OpenTelemetry](architecture/opentelemetry.md) — exporter configuration, +- [OpenTelemetry](architecture/opentelemetry.md) - exporter configuration, secret handling, image requirements, and the Praxis 0.7.1 trace-linkage limit. - [External Client Ingress](architecture/external-ingress.md) — GTM/GLB edge selection, AGN provider routing, trust boundaries, affinity, snapshot diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 36fb4d6b2..621e51a73 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -84,7 +84,11 @@ fn main() -> ExitCode { fn validate_otlp_endpoint_transport(config: &Config) -> Result<(), &'static str> { let environment_endpoint = std::env::var(OTLP_ENDPOINT_ENV_VAR).ok(); let environment_headers_present = std::env::var_os(OTLP_HEADERS_ENV_VAR).is_some_and(|value| !value.is_empty()); - let configured_headers_present = config.telemetry.otlp_headers.as_ref().map(|headers| !headers.is_empty()); + let configured_headers_present = config + .telemetry + .otlp_headers + .as_ref() + .map(|headers| !headers.is_empty()); validate_otlp_endpoint_transport_values( config.telemetry.otlp_endpoint.as_deref(), @@ -226,13 +230,8 @@ mod tests { (None, Some("https://collector:4317"), None, true), ] { assert!( - validate_otlp_endpoint_transport_values( - configured, - fallback, - configured_headers, - environment_headers, - ) - .is_ok(), + validate_otlp_endpoint_transport_values(configured, fallback, configured_headers, environment_headers,) + .is_ok(), "HTTPS endpoints must remain usable with headers" ); } diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 804a7c64e..549c1e899 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -894,12 +894,12 @@ pub struct GatewayTelemetryConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option, - /// Batch exporter interval in seconds. Must be in the inclusive range 1–300 when set. + /// Batch exporter interval in seconds. Must be from 1 through 300 when set. #[schemars(range(min = 1, max = 300))] #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_interval_secs: Option, - /// Maximum spans per export batch. Must be in the inclusive range 1–65,536 when set. + /// Maximum spans per export batch. Must be from 1 through 65,536 when set. #[schemars(range(min = 1, max = 65_536))] #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_size: Option, From 27a3f113a4c993de1c60375f4b4ff4b1ce9294e4 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Tue, 6 Oct 2026 04:14:17 +0000 Subject: [PATCH 8/9] fix(gateway): guard credentialed OTLP transport Signed-off-by: Brent Salisbury --- charts/praxis-gateway/README.md | 4 +- docs/architecture/opentelemetry.md | 22 ++++- gateway/Cargo.lock | 2 + gateway/Cargo.toml | 3 +- gateway/src/main.rs | 134 ++++++++++++++++++++++------- gateway/tests/startup_log.rs | 42 ++++++++- 6 files changed, 164 insertions(+), 43 deletions(-) diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index 6c714bba3..a1ad2bcd8 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -206,8 +206,8 @@ Praxis AI image; these values may advance independently. | `config.key` | string | `praxis.yaml` | Key in the ConfigMap. | | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | -| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.1, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | -| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If `OTEL_EXPORTER_OTLP_HEADERS` is set, the resolved endpoint must use HTTPS; the gateway refuses to send exporter headers over HTTP. | +| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP/gRPC export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.3, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | +| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If configured or generic/trace-specific exporter headers are present, the endpoint must explicitly use `https://`; scheme-less endpoints are rejected. | | `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | | `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | | `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1 through 300 seconds and maximum batch size from 1 through 65,536 spans. | diff --git a/docs/architecture/opentelemetry.md b/docs/architecture/opentelemetry.md index b20f0504a..910d94893 100644 --- a/docs/architecture/opentelemetry.md +++ b/docs/architecture/opentelemetry.md @@ -63,11 +63,25 @@ env: key: headers ``` -The Secret value uses the OpenTelemetry `key=value` header format. The operator +The Secret value uses the OpenTelemetry `key=value` header format. Generic +`OTEL_EXPORTER_OTLP_HEADERS`, trace-specific +`OTEL_EXPORTER_OTLP_TRACES_HEADERS`, and configured `otlp_headers` all require +an explicitly specified `https://` collector endpoint. A scheme-less endpoint +is rejected when headers are present, regardless of the OTLP `INSECURE` +environment settings. This gateway build supports OTLP/gRPC; it rejects +`OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` while the locked Praxis HTTP +exporter and batch processor are incompatible. The operator only creates the consumer `ConfigMap`; the deployment manager must add the same Secret-backed environment reference to its gateway Deployment. Praxis redacts OTLP header values from its config debug representation. +The HTTPS preflight protects exporter headers from plaintext transport; it +does not prove certificate-verified delivery. Credentialed HTTPS export +requires the Praxis trust fix in +[praxis#1354](https://github.com/praxis-proxy/praxis/issues/1354), a Praxis +release containing it, and the Grid qualification tracked in +[Grid #301](https://github.com/praxis-proxy/grid/issues/301). + Exporter setup runs once at process startup. Helm rolls gateway pods when its telemetry values change. An externally managed consumer Deployment must be restarted after its generated ConfigMap changes, and a gateway must be restarted @@ -79,7 +93,7 @@ provider and flushes queued spans. The `grid-gateway` binary built by `deploy/gateway/Containerfile` enables the Praxis `otel` feature and the Praxis AI v0.4.1 `opentelemetry` feature. The -tracked gateway lockfile currently resolves Praxis 0.7.1. Use an image built +tracked gateway lockfile currently resolves Praxis 0.7.3. Use an image built from this Grid target, published as `ghcr.io/praxis-proxy/grid-gateway`, for this configuration. The chart's default `ghcr.io/praxis-proxy/ai:0.4.0` image does not include the Grid build features. @@ -87,7 +101,7 @@ does not include the Grid build features. With the AI feature enabled, these short semantic spans are supported when the corresponding filters run: -- HTTP server spans and `upstream_exchange` internal spans from Praxis 0.7.1. +- HTTP server spans and `upstream_exchange` internal spans from Praxis 0.7.3. - `routing.select` from `intelligent_route`, including the serving overlay semantic revision when that revision is available to the filter. - `provider.route` from `provider_route`, including a validated edge overlay @@ -101,7 +115,7 @@ served the request. ## Cross-gateway trace linkage -Praxis 0.7.1 forwards W3C trace headers but does not connect them to the +The locked Praxis 0.7.3 build forwards W3C trace headers but does not connect them to the exported HTTP server span or emit an exported HTTP client span for the upstream attempt. Edge and provider gateways therefore export separate local traces, even when the backend receives a forwarded `traceparent`. The routing spans diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 77c3cbcde..2d370395b 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -1424,6 +1424,7 @@ version = "0.1.4" dependencies = [ "ai-grid-filters", "certs", + "opentelemetry-otlp", "praxis-ai-filters", "praxis-proxy", "praxis-proxy-core", @@ -4646,6 +4647,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 848e0ba01..91c8b723e 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -311,6 +311,7 @@ path = "src/main.rs" praxis-proxy = { version = "0.7.3", features = ["otel", "spiffe"] } praxis-proxy-filter = "0.7.3" praxis-proxy-core = "0.7.3" +opentelemetry-otlp = { version = "0.33.0", default-features = false, features = ["tls-provider-agnostic"] } praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false, features = ["opentelemetry"] } ai-grid-filters = { path = "ai-grid-filters" } tracing = "0.1.44" @@ -322,7 +323,7 @@ rustls = { version = "0.23.45", features = ["ring"] } # machete can't see these: used as `praxis` / `praxis_core` / `praxis_filter`, not their package names. [package.metadata.cargo-machete] -ignored = ["praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter"] +ignored = ["opentelemetry-otlp", "praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter"] [lints] workspace = true diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 621e51a73..1197165b3 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -12,7 +12,7 @@ //! Kubernetes client stack. See `deploy/gateway/Containerfile` and the //! `gateway-image` make target. -use std::process::ExitCode; +use std::{ffi::OsStr, process::ExitCode}; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; use tracing::info; @@ -23,6 +23,10 @@ const STARTUP_MESSAGE: &str = "starting grid-gateway"; const OTLP_ENDPOINT_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; /// OTel-standard environment variable containing exporter headers. const OTLP_HEADERS_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_HEADERS"; +/// Signal-specific headers are merged into the exporter after Praxis config. +const OTLP_TRACES_HEADERS_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_TRACES_HEADERS"; +/// Praxis currently selects the exporter protocol from this variable. +const OTLP_PROTOCOL_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_PROTOCOL"; fn main() -> ExitCode { // Install the crypto provider before anything builds a TLS config. @@ -75,44 +79,57 @@ fn main() -> ExitCode { exit_code } -/// Refuse to send configured OTLP headers to an unencrypted HTTP endpoint. +/// Refuse credentialed export without explicit TLS and unsupported HTTP export. /// -/// Praxis resolves the endpoint and headers from config first, then environment -/// variables. Validate the same effective values before initializing its -/// exporter so Secret-backed `OTEL_EXPORTER_OTLP_HEADERS` cannot be sent in -/// cleartext through either endpoint source. +/// The OTLP SDK can merge generic or trace-specific environment headers into +/// programmatic headers. Treat every source as potentially credential-bearing. fn validate_otlp_endpoint_transport(config: &Config) -> Result<(), &'static str> { let environment_endpoint = std::env::var(OTLP_ENDPOINT_ENV_VAR).ok(); - let environment_headers_present = std::env::var_os(OTLP_HEADERS_ENV_VAR).is_some_and(|value| !value.is_empty()); - let configured_headers_present = config - .telemetry - .otlp_headers - .as_ref() - .map(|headers| !headers.is_empty()); + let generic_headers = std::env::var_os(OTLP_HEADERS_ENV_VAR); + let traces_headers = std::env::var_os(OTLP_TRACES_HEADERS_ENV_VAR); + let headers_present = otlp_headers_present( + config + .telemetry + .otlp_headers + .as_ref() + .is_some_and(|headers| !headers.is_empty()), + generic_headers.as_deref(), + traces_headers.as_deref(), + ); + let protocol = std::env::var(OTLP_PROTOCOL_ENV_VAR).ok(); validate_otlp_endpoint_transport_values( config.telemetry.otlp_endpoint.as_deref(), environment_endpoint.as_deref(), - configured_headers_present, - environment_headers_present, + headers_present, + protocol.as_deref(), ) } -/// Validate endpoint/header pairs using Praxis's config-before-environment precedence. +/// The SDK may append either environment source even when config has an empty map. +fn otlp_headers_present(configured: bool, generic: Option<&OsStr>, traces: Option<&OsStr>) -> bool { + configured || generic.is_some_and(|value| !value.is_empty()) || traces.is_some_and(|value| !value.is_empty()) +} + +/// Validate all header sources against the endpoint Praxis will pass to OTLP. fn validate_otlp_endpoint_transport_values( configured_endpoint: Option<&str>, environment_endpoint: Option<&str>, - configured_headers_present: Option, - environment_headers_present: bool, + headers_present: bool, + protocol: Option<&str>, ) -> Result<(), &'static str> { let endpoint = configured_endpoint.or_else(|| environment_endpoint.filter(|value| !value.trim().is_empty())); - let headers_present = configured_headers_present.unwrap_or(environment_headers_present); - let endpoint_uses_http = endpoint + if endpoint.is_some() && protocol.is_some_and(|value| value.trim() == "http/protobuf") { + return Err("OTLP HTTP/protobuf is unsupported by this gateway build; use OTLP/gRPC"); + } + let endpoint_uses_https = endpoint .and_then(|value| value.trim().split_once("://")) - .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("http")); + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("https")); - if headers_present && endpoint_uses_http { - return Err("OTLP exporter headers require HTTPS; refusing to send OTLP credentials over HTTP"); + if headers_present && !endpoint_uses_https { + return Err( + "OTLP exporter headers require an explicit HTTPS endpoint; refusing to send OTLP credentials without TLS", + ); } Ok(()) @@ -169,7 +186,9 @@ fn config_arg>(args: I) -> Result, #[cfg(test)] mod tests { - use super::{USAGE, config_arg, validate_otlp_endpoint_transport_values}; + use std::ffi::OsStr; + + use super::{USAGE, config_arg, otlp_headers_present, validate_otlp_endpoint_transport_values}; fn parse(args: &[&str]) -> Result, String> { config_arg(args.iter().map(|arg| (*arg).to_owned())) @@ -210,7 +229,7 @@ mod tests { #[test] fn rejects_http_endpoint_when_otlp_headers_are_configured() { assert!( - validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, Some(true), false).is_err(), + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, true, None).is_err(), "an explicitly configured HTTP endpoint must not receive headers" ); } @@ -218,20 +237,19 @@ mod tests { #[test] fn rejects_http_environment_fallback_when_otlp_headers_are_configured() { assert!( - validate_otlp_endpoint_transport_values(None, Some("http://collector:4317"), None, true).is_err(), + validate_otlp_endpoint_transport_values(None, Some("http://collector:4317"), true, None).is_err(), "the OTEL_EXPORTER_OTLP_ENDPOINT fallback must not receive headers over HTTP" ); } #[test] fn accepts_https_endpoints_with_otlp_headers() { - for (configured, fallback, configured_headers, environment_headers) in [ - (Some("https://collector:4317"), None, Some(true), false), - (None, Some("https://collector:4317"), None, true), + for (configured, fallback) in [ + (Some("https://collector:4317"), None), + (None, Some("https://collector:4317")), ] { assert!( - validate_otlp_endpoint_transport_values(configured, fallback, configured_headers, environment_headers,) - .is_ok(), + validate_otlp_endpoint_transport_values(configured, fallback, true, None).is_ok(), "HTTPS endpoints must remain usable with headers" ); } @@ -240,7 +258,7 @@ mod tests { #[test] fn preserves_http_behavior_when_otlp_headers_are_absent() { assert!( - validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, None, false).is_ok(), + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, false, None).is_ok(), "HTTP without exporter headers remains supported" ); } @@ -251,11 +269,63 @@ mod tests { validate_otlp_endpoint_transport_values( Some("https://configured:4317"), Some("http://fallback:4317"), - None, true, + None, ) .is_ok(), "an unused HTTP fallback must not reject the configured HTTPS endpoint" ); } + + #[test] + fn rejects_http_when_trace_specific_headers_are_present() { + let headers_present = otlp_headers_present(false, None, Some(OsStr::new("Authorization=secret"))); + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, headers_present, None) + .is_err(), + "trace-specific exporter headers must not be sent over HTTP" + ); + } + + #[test] + fn empty_configured_headers_do_not_hide_environment_headers() { + let headers_present = otlp_headers_present(false, Some(OsStr::new("Authorization=secret")), None); + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, headers_present, None) + .is_err(), + "the OTLP SDK can merge environment headers into an empty configured map" + ); + } + + #[test] + fn rejects_schemeless_endpoints_with_headers_regardless_of_insecure_flags() { + for endpoint in [Some("collector:4317"), None] { + assert!( + validate_otlp_endpoint_transport_values(endpoint, Some("collector:4317"), true, None).is_err(), + "a schemeless endpoint must not carry credentials; OTLP_INSECURE may make it plaintext" + ); + } + } + + #[test] + fn rejects_headers_without_an_endpoint() { + assert!( + validate_otlp_endpoint_transport_values(None, None, true, None).is_err(), + "headers require an explicit HTTPS endpoint even without an OTLP endpoint override" + ); + } + + #[test] + fn rejects_http_protobuf_until_praxis_supports_the_batch_processor_pairing() { + assert!( + validate_otlp_endpoint_transport_values( + Some("https://collector:4318"), + None, + false, + Some("http/protobuf"), + ) + .is_err(), + "the shipped Praxis HTTP client is incompatible with its thread-based batch processor" + ); + } } diff --git a/gateway/tests/startup_log.rs b/gateway/tests/startup_log.rs index 818f3f94d..b2acbd782 100644 --- a/gateway/tests/startup_log.rs +++ b/gateway/tests/startup_log.rs @@ -34,18 +34,28 @@ mod tests { } /// Start the gateway on `config`, with `serving_config` as `GRID_SERVING_CONFIG` when set. - fn spawn(config: PathBuf, serving_config: Option<&str>) -> io::Result<(Child, mpsc::Receiver)> { + fn spawn( + config: PathBuf, + serving_config: Option<&str>, + otlp_headers: Option<&str>, + ) -> io::Result<(Child, mpsc::Receiver)> { let mut command = Command::new(env!("CARGO_BIN_EXE_grid-gateway")); command .arg("--config") .arg(config) .env("RUST_LOG", "info") + .env_remove("OTEL_EXPORTER_OTLP_PROTOCOL") + .env_remove("OTEL_EXPORTER_OTLP_TRACES_HEADERS") .stdout(Stdio::piped()) .stderr(Stdio::piped()); match serving_config { Some(path) => command.env("GRID_SERVING_CONFIG", path), None => command.env_remove("GRID_SERVING_CONFIG"), }; + match otlp_headers { + Some(headers) => command.env("OTEL_EXPORTER_OTLP_HEADERS", headers), + None => command.env_remove("OTEL_EXPORTER_OTLP_HEADERS"), + }; let mut child = command.spawn()?; let (lines_tx, lines_rx) = mpsc::channel(); let missing = || io::Error::other("child output not piped"); @@ -65,7 +75,7 @@ mod tests { #[test] fn startup_emits_a_log_line() -> TestResult { - let (mut child, lines) = spawn(write_config("startup-log.yaml")?, None)?; + let (mut child, lines) = spawn(write_config("startup-log.yaml")?, None, None)?; let mut seen = Vec::new(); let found = loop { match lines.recv_timeout(TIMEOUT) { @@ -82,7 +92,7 @@ mod tests { #[test] fn fatal_startup_flushes_its_logs() -> TestResult { - let (mut child, lines) = spawn(write_config("fatal-log.yaml")?, Some("/nonexistent/serving.json"))?; + let (mut child, lines) = spawn(write_config("fatal-log.yaml")?, Some("/nonexistent/serving.json"), None)?; // Both streams close on exit, which disconnects the channel. let deadline = Instant::now() + TIMEOUT; let mut output = Vec::new(); @@ -116,7 +126,7 @@ mod tests { let yaml = std::fs::read_to_string(&path)? .replace("filter: static_response", "filter: not_registered_for_startup_test"); std::fs::write(&path, yaml)?; - let (mut child, lines) = spawn(path, None)?; + let (mut child, lines) = spawn(path, None, None)?; // Both streams close on exit, which disconnects the channel. let deadline = Instant::now() + TIMEOUT; @@ -140,4 +150,28 @@ mod tests { ); Ok(()) } + + #[test] + fn credentialed_https_grpc_exporter_starts() -> TestResult { + let path = write_config("credentialed-https-grpc.yaml")?; + let mut yaml = std::fs::read_to_string(&path)?; + yaml.push_str("telemetry:\n otlp_endpoint: https://127.0.0.1:4317\n"); + std::fs::write(&path, yaml)?; + + let (mut child, lines_rx) = spawn(path, None, Some("Authorization=test-only"))?; + + let deadline = Instant::now() + TIMEOUT; + let mut output = Vec::new(); + let started = loop { + match lines_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) if line.contains(STARTUP_MESSAGE) => break true, + Ok(line) => output.push(line), + Err(_) => break false, + } + }; + drop(child.kill()); + child.wait()?; + assert!(started, "credentialed HTTPS/gRPC exporter did not start: {output:#?}"); + Ok(()) + } } From 5e7981d68485d9a892d9350ec45942544fa0265f Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Tue, 6 Oct 2026 04:39:57 +0000 Subject: [PATCH 9/9] test(helm): use HTTPS in credentialed telemetry fixture Signed-off-by: Brent Salisbury --- charts/praxis-gateway/tests/telemetry_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/praxis-gateway/tests/telemetry_test.yaml b/charts/praxis-gateway/tests/telemetry_test.yaml index 4dbf41d4d..d358462e2 100644 --- a/charts/praxis-gateway/tests/telemetry_test.yaml +++ b/charts/praxis-gateway/tests/telemetry_test.yaml @@ -17,7 +17,7 @@ tests: transport: mode: plaintext gatewayConfig.telemetry.enabled: true - gatewayConfig.telemetry.otlpEndpoint: http://otel-collector.observability:4317 + gatewayConfig.telemetry.otlpEndpoint: https://otel-collector.observability:4317 gatewayConfig.telemetry.samplingRate: 0.25 gatewayConfig.telemetry.serviceName: grid-edge gatewayConfig.telemetry.batchIntervalSecs: 3 @@ -31,7 +31,7 @@ tests: asserts: - matchRegex: path: data["praxis.yaml"] - pattern: "(?s)telemetry:\\n\\s+otlp_endpoint: \\\"http://otel-collector\\.observability:4317\\\".*sampling_rate: 0.25.*service_name: \\\"grid-edge\\\".*batch_interval_secs: 3.*batch_size: 32" + pattern: "(?s)telemetry:\\n\\s+otlp_endpoint: \\\"https://otel-collector\\.observability:4317\\\".*sampling_rate: 0.25.*service_name: \\\"grid-edge\\\".*batch_interval_secs: 3.*batch_size: 32" - matchRegex: path: data["praxis.yaml"] pattern: "(?s)filters:\\n\\s+- filter: trace_context"