diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 90d4900b..46ed6bdb 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -287,6 +287,63 @@ spec: maximum: 65535.0 minimum: 0.0 type: integer + telemetry: + description: |- + Optional OpenTelemetry exporter and sampling settings for this gateway. + + When present, the generated Praxis configuration opts into telemetry and + adds the `trace_context` filter for outbound W3C header propagation. + Collector authentication must be provided to the gateway Deployment via + `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + into this `ConfigMap`. + nullable: true + properties: + batchIntervalSecs: + description: Batch exporter interval in seconds. Must be from 1 through 300 when set. + format: uint64 + maximum: 300.0 + minimum: 1.0 + nullable: true + type: integer + batchSize: + description: Maximum spans per export batch. Must be from 1 through 65,536 when set. + format: uint + maximum: 65536.0 + minimum: 1.0 + nullable: true + type: integer + environment: + description: Deployment environment resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + otlpEndpoint: + description: |- + OTLP collector endpoint, for example `http://otel-collector:4317`. + + If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + gateway Deployment. An empty string has the same meaning as omission. + nullable: true + pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ + type: string + samplingRate: + description: Root trace sampling probability from `0.0` through `1.0`. + format: double + maximum: 1.0 + minimum: 0.0 + nullable: true + type: number + serviceName: + description: OpenTelemetry `service.name` resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + serviceVersion: + description: OpenTelemetry `service.version` resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + type: object tlsCertMountPath: default: /etc/praxis/tls description: |- diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index f86c8f82..a1ad2bcd 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -186,7 +186,7 @@ Praxis AI image; these values may advance independently. | `image.repository` | string | `ghcr.io/praxis-proxy/ai` | Image repository. | | `image.tag` | string | `0.4.0` | Image tag (ignored when `image.digest` is set). | | `image.digest` | string | `""` | Immutable digest (sha256:…). When set, tag is ignored. | -| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider` and `gridServing` need. A repository ending in `/grid-gateway` sets it. | +| `image.flavor` | string | `ai` | `ai` or `grid-gateway`, the grid build that `gatewayConfig.role: provider`, `gridServing`, and telemetry need. A repository ending in `/grid-gateway` sets it. | | `image.pullPolicy` | string | `IfNotPresent` | Image pull policy. | | `imagePullSecrets` | list | `[]` | Pull secrets for private registries. | | `log.level` | string | `""` | Level for every module, rendered as RUST_LOG on the gateway and overlay-sync: off, error, warn, info, debug, or trace, in any case. Empty leaves RUST_LOG unset, so the binaries use their info default. An `env` entry named RUST_LOG takes precedence on the gateway. | @@ -206,6 +206,11 @@ Praxis AI image; these values may advance independently. | `config.key` | string | `praxis.yaml` | Key in the ConfigMap. | | `config.inline` | string | answers `GET /` with a JSON status, else 404 | Praxis config stored in a chart-managed ConfigMap when neither `config.existingConfigMap` nor `gatewayConfig.render` applies. Changing it rolls the pods. | | `gatewayConfig.render` | bool | `false` | Render the Praxis config from these values instead of a BYO ConfigMap. Also on when `config.existingConfigMap` is empty and the values configure grid routing (`gatewayConfig.backends`, `role: provider`, or `gridServing`). Never emits `insecure_options`. Changing the rendered config rolls the pods. See [AI Grid Network](#ai-grid-network-agn). | +| `gatewayConfig.telemetry.enabled` | bool | `false` | Enable OTLP/gRPC export and W3C header propagation in generated `praxis.yaml`. Requires `gatewayConfig.render: true` and `image.flavor: grid-gateway`. Configuration changes roll the pods. The tracked Grid build uses Praxis 0.7.3, which exports local HTTP and AI routing spans while forwarding W3C headers. Cross-gateway exported parentage requires the follow-up Praxis framework release described in [OpenTelemetry for Grid gateways](../../docs/architecture/opentelemetry.md). | +| `gatewayConfig.telemetry.otlpEndpoint` | string | `""` | OTLP endpoint without URL userinfo, query, or fragment credentials. Omitted or empty uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the container environment. If configured or generic/trace-specific exporter headers are present, the endpoint must explicitly use `https://`; scheme-less endpoints are rejected. | +| `gatewayConfig.telemetry.samplingRate` | number | unset | Root sampling probability, from `0.0` through `1.0`. | +| `gatewayConfig.telemetry.serviceName` / `serviceVersion` / `environment` | string | unset | OpenTelemetry resource attributes. | +| `gatewayConfig.telemetry.batchIntervalSecs` / `batchSize` | int | unset | OTLP batch export interval from 1 through 300 seconds and maximum batch size from 1 through 65,536 spans. | | `gatewayConfig.model` | string | **required** for a consumer without `gridServing` | Model advertised on the routing candidates. | | `gatewayConfig.backends` | map | **required** when rendered | Backends keyed by site, each with `endpoint` and optional `healthCheck` and `transport`. A consumer's key is the site it reaches over mutual TLS. A provider's `local` key is its one plaintext backend. The older list of `cluster`, `endpoints` entries still renders. | | `gatewayConfig.backends[].site` | string | `localSite` | Grid site the backend serves. A consumer's remote `mutual_tls` backend must name it, and it must differ from `localSite`. Its `transport.sni` defaults to `.grid.internal`. | diff --git a/charts/praxis-gateway/templates/_gateway-config.tpl b/charts/praxis-gateway/templates/_gateway-config.tpl index c8568149..ac589686 100644 --- a/charts/praxis-gateway/templates/_gateway-config.tpl +++ b/charts/praxis-gateway/templates/_gateway-config.tpl @@ -5,11 +5,44 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf {{- $cfg := .Values.gatewayConfig }} {{- $provider := eq ($cfg.role | default "consumer") "provider" }} {{- $apiKey := and (not $provider) (eq ($cfg.auth.mode | default "none") "api-key") }} +{{- $telemetry := $cfg.telemetry | default dict }} +{{- $hasSamplingRate := and (hasKey $telemetry "samplingRate") (ne $telemetry.samplingRate nil) }} +{{- $hasBatchInterval := and (hasKey $telemetry "batchIntervalSecs") (ne $telemetry.batchIntervalSecs nil) }} +{{- $hasBatchSize := and (hasKey $telemetry "batchSize") (ne $telemetry.batchSize nil) }} +{{- $hasTelemetryValues := or $telemetry.otlpEndpoint $hasSamplingRate $telemetry.serviceName $telemetry.serviceVersion $telemetry.environment $hasBatchInterval $hasBatchSize }} praxis.yaml: | {{- with $cfg.upstreamCA.secretName }} runtime: upstream_ca_file: {{ printf "%s/%s" $cfg.upstreamCA.mountPath ($cfg.upstreamCA.key | default "ca.crt") | quote }} {{- end }} + {{- if $telemetry.enabled }} + telemetry: + {{- if not $hasTelemetryValues }} + {} + {{- else }} + {{- with $telemetry.otlpEndpoint }} + otlp_endpoint: {{ . | quote }} + {{- end }} + {{- if $hasSamplingRate }} + sampling_rate: {{ $telemetry.samplingRate }} + {{- end }} + {{- with $telemetry.serviceName }} + service_name: {{ . | quote }} + {{- end }} + {{- with $telemetry.serviceVersion }} + service_version: {{ . | quote }} + {{- end }} + {{- with $telemetry.environment }} + environment: {{ . | quote }} + {{- end }} + {{- if $hasBatchInterval }} + batch_interval_secs: {{ $telemetry.batchIntervalSecs }} + {{- end }} + {{- if $hasBatchSize }} + batch_size: {{ $telemetry.batchSize }} + {{- end }} + {{- end }} + {{- end }} admin: address: {{ include "praxis-gateway.renderedAdminAddress" . | quote }} listeners: @@ -43,6 +76,9 @@ The data of the chart-rendered gateway ConfigMap, also hashed into checksum/conf filter_chains: - name: main filters: + {{- if $telemetry.enabled }} + - filter: trace_context + {{- end }} {{- if $provider }} {{- if ne (($cfg.peerTrust).mode | default "pin") "spiffe" }} - filter: peer_identity_trust diff --git a/charts/praxis-gateway/templates/_helpers.tpl b/charts/praxis-gateway/templates/_helpers.tpl index b09ec3ba..a08505fb 100644 --- a/charts/praxis-gateway/templates/_helpers.tpl +++ b/charts/praxis-gateway/templates/_helpers.tpl @@ -132,6 +132,15 @@ Validate image digest format when provided. Validate required config ConfigMap name. */}} {{- define "praxis-gateway.validateConfig" -}} +{{- $telemetry := .Values.gatewayConfig.telemetry | default dict }} +{{- if $telemetry.enabled }} +{{- if ne .Values.image.flavor "grid-gateway" }} +{{- fail "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway, whose Grid gateway build includes the OTLP and AI routing span features" }} +{{- end }} +{{- if not .Values.gatewayConfig.render }} +{{- fail "gatewayConfig.telemetry.enabled needs gatewayConfig.render true so the exporter settings are written to praxis.yaml" }} +{{- end }} +{{- end }} {{- if .Values.gatewayConfig.render }} {{- $consumer := ne (.Values.gatewayConfig.role | default "consumer") "provider" }} {{- if and $consumer (not (.Values.gridServing).enabled) (not (trim (toString .Values.gatewayConfig.model))) }} diff --git a/charts/praxis-gateway/templates/deployment.yaml b/charts/praxis-gateway/templates/deployment.yaml index 4952793e..97c88d9d 100644 --- a/charts/praxis-gateway/templates/deployment.yaml +++ b/charts/praxis-gateway/templates/deployment.yaml @@ -33,6 +33,10 @@ spec: {{- else if eq $configSource "render" }} {{- $_ := set $podAnnotations "checksum/config" (include "praxis-gateway.gatewayConfigData" . | sha256sum) }} {{- end }} + {{- if (.Values.gatewayConfig.telemetry).enabled }} + {{- /* Exporter initialization is process-scoped, so changed settings need a restart. */}} + {{- $_ := set $podAnnotations "checksum/telemetry" (toJson .Values.gatewayConfig.telemetry | sha256sum) }} + {{- end }} {{- with $podAnnotations }} annotations: {{- toYaml . | nindent 8 }} diff --git a/charts/praxis-gateway/tests/telemetry_test.yaml b/charts/praxis-gateway/tests/telemetry_test.yaml new file mode 100644 index 00000000..d358462e --- /dev/null +++ b/charts/praxis-gateway/tests/telemetry_test.yaml @@ -0,0 +1,156 @@ +suite: gateway telemetry +templates: + - templates/gateway-config.yaml + - templates/deployment.yaml +tests: + - it: emits telemetry and W3C propagation from explicit settings + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.otlpEndpoint: https://otel-collector.observability:4317 + gatewayConfig.telemetry.samplingRate: 0.25 + gatewayConfig.telemetry.serviceName: grid-edge + gatewayConfig.telemetry.batchIntervalSecs: 3 + gatewayConfig.telemetry.batchSize: 32 + env: + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: collector-credentials + key: headers + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+otlp_endpoint: \\\"https://otel-collector\\.observability:4317\\\".*sampling_rate: 0.25.*service_name: \\\"grid-edge\\\".*batch_interval_secs: 3.*batch_size: 32" + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)filters:\\n\\s+- filter: trace_context" + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.name + value: collector-credentials + template: templates/deployment.yaml + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "OTEL_EXPORTER_OTLP_HEADERS")].valueFrom.secretKeyRef.key + value: headers + template: templates/deployment.yaml + - exists: + path: spec.template.metadata.annotations["checksum/telemetry"] + template: templates/deployment.yaml + + - it: accepts the lower sampling boundary + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.samplingRate: 0.0 + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "sampling_rate: 0" + + - it: accepts the upper sampling boundary + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.samplingRate: 1.0 + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "sampling_rate: 1" + + - it: renders an empty telemetry mapping for environment-only settings + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+\\{\\}" + + - it: treats an empty endpoint as environment fallback + template: templates/gateway-config.yaml + set: + image.flavor: grid-gateway + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + gatewayConfig.telemetry.otlpEndpoint: "" + asserts: + - matchRegex: + path: data["praxis.yaml"] + pattern: "(?s)telemetry:\\n\\s+\\{\\}" + - notMatchRegex: + path: data["praxis.yaml"] + pattern: "otlp_endpoint:" + + - it: requires a Grid gateway image when telemetry is enabled + template: templates/deployment.yaml + set: + gatewayConfig.render: true + gatewayConfig.localSite: edge + gatewayConfig.model: test-model + gatewayConfig.auth.mode: none + gatewayConfig.backends: + site-a: + endpoint: 203.0.113.20:8080 + transport: + mode: plaintext + gatewayConfig.telemetry.enabled: true + asserts: + - failedTemplate: + errorPattern: "gatewayConfig.telemetry.enabled needs image.flavor grid-gateway" + + - it: requires generated Praxis config before enabling telemetry + template: templates/deployment.yaml + set: + image.flavor: grid-gateway + gatewayConfig.telemetry.enabled: true + asserts: + - failedTemplate: + errorPattern: "gatewayConfig.telemetry.enabled needs gatewayConfig.render true" diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index aa48826d..aa14c3cc 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -176,6 +176,21 @@ "existingSecret": { "type": "string" }, "mountPath": { "type": "string" } } + }, + "telemetry": { + "type": "object", + "additionalProperties": false, + "description": "Optional exporter settings for generated praxis.yaml. Collector credentials must come from deployment-managed Secret environment references.", + "properties": { + "enabled": { "type": "boolean", "description": "Enable OTLP export and the trace_context propagation filter." }, + "otlpEndpoint": { "type": "string", "pattern": "^(|https?://[^/?#@\\s]+(:[0-9]+)?(/[^\\s?#]*)?)$", "description": "OTLP/gRPC endpoint without userinfo, query, or fragment credentials. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT." }, + "samplingRate": { "type": ["number", "null"], "minimum": 0, "maximum": 1, "description": "Root trace sampling probability from 0.0 through 1.0." }, + "serviceName": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.name resource attribute." }, + "serviceVersion": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "OpenTelemetry service.version resource attribute." }, + "environment": { "type": "string", "pattern": "^(|.*\\S.*)$", "description": "deployment.environment resource attribute." }, + "batchIntervalSecs": { "type": ["integer", "null"], "minimum": 1, "maximum": 300, "description": "Positive batch exporter interval in seconds." }, + "batchSize": { "type": ["integer", "null"], "minimum": 1, "maximum": 65536, "description": "Positive maximum spans per export batch." } + } } } }, diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index 57c56fc1..834d6b46 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -132,6 +132,26 @@ gatewayConfig: # when config.existingConfigMap is empty and backends, role provider, or gridServing is set. # Changing the rendered config rolls the pods. render: false + # -- Opt in to OTLP tracing in the generated praxis.yaml. Requires + # image.flavor: grid-gateway, whose Grid image build includes the Praxis OTLP + # exporter and Praxis AI routing-span features. Credentials belong in env as + # Secret-backed OTEL_EXPORTER_OTLP_HEADERS, never here. + telemetry: + enabled: false + # -- OTLP/gRPC endpoint. Empty uses OTEL_EXPORTER_OTLP_ENDPOINT from env. + otlpEndpoint: "" + # -- Root sampling probability in the inclusive range 0.0..1.0. + samplingRate: null + # -- OpenTelemetry service.name resource attribute. + serviceName: "" + # -- OpenTelemetry service.version resource attribute. + serviceVersion: "" + # -- deployment.environment resource attribute. + environment: "" + # -- Batch export interval in seconds; must be positive. + batchIntervalSecs: null + # -- Maximum spans per export batch; must be positive. + batchSize: null # -- consumer routes callers. provider serves grid peers on the grid identity and forwards to one backend. role: consumer # -- Grid peers a provider accepts, each with a Grid-CA client certificate. diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index f8106023..d6b082a6 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -279,6 +279,63 @@ spec: maximum: 65535.0 minimum: 0.0 type: integer + telemetry: + description: |- + Optional OpenTelemetry exporter and sampling settings for this gateway. + + When present, the generated Praxis configuration opts into telemetry and + adds the `trace_context` filter for outbound W3C header propagation. + Collector authentication must be provided to the gateway Deployment via + `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + into this `ConfigMap`. + nullable: true + properties: + batchIntervalSecs: + description: Batch exporter interval in seconds. Must be from 1 through 300 when set. + format: uint64 + maximum: 300.0 + minimum: 1.0 + nullable: true + type: integer + batchSize: + description: Maximum spans per export batch. Must be from 1 through 65,536 when set. + format: uint + maximum: 65536.0 + minimum: 1.0 + nullable: true + type: integer + environment: + description: Deployment environment resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + otlpEndpoint: + description: |- + OTLP collector endpoint, for example `http://otel-collector:4317`. + + If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + gateway Deployment. An empty string has the same meaning as omission. + nullable: true + pattern: ^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$ + type: string + samplingRate: + description: Root trace sampling probability from `0.0` through `1.0`. + format: double + maximum: 1.0 + minimum: 0.0 + nullable: true + type: number + serviceName: + description: OpenTelemetry `service.name` resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + serviceVersion: + description: OpenTelemetry `service.version` resource attribute. + nullable: true + pattern: ^.*\S.*$ + type: string + type: object tlsCertMountPath: default: /etc/praxis/tls description: |- diff --git a/docs/README.md b/docs/README.md index 1247e558..6dc5458e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,8 @@ access policy, and trust model. - [Consumer Config](architecture/consumer-config.md) — operator-generated consumer Praxis `ConfigMap` and the `GatewayRef.consumerConfig` API. +- [OpenTelemetry](architecture/opentelemetry.md) - exporter configuration, + secret handling, image requirements, and the Praxis 0.7.1 trace-linkage limit. - [External Client Ingress](architecture/external-ingress.md) — GTM/GLB edge selection, AGN provider routing, trust boundaries, affinity, snapshot delivery, and provider-boundary ownership. diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index a416e664..6c4baff5 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -72,6 +72,14 @@ The generated config is a complete, runnable Praxis config containing: - `admin:` — admin listener at `127.0.0.1:9901` - `shutdown_timeout_secs: 5` +Set `consumerConfig.telemetry` to add process-level OTLP settings and the +`trace_context` propagation filter. These settings are written at the Praxis +config root and never enter the routing overlay. The consumer ConfigMap does +not contain collector headers; configure `OTEL_EXPORTER_OTLP_HEADERS` on the +gateway Deployment with a Secret-backed environment reference. See +[OpenTelemetry for Grid gateways](opentelemetry.md) for examples and the +Praxis 0.7.1 trace-linkage limitation. + This generated config covers the direct API-provider path where the consumer gateway is often also the final-hop gateway for the provider API call. Remote provider sites follow the same SecretRef contract, but the provider credential diff --git a/docs/architecture/opentelemetry.md b/docs/architecture/opentelemetry.md new file mode 100644 index 00000000..910d9489 --- /dev/null +++ b/docs/architecture/opentelemetry.md @@ -0,0 +1,128 @@ +# OpenTelemetry for Grid gateways + +Grid gateway builds can export traces to an OTLP/gRPC collector. Export is +configured on the gateway process; routing overlays remain limited to routing +state. + +## Enable export + +The default configuration has no collector endpoint, so it starts without an +OTLP exporter and does not require a collector. To opt in, use either +`GatewayRef.consumerConfig.telemetry` for operator-generated consumer config or +`gatewayConfig.telemetry` for the Helm-generated Praxis config. + +An operator example: + +```yaml +spec: + gatewayRefs: + - name: edge + namespace: grid + consumerConfig: + enabled: true + telemetry: + otlpEndpoint: http://otel-collector.observability:4317 + samplingRate: 0.1 + serviceName: grid-edge + environment: production +``` + +For Helm, enable `gatewayConfig.telemetry` and use the Grid gateway image: + +```yaml +image: + repository: ghcr.io/praxis-proxy/grid-gateway + flavor: grid-gateway +gatewayConfig: + render: true + telemetry: + enabled: true + otlpEndpoint: http://otel-collector.observability:4317 + samplingRate: 0.1 + serviceName: grid-edge +``` + +Both paths render a top-level Praxis `telemetry` block. The Helm chart adds the +`trace_context` filter when telemetry is enabled. The operator-generated +consumer config does the same. Neither path places exporter settings in the +routing overlay. + +## Credentials and lifecycle + +Do not put credentials in `otlpEndpoint`, telemetry fields, routing overlays, +or filter configuration. Praxis reads OTLP headers from +`OTEL_EXPORTER_OTLP_HEADERS`; provide that variable from a Deployment-managed +Secret. Helm exposes the container `env` list for a Secret reference: + +```yaml +env: + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: collector-credentials + key: headers +``` + +The Secret value uses the OpenTelemetry `key=value` header format. Generic +`OTEL_EXPORTER_OTLP_HEADERS`, trace-specific +`OTEL_EXPORTER_OTLP_TRACES_HEADERS`, and configured `otlp_headers` all require +an explicitly specified `https://` collector endpoint. A scheme-less endpoint +is rejected when headers are present, regardless of the OTLP `INSECURE` +environment settings. This gateway build supports OTLP/gRPC; it rejects +`OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` while the locked Praxis HTTP +exporter and batch processor are incompatible. The operator +only creates the consumer `ConfigMap`; the deployment manager must add the same +Secret-backed environment reference to its gateway Deployment. Praxis redacts +OTLP header values from its config debug representation. + +The HTTPS preflight protects exporter headers from plaintext transport; it +does not prove certificate-verified delivery. Credentialed HTTPS export +requires the Praxis trust fix in +[praxis#1354](https://github.com/praxis-proxy/praxis/issues/1354), a Praxis +release containing it, and the Grid qualification tracked in +[Grid #301](https://github.com/praxis-proxy/grid/issues/301). + +Exporter setup runs once at process startup. Helm rolls gateway pods when its +telemetry values change. An externally managed consumer Deployment must be +restarted after its generated ConfigMap changes, and a gateway must be restarted +after the collector endpoint or Secret-backed environment changes. On normal +server return, Grid drops Praxis's `TracingGuard`, which shuts down the OTLP +provider and flushes queued spans. + +## Build and spans + +The `grid-gateway` binary built by `deploy/gateway/Containerfile` enables the +Praxis `otel` feature and the Praxis AI v0.4.1 `opentelemetry` feature. The +tracked gateway lockfile currently resolves Praxis 0.7.3. Use an image built +from this Grid target, published as `ghcr.io/praxis-proxy/grid-gateway`, for +this configuration. The chart's default `ghcr.io/praxis-proxy/ai:0.4.0` image +does not include the Grid build features. + +With the AI feature enabled, these short semantic spans are supported when the +corresponding filters run: + +- HTTP server spans and `upstream_exchange` internal spans from Praxis 0.7.3. +- `routing.select` from `intelligent_route`, including the serving overlay + semantic revision when that revision is available to the filter. +- `provider.route` from `provider_route`, including a validated edge overlay + revision when present. + +The AI implementation projects bounded routing fields into those spans; +it does not add prompt or body contents, credential values, authorization +headers, cookies, session keys, or raw request IDs. These spans describe route +selection and resolution; `provider.route` does not prove the model backend +served the request. + +## Cross-gateway trace linkage + +The locked Praxis 0.7.3 build forwards W3C trace headers but does not connect them to the +exported HTTP server span or emit an exported HTTP client span for the upstream +attempt. Edge and provider gateways therefore export separate local traces, +even when the backend receives a forwarded `traceparent`. The routing spans +remain local children of their gateway's HTTP server span. + +Cross-gateway exported parentage requires a Praxis framework release containing +the inbound context and upstream-attempt span changes. Once Grid updates its +gateway dependency to that release, rebuild the tracked image and verify the +collector's parent IDs across both gateways before claiming linked traces. +The Praxis AI routing-span feature alone does not supply that linkage. diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 9bf30a3b..2d370395 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -310,6 +310,49 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -1381,6 +1424,7 @@ version = "0.1.4" dependencies = [ "ai-grid-filters", "certs", + "opentelemetry-otlp", "praxis-ai-filters", "praxis-proxy", "praxis-proxy-core", @@ -1643,9 +1687,11 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1653,12 +1699,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ + "base64 0.23.1", "bytes", "futures-channel", "futures-util", @@ -1666,7 +1726,9 @@ dependencies = [ "http-body", "httparse", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1851,6 +1913,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2176,6 +2244,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" @@ -2256,6 +2330,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2525,6 +2605,83 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-http" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +dependencies = [ + "http", + "httpdate", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ouroboros" version = "0.18.5" @@ -2653,6 +2810,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2951,6 +3128,9 @@ dependencies = [ "dashmap", "http", "metrics", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "percent-encoding", "praxis-proxy-tls", "quixotic-plecostomus-core", @@ -2960,8 +3140,10 @@ dependencies = [ "serde", "thiserror", "tokio", + "tonic", "tracing", "tracing-appender", + "tracing-opentelemetry", "tracing-subscriber", "yaml_serde", ] @@ -2980,6 +3162,7 @@ dependencies = [ "http", "metrics", "openssl", + "opentelemetry", "parking_lot", "percent-encoding", "praxis-policy", @@ -2996,6 +3179,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-opentelemetry", "yaml_serde", "zeroize", ] @@ -3087,6 +3271,38 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "psm" version = "0.1.32" @@ -3569,6 +3785,35 @@ dependencies = [ "yaml_serde", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -4156,6 +4401,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4371,6 +4625,101 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.2", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4433,6 +4782,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -4716,6 +5081,17 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.129" @@ -4758,6 +5134,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wildmatch" version = "2.6.1" diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index e36c2b44..91c8b723 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -308,10 +308,11 @@ name = "grid-gateway" path = "src/main.rs" [dependencies] -praxis-proxy = { version = "0.7.3", features = ["spiffe"] } +praxis-proxy = { version = "0.7.3", features = ["otel", "spiffe"] } praxis-proxy-filter = "0.7.3" praxis-proxy-core = "0.7.3" -praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false } +opentelemetry-otlp = { version = "0.33.0", default-features = false, features = ["tls-provider-agnostic"] } +praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false, features = ["opentelemetry"] } ai-grid-filters = { path = "ai-grid-filters" } tracing = "0.1.44" @@ -322,7 +323,7 @@ rustls = { version = "0.23.45", features = ["ring"] } # machete can't see these: used as `praxis` / `praxis_core` / `praxis_filter`, not their package names. [package.metadata.cargo-machete] -ignored = ["praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter"] +ignored = ["opentelemetry-otlp", "praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter"] [lints] workspace = true diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 629a2da2..1197165b 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -12,13 +12,21 @@ //! Kubernetes client stack. See `deploy/gateway/Containerfile` and the //! `gateway-image` make target. -use std::process::ExitCode; +use std::{ffi::OsStr, process::ExitCode}; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; use tracing::info; /// Log line emitted once tracing is up; the startup test waits for it. const STARTUP_MESSAGE: &str = "starting grid-gateway"; +/// OTel-standard environment variable used as the OTLP endpoint fallback. +const OTLP_ENDPOINT_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; +/// OTel-standard environment variable containing exporter headers. +const OTLP_HEADERS_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_HEADERS"; +/// Signal-specific headers are merged into the exporter after Praxis config. +const OTLP_TRACES_HEADERS_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_TRACES_HEADERS"; +/// Praxis currently selects the exporter protocol from this variable. +const OTLP_PROTOCOL_ENV_VAR: &str = "OTEL_EXPORTER_OTLP_PROTOCOL"; fn main() -> ExitCode { // Install the crypto provider before anything builds a TLS config. @@ -36,6 +44,8 @@ fn main() -> ExitCode { let config = praxis::with_bootstrap_logging(|| Config::from_config_file_or(config_file.as_ref(), DEFAULT_CONFIG)) .unwrap_or_else(|err| praxis::fatal(&err)); + validate_otlp_endpoint_transport(&config).unwrap_or_else(|err| praxis::fatal(&err)); + // Without a subscriber every log line, including reload results, is dropped. let tracing_guard = praxis::init_tracing(&config).unwrap_or_else(|err| praxis::fatal(&err)); let log_level = Some(tracing_guard.log_level_state()); @@ -59,10 +69,70 @@ fn main() -> ExitCode { None => None, }; - // Returning instead of exiting drops the guard, flushing queued log lines. + // Use the returning server path so both the routing runtime and tracing + // provider can shut down cleanly after the listeners stop. let result = praxis::try_run_server_with_registry(config, registry, config_file, log_level); drop(grid_runtime); - result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS) + let exit_code = result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS); + // The Praxis guard shuts down the OTLP provider and flushes queued spans. + drop(tracing_guard); + exit_code +} + +/// Refuse credentialed export without explicit TLS and unsupported HTTP export. +/// +/// The OTLP SDK can merge generic or trace-specific environment headers into +/// programmatic headers. Treat every source as potentially credential-bearing. +fn validate_otlp_endpoint_transport(config: &Config) -> Result<(), &'static str> { + let environment_endpoint = std::env::var(OTLP_ENDPOINT_ENV_VAR).ok(); + let generic_headers = std::env::var_os(OTLP_HEADERS_ENV_VAR); + let traces_headers = std::env::var_os(OTLP_TRACES_HEADERS_ENV_VAR); + let headers_present = otlp_headers_present( + config + .telemetry + .otlp_headers + .as_ref() + .is_some_and(|headers| !headers.is_empty()), + generic_headers.as_deref(), + traces_headers.as_deref(), + ); + let protocol = std::env::var(OTLP_PROTOCOL_ENV_VAR).ok(); + + validate_otlp_endpoint_transport_values( + config.telemetry.otlp_endpoint.as_deref(), + environment_endpoint.as_deref(), + headers_present, + protocol.as_deref(), + ) +} + +/// The SDK may append either environment source even when config has an empty map. +fn otlp_headers_present(configured: bool, generic: Option<&OsStr>, traces: Option<&OsStr>) -> bool { + configured || generic.is_some_and(|value| !value.is_empty()) || traces.is_some_and(|value| !value.is_empty()) +} + +/// Validate all header sources against the endpoint Praxis will pass to OTLP. +fn validate_otlp_endpoint_transport_values( + configured_endpoint: Option<&str>, + environment_endpoint: Option<&str>, + headers_present: bool, + protocol: Option<&str>, +) -> Result<(), &'static str> { + let endpoint = configured_endpoint.or_else(|| environment_endpoint.filter(|value| !value.trim().is_empty())); + if endpoint.is_some() && protocol.is_some_and(|value| value.trim() == "http/protobuf") { + return Err("OTLP HTTP/protobuf is unsupported by this gateway build; use OTLP/gRPC"); + } + let endpoint_uses_https = endpoint + .and_then(|value| value.trim().split_once("://")) + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("https")); + + if headers_present && !endpoint_uses_https { + return Err( + "OTLP exporter headers require an explicit HTTPS endpoint; refusing to send OTLP credentials without TLS", + ); + } + + Ok(()) } /// Start the cross-site pollers and register `grid_site_route` over their snapshot. @@ -116,7 +186,9 @@ fn config_arg>(args: I) -> Result, #[cfg(test)] mod tests { - use super::{USAGE, config_arg}; + use std::ffi::OsStr; + + use super::{USAGE, config_arg, otlp_headers_present, validate_otlp_endpoint_transport_values}; fn parse(args: &[&str]) -> Result, String> { config_arg(args.iter().map(|arg| (*arg).to_owned())) @@ -153,4 +225,107 @@ mod tests { assert_eq!(parse(args), Err(USAGE.to_owned()), "{args:?}"); } } + + #[test] + fn rejects_http_endpoint_when_otlp_headers_are_configured() { + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, true, None).is_err(), + "an explicitly configured HTTP endpoint must not receive headers" + ); + } + + #[test] + fn rejects_http_environment_fallback_when_otlp_headers_are_configured() { + assert!( + validate_otlp_endpoint_transport_values(None, Some("http://collector:4317"), true, None).is_err(), + "the OTEL_EXPORTER_OTLP_ENDPOINT fallback must not receive headers over HTTP" + ); + } + + #[test] + fn accepts_https_endpoints_with_otlp_headers() { + for (configured, fallback) in [ + (Some("https://collector:4317"), None), + (None, Some("https://collector:4317")), + ] { + assert!( + validate_otlp_endpoint_transport_values(configured, fallback, true, None).is_ok(), + "HTTPS endpoints must remain usable with headers" + ); + } + } + + #[test] + fn preserves_http_behavior_when_otlp_headers_are_absent() { + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, false, None).is_ok(), + "HTTP without exporter headers remains supported" + ); + } + + #[test] + fn configured_endpoint_takes_precedence_over_environment_fallback() { + assert!( + validate_otlp_endpoint_transport_values( + Some("https://configured:4317"), + Some("http://fallback:4317"), + true, + None, + ) + .is_ok(), + "an unused HTTP fallback must not reject the configured HTTPS endpoint" + ); + } + + #[test] + fn rejects_http_when_trace_specific_headers_are_present() { + let headers_present = otlp_headers_present(false, None, Some(OsStr::new("Authorization=secret"))); + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, headers_present, None) + .is_err(), + "trace-specific exporter headers must not be sent over HTTP" + ); + } + + #[test] + fn empty_configured_headers_do_not_hide_environment_headers() { + let headers_present = otlp_headers_present(false, Some(OsStr::new("Authorization=secret")), None); + assert!( + validate_otlp_endpoint_transport_values(Some("http://collector:4317"), None, headers_present, None) + .is_err(), + "the OTLP SDK can merge environment headers into an empty configured map" + ); + } + + #[test] + fn rejects_schemeless_endpoints_with_headers_regardless_of_insecure_flags() { + for endpoint in [Some("collector:4317"), None] { + assert!( + validate_otlp_endpoint_transport_values(endpoint, Some("collector:4317"), true, None).is_err(), + "a schemeless endpoint must not carry credentials; OTLP_INSECURE may make it plaintext" + ); + } + } + + #[test] + fn rejects_headers_without_an_endpoint() { + assert!( + validate_otlp_endpoint_transport_values(None, None, true, None).is_err(), + "headers require an explicit HTTPS endpoint even without an OTLP endpoint override" + ); + } + + #[test] + fn rejects_http_protobuf_until_praxis_supports_the_batch_processor_pairing() { + assert!( + validate_otlp_endpoint_transport_values( + Some("https://collector:4318"), + None, + false, + Some("http/protobuf"), + ) + .is_err(), + "the shipped Praxis HTTP client is incompatible with its thread-based batch processor" + ); + } } diff --git a/gateway/tests/startup_log.rs b/gateway/tests/startup_log.rs index 88a8a7d5..b2acbd78 100644 --- a/gateway/tests/startup_log.rs +++ b/gateway/tests/startup_log.rs @@ -34,18 +34,28 @@ mod tests { } /// Start the gateway on `config`, with `serving_config` as `GRID_SERVING_CONFIG` when set. - fn spawn(config: PathBuf, serving_config: Option<&str>) -> io::Result<(Child, mpsc::Receiver)> { + fn spawn( + config: PathBuf, + serving_config: Option<&str>, + otlp_headers: Option<&str>, + ) -> io::Result<(Child, mpsc::Receiver)> { let mut command = Command::new(env!("CARGO_BIN_EXE_grid-gateway")); command .arg("--config") .arg(config) .env("RUST_LOG", "info") + .env_remove("OTEL_EXPORTER_OTLP_PROTOCOL") + .env_remove("OTEL_EXPORTER_OTLP_TRACES_HEADERS") .stdout(Stdio::piped()) .stderr(Stdio::piped()); match serving_config { Some(path) => command.env("GRID_SERVING_CONFIG", path), None => command.env_remove("GRID_SERVING_CONFIG"), }; + match otlp_headers { + Some(headers) => command.env("OTEL_EXPORTER_OTLP_HEADERS", headers), + None => command.env_remove("OTEL_EXPORTER_OTLP_HEADERS"), + }; let mut child = command.spawn()?; let (lines_tx, lines_rx) = mpsc::channel(); let missing = || io::Error::other("child output not piped"); @@ -65,7 +75,7 @@ mod tests { #[test] fn startup_emits_a_log_line() -> TestResult { - let (mut child, lines) = spawn(write_config("startup-log.yaml")?, None)?; + let (mut child, lines) = spawn(write_config("startup-log.yaml")?, None, None)?; let mut seen = Vec::new(); let found = loop { match lines.recv_timeout(TIMEOUT) { @@ -82,7 +92,7 @@ mod tests { #[test] fn fatal_startup_flushes_its_logs() -> TestResult { - let (mut child, lines) = spawn(write_config("fatal-log.yaml")?, Some("/nonexistent/serving.json"))?; + let (mut child, lines) = spawn(write_config("fatal-log.yaml")?, Some("/nonexistent/serving.json"), None)?; // Both streams close on exit, which disconnects the channel. let deadline = Instant::now() + TIMEOUT; let mut output = Vec::new(); @@ -109,4 +119,59 @@ mod tests { ); Ok(()) } + + #[test] + fn server_pipeline_error_is_reported_before_tracing_guard_flush() -> TestResult { + let path = write_config("invalid-filter.yaml")?; + let yaml = std::fs::read_to_string(&path)? + .replace("filter: static_response", "filter: not_registered_for_startup_test"); + std::fs::write(&path, yaml)?; + let (mut child, lines) = spawn(path, None, None)?; + + // Both streams close on exit, which disconnects the channel. + let deadline = Instant::now() + TIMEOUT; + let mut output = Vec::new(); + loop { + match lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) => output.push(line), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => { + child.kill()?; + child.wait()?; + return Err(format!("gateway still running after {TIMEOUT:?}").into()); + }, + } + } + let status = child.wait()?; + assert!(!status.success(), "an invalid pipeline filter must fail startup"); + assert!( + output.iter().any(|line| line.contains("fatal error; exiting")), + "the server startup error was not logged before the tracing guard dropped: {output:#?}" + ); + Ok(()) + } + + #[test] + fn credentialed_https_grpc_exporter_starts() -> TestResult { + let path = write_config("credentialed-https-grpc.yaml")?; + let mut yaml = std::fs::read_to_string(&path)?; + yaml.push_str("telemetry:\n otlp_endpoint: https://127.0.0.1:4317\n"); + std::fs::write(&path, yaml)?; + + let (mut child, lines_rx) = spawn(path, None, Some("Authorization=test-only"))?; + + let deadline = Instant::now() + TIMEOUT; + let mut output = Vec::new(); + let started = loop { + match lines_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) if line.contains(STARTUP_MESSAGE) => break true, + Ok(line) => output.push(line), + Err(_) => break false, + } + }; + drop(child.kill()); + child.wait()?; + assert!(started, "credentialed HTTPS/gRPC exporter did not start: {output:#?}"); + Ok(()) + } } diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 9438805c..4137f666 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2085,12 +2085,13 @@ fn consumer_config_map( gw_ref: &GatewayRef, cc: &ConsumerConfig, ) -> Result { - let config_yaml = consumer_config::generate_consumer_praxis_config( + let config_yaml = consumer_config::generate_consumer_praxis_config_with_telemetry( overlay, &cc.credential_mount_base, &cc.cluster_endpoints, &cc.tls_cert_mount_path, cc.listener_port, + cc.telemetry.as_ref(), )?; Ok(consumer_config::build_consumer_config_map( &config_yaml, diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 9ef3523c..549c1e89 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -778,7 +778,7 @@ pub struct GatewayRef { /// The generated `ConfigMap` never contains credential token bytes. Credential /// entries use a `file:` source under `credentialMountBase`; the mounted /// Kubernetes Secret provides the token at runtime. -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct ConsumerConfig { /// Enable operator-managed consumer Praxis config generation for this gateway. @@ -840,6 +840,133 @@ pub struct ConsumerConfig { /// Default: `8080`. #[serde(default = "default_listener_port")] pub listener_port: u16, + + /// Optional OpenTelemetry exporter and sampling settings for this gateway. + /// + /// When present, the generated Praxis configuration opts into telemetry and + /// adds the `trace_context` filter for outbound W3C header propagation. + /// Collector authentication must be provided to the gateway Deployment via + /// `OTEL_EXPORTER_OTLP_HEADERS` from a Secret; credentials are never copied + /// into this `ConfigMap`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub telemetry: Option, +} + +/// Maximum batch interval accepted by Praxis telemetry configuration. +const MAX_TELEMETRY_BATCH_INTERVAL_SECS: u64 = 300; +/// Maximum batch size accepted by Praxis telemetry configuration. +const MAX_TELEMETRY_BATCH_SIZE: usize = 65_536; + +/// Validated OpenTelemetry settings rendered into operator-generated Praxis YAML. +/// +/// Secret material is intentionally not part of this type. Supply collector +/// credentials through the gateway Deployment's `OTEL_EXPORTER_OTLP_HEADERS` +/// environment variable using a Secret-backed `valueFrom` reference. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct GatewayTelemetryConfig { + /// OTLP collector endpoint, for example `http://otel-collector:4317`. + /// + /// If omitted, Praxis can read `OTEL_EXPORTER_OTLP_ENDPOINT` from the + /// gateway Deployment. An empty string has the same meaning as omission. + #[schemars(regex(pattern = r"^(|https?://[^/?#@\s]+(:[0-9]+)?(/[^\s?#]*)?)$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub otlp_endpoint: Option, + + /// Root trace sampling probability from `0.0` through `1.0`. + #[schemars(range(min = 0.0, max = 1.0))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sampling_rate: Option, + + /// OpenTelemetry `service.name` resource attribute. + #[schemars(regex(pattern = r"^.*\S.*$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_name: Option, + + /// OpenTelemetry `service.version` resource attribute. + #[schemars(regex(pattern = r"^.*\S.*$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_version: Option, + + /// Deployment environment resource attribute. + #[schemars(regex(pattern = r"^.*\S.*$"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option, + + /// Batch exporter interval in seconds. Must be from 1 through 300 when set. + #[schemars(range(min = 1, max = 300))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_interval_secs: Option, + + /// Maximum spans per export batch. Must be from 1 through 65,536 when set. + #[schemars(range(min = 1, max = 65_536))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_size: Option, +} + +impl GatewayTelemetryConfig { + /// Validate values before rendering them into the consumer configuration. + /// + /// # Errors + /// + /// Returns an explanatory message for invalid strings, sampling rates, or + /// batch settings outside their supported ranges. + #[expect( + clippy::too_many_lines, + reason = "validates all telemetry settings before CRD rendering" + )] + pub fn validate(&self) -> Result<(), String> { + if self.otlp_endpoint.as_ref().is_some_and(|endpoint| { + if endpoint.is_empty() { + return false; + } + let endpoint = endpoint.trim(); + let Some((scheme, rest)) = endpoint.split_once("://") else { + return true; + }; + let authority = rest.split('/').next().unwrap_or_default(); + !matches!(scheme, "http" | "https") + || authority.is_empty() + || authority.contains('@') + || endpoint.contains('?') + || endpoint.contains('#') + || endpoint.chars().any(char::is_whitespace) + }) { + return Err("telemetry.otlpEndpoint must be an http(s) URL without credentials".to_owned()); + } + if let Some(rate) = self.sampling_rate + && (!rate.is_finite() || !(0.0..=1.0).contains(&rate)) + { + return Err("telemetry.samplingRate must be between 0.0 and 1.0".to_owned()); + } + if self + .batch_interval_secs + .is_some_and(|seconds| !(1..=MAX_TELEMETRY_BATCH_INTERVAL_SECS).contains(&seconds)) + { + return Err(format!( + "telemetry.batchIntervalSecs must be between 1 and {MAX_TELEMETRY_BATCH_INTERVAL_SECS}" + )); + } + if self + .batch_size + .is_some_and(|size| !(1..=MAX_TELEMETRY_BATCH_SIZE).contains(&size)) + { + return Err(format!( + "telemetry.batchSize must be between 1 and {MAX_TELEMETRY_BATCH_SIZE}" + )); + } + for (field, value) in [ + ("serviceName", self.service_name.as_deref()), + ("serviceVersion", self.service_version.as_deref()), + ("environment", self.environment.as_deref()), + ] { + if value.is_some_and(|value| value.trim().is_empty()) { + return Err(format!("telemetry.{field} must not be blank")); + } + } + Ok(()) + } } impl Default for ConsumerConfig { @@ -851,6 +978,7 @@ impl Default for ConsumerConfig { cluster_endpoints: Vec::new(), tls_cert_mount_path: default_tls_cert_mount_path(), listener_port: default_listener_port(), + telemetry: None, } } } @@ -1505,7 +1633,12 @@ mod tests { "mode": "mutual_tls", "sni": "site-a.grid.internal" } - }] + }], + "telemetry": { + "otlpEndpoint": "http://otel-collector:4317", + "samplingRate": 0.125, + "serviceName": "grid-edge" + } } }); let gw: GatewayRef = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); @@ -1538,6 +1671,14 @@ mod tests { Some("site-a.grid.internal"), "transport SNI must round-trip" ); + let telemetry = cc.telemetry.as_ref().unwrap_or_else(|| std::process::abort()); + assert_eq!(telemetry.otlp_endpoint.as_deref(), Some("http://otel-collector:4317")); + assert_eq!(telemetry.sampling_rate, Some(0.125)); + assert_eq!(telemetry.service_name.as_deref(), Some("grid-edge")); + assert!( + telemetry.validate().is_ok(), + "valid telemetry settings must pass validation" + ); } #[test] @@ -1592,12 +1733,122 @@ mod tests { cc.cluster_endpoints.is_empty(), "clusterEndpoints must default to empty" ); + assert!(cc.telemetry.is_none(), "telemetry must remain disabled by default"); assert_eq!( cc.tls_cert_mount_path, "/etc/praxis/tls", "tlsCertMountPath must use default" ); } + #[test] + fn telemetry_config_rejects_embedded_headers() { + let with_headers = serde_json::json!({ + "otlpEndpoint": "http://collector:4317", + "otlpHeaders": {"authorization": "test-value"} + }); + assert!( + serde_json::from_value::(with_headers).is_err(), + "collector credentials must not be accepted as config fields" + ); + } + + #[test] + fn telemetry_config_accepts_empty_endpoint_fallback() { + let empty_endpoint = serde_json::from_value::(serde_json::json!({ + "otlpEndpoint": "" + })) + .unwrap_or_else(|_| std::process::abort()); + assert!( + empty_endpoint.validate().is_ok(), + "an empty endpoint must use the deployment environment fallback" + ); + } + + #[test] + fn telemetry_batch_bounds_appear_in_crd_schema() { + let crd = crd_json(); + let properties = crd + .pointer( + "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/gatewayRefs/items/properties/consumerConfig/properties/telemetry/properties", + ) + .unwrap_or_else(|| std::process::abort()); + + for (field, maximum) in [("batchIntervalSecs", 300.0), ("batchSize", 65_536.0)] { + let schema = properties.get(field).unwrap_or_else(|| std::process::abort()); + assert_eq!( + schema.get("minimum").and_then(serde_json::Value::as_f64), + Some(1.0), + "{field} CRD schema must enforce the inclusive lower bound" + ); + assert_eq!( + schema.get("maximum").and_then(serde_json::Value::as_f64), + Some(maximum), + "{field} CRD schema must enforce the inclusive upper bound" + ); + } + } + + #[test] + fn telemetry_batch_bounds_are_enforced_at_runtime() { + for (field, maximum) in [("batchIntervalSecs", 300_u64), ("batchSize", 65_536_u64)] { + for (value, expected_valid) in [(0, false), (1, true), (maximum, true), (maximum + 1, false)] { + let telemetry = serde_json::from_value::(serde_json::json!({(field): value})) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + telemetry.validate().is_ok(), + expected_valid, + "{field}={value} runtime validation mismatch" + ); + } + } + } + + #[test] + fn telemetry_resource_attributes_reject_blank_values_at_admission_and_runtime() { + let crd = crd_json(); + let telemetry_properties = crd + .pointer( + "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/gatewayRefs/items/properties/consumerConfig/properties/telemetry/properties", + ) + .unwrap_or_else(|| std::process::abort()); + for field in ["serviceName", "serviceVersion", "environment"] { + let pattern = telemetry_properties + .get(field) + .and_then(|property| property.get("pattern")) + .and_then(serde_json::Value::as_str); + assert_eq!( + pattern, + Some(r"^.*\S.*$"), + "{field} must reject blank values at admission" + ); + let telemetry: GatewayTelemetryConfig = + serde_json::from_value(serde_json::json!({(field): ""})).unwrap_or_else(|_| std::process::abort()); + assert!( + telemetry.validate().is_err(), + "{field} must reject blank values at runtime" + ); + } + } + + #[test] + fn telemetry_config_rejects_credentials_in_endpoint() { + for endpoint in [ + "https://user:password@collector:4317", + "https://collector:4317?api_key=secret", + "https://collector:4317/otlp#token=secret", + ] { + let telemetry = serde_json::from_value::(serde_json::json!({ + "otlpEndpoint": endpoint, + "samplingRate": 0.5 + })) + .unwrap_or_else(|_| std::process::abort()); + assert!( + telemetry.validate().is_err(), + "credentials in endpoint userinfo, query, or fragment must be rejected: {endpoint}" + ); + } + } + #[test] fn consumer_config_absent_not_serialized() { let gw = GatewayRef { @@ -1614,6 +1865,10 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "asserts generated telemetry fields and all opt-in security boundaries" + )] fn grid_network_crd_has_consumer_config_field_on_gateway_ref() { let crd = crd_json(); let gateway_ref_properties = crd @@ -1637,6 +1892,47 @@ mod tests { consumer_config_properties.contains_key("tlsCertMountPath"), "CRD schema must include consumerConfig.tlsCertMountPath" ); + let telemetry_properties = consumer_config_properties + .get("telemetry") + .and_then(|v| v.pointer("/properties")) + .and_then(serde_json::Value::as_object) + .unwrap_or_else(|| std::process::abort()); + let sampling_rate = telemetry_properties + .get("samplingRate") + .unwrap_or_else(|| std::process::abort()); + assert_eq!( + sampling_rate.pointer("/minimum").and_then(serde_json::Value::as_f64), + Some(0.0), + "CRD schema must reject sampling rates below zero" + ); + assert_eq!( + sampling_rate.pointer("/maximum").and_then(serde_json::Value::as_f64), + Some(1.0), + "CRD schema must reject sampling rates above one" + ); + let endpoint_schema = telemetry_properties + .get("otlpEndpoint") + .unwrap_or_else(|| std::process::abort()); + assert_eq!( + endpoint_schema.get("type").and_then(serde_json::Value::as_str), + Some("string"), + "OTLP endpoint remains an optional string" + ); + assert!( + !telemetry_properties + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and(|required| required.iter().any(|field| field.as_str() == Some("otlpEndpoint"))), + "omitted endpoint must remain valid for environment fallback" + ); + let endpoint_pattern = endpoint_schema + .get("pattern") + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| std::process::abort()); + assert!( + endpoint_pattern.starts_with("^(|") && endpoint_pattern.contains("https?://"), + "CRD schema must accept empty environment fallback and explicit HTTP(S) URLs" + ); } #[test] diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index c4012f46..d7da6b1e 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -23,7 +23,7 @@ use std::collections::{BTreeMap, BTreeSet}; use k8s_openapi::api::core::v1::ConfigMap; use crate::{ - crd::grid_network::{ClusterEndpointConfig, SelectionMode, TransportMode}, + crd::grid_network::{ClusterEndpointConfig, GatewayTelemetryConfig, SelectionMode, TransportMode}, resources::routing_overlay::{RoutingCandidate, RoutingOverlay}, }; @@ -86,6 +86,10 @@ pub enum ConsumerConfigError { cluster: String, }, + /// Telemetry configuration failed validation. + #[error("invalid telemetry configuration: {0}")] + InvalidTelemetry(String), + /// JSON serialization failed. #[error("json error: {0}")] Json(#[from] serde_json::Error), @@ -123,16 +127,48 @@ pub enum ConsumerConfigError { /// - Any candidate cluster has no matching endpoint in `cluster_endpoints`. /// - Any cluster endpoint has no `transport` configuration. /// - Any `mutual_tls` endpoint has no (or blank) `sni`. +#[cfg(test)] +pub(crate) fn generate_consumer_praxis_config( + overlay: &RoutingOverlay, + credential_mount_base: &str, + cluster_endpoints: &[ClusterEndpointConfig], + tls_cert_mount_path: &str, + listener_port: u16, +) -> Result { + generate_consumer_praxis_config_with_telemetry( + overlay, + credential_mount_base, + cluster_endpoints, + tls_cert_mount_path, + listener_port, + None, + ) +} + +/// Generate the consumer config with optional process-level telemetry settings. +/// +/// The telemetry block is emitted at the Praxis config root, beside listeners +/// and admin settings. It is never copied into the routing overlay. +/// +/// # Errors +/// +/// Returns [`ConsumerConfigError`] when the overlay, endpoint topology, or +/// optional telemetry settings are invalid. #[expect( clippy::too_many_lines, reason = "sequential validation + three rendering passes; splitting would obscure the overall config shape" )] -pub(crate) fn generate_consumer_praxis_config( +#[expect( + clippy::too_many_arguments, + reason = "preserves the renderer's established inputs and adds optional telemetry" +)] +pub(crate) fn generate_consumer_praxis_config_with_telemetry( overlay: &RoutingOverlay, credential_mount_base: &str, cluster_endpoints: &[ClusterEndpointConfig], tls_cert_mount_path: &str, listener_port: u16, + telemetry: Option<&GatewayTelemetryConfig>, ) -> Result { if overlay.local_site.trim().is_empty() { return Err(ConsumerConfigError::BlankLocalSite); @@ -148,10 +184,18 @@ pub(crate) fn generate_consumer_praxis_config( }); } } + if let Some(telemetry) = telemetry { + telemetry.validate().map_err(ConsumerConfigError::InvalidTelemetry)?; + } let candidates_yaml = render_candidates(&overlay.candidates); let selection_policy_yaml = render_selection_policy(overlay.selection_policy.as_ref()); let local_site = yaml_scalar(&overlay.local_site)?; + let trace_context_filter = if telemetry.is_some() { + " - filter: trace_context\n" + } else { + "" + }; let credential_inject_section = render_credential_inject(&overlay.candidates, credential_mount_base); let load_balancer_section = render_load_balancer(&overlay.candidates, cluster_endpoints, tls_cert_mount_path)?; @@ -166,6 +210,7 @@ pub(crate) fn generate_consumer_praxis_config( filter_chains:\n\ \x20 - name: consumer-chain\n\ \x20 filters:\n\ + {trace_context_filter}\ \x20 - filter: json_body_field\n\ \x20 field: model\n\ \x20 header: X-Model\n\ @@ -183,12 +228,73 @@ pub(crate) fn generate_consumer_praxis_config( config.push_str(&load_balancer_section); + if let Some(telemetry) = telemetry { + config.push_str(&render_telemetry(telemetry)?); + } + // Admin interface and graceful shutdown — standard constants for consumer gateways. config.push_str("\nadmin:\n address: \"127.0.0.1:9901\"\nshutdown_timeout_secs: 5\n"); Ok(config) } +/// Render validated exporter settings without secret material. +#[expect( + clippy::too_many_lines, + reason = "serializes every optional telemetry value in a stable YAML order" +)] +fn render_telemetry(telemetry: &GatewayTelemetryConfig) -> Result { + if telemetry.otlp_endpoint.as_deref().is_none_or(str::is_empty) + && telemetry.sampling_rate.is_none() + && telemetry.service_name.is_none() + && telemetry.service_version.is_none() + && telemetry.environment.is_none() + && telemetry.batch_interval_secs.is_none() + && telemetry.batch_size.is_none() + { + return Ok("\ntelemetry: {}\n".to_owned()); + } + let mut output = String::from("\ntelemetry:\n"); + if let Some(endpoint) = telemetry + .otlp_endpoint + .as_deref() + .filter(|endpoint| !endpoint.is_empty()) + { + output.push_str(" otlp_endpoint: "); + output.push_str(&yaml_scalar(endpoint)?); + output.push('\n'); + } + if let Some(rate) = telemetry.sampling_rate { + output.push_str(" sampling_rate: "); + output.push_str(&rate.to_string()); + output.push('\n'); + } + for (name, value) in [ + ("service_name", telemetry.service_name.as_deref()), + ("service_version", telemetry.service_version.as_deref()), + ("environment", telemetry.environment.as_deref()), + ] { + if let Some(value) = value { + output.push_str(" "); + output.push_str(name); + output.push_str(": "); + output.push_str(&yaml_scalar(value)?); + output.push('\n'); + } + } + if let Some(interval) = telemetry.batch_interval_secs { + output.push_str(" batch_interval_secs: "); + output.push_str(&interval.to_string()); + output.push('\n'); + } + if let Some(size) = telemetry.batch_size { + output.push_str(" batch_size: "); + output.push_str(&size.to_string()); + output.push('\n'); + } + Ok(output) +} + /// Build the Kubernetes `ConfigMap` for the generated consumer Praxis config. /// /// The `ConfigMap` contains a single `praxis.yaml` key with the rendered YAML. @@ -650,6 +756,162 @@ mod tests { // Renderer: basic structure // ----------------------------------------------------------------------- + #[test] + #[expect( + clippy::too_many_lines, + reason = "checks opt-in output, secret exclusion, and overlay separation" + )] + fn telemetry_is_opt_in_and_rendered_outside_routing_filters() { + let overlay = simple_overlay(vec![plain_candidate( + "inference_model", + "model-a", + "site-a", + "gateway-site-a", + true, + )]); + let endpoints = endpoint_coverage(&overlay); + let disabled = generate_consumer_praxis_config(&overlay, MOUNT_BASE, &endpoints, "/etc/praxis/tls", 8080) + .unwrap_or_else(|_| std::process::abort()); + assert!( + !disabled.contains("telemetry:"), + "telemetry must remain absent by default" + ); + assert!( + !disabled.contains("filter: trace_context"), + "propagation must remain opt-in" + ); + + let telemetry = GatewayTelemetryConfig { + otlp_endpoint: Some("http://collector.observability:4317".to_owned()), + sampling_rate: Some(0.25), + service_name: Some("grid-edge-site-a".to_owned()), + service_version: Some("0.1.4".to_owned()), + environment: Some("test".to_owned()), + batch_interval_secs: Some(3), + batch_size: Some(32), + }; + let enabled = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &endpoints, + "/etc/praxis/tls", + 8080, + Some(&telemetry), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + enabled.contains("filter: trace_context"), + "telemetry must enable W3C propagation" + ); + assert!(enabled.contains("telemetry:\n otlp_endpoint: \"http://collector.observability:4317\"")); + assert!(enabled.contains(" sampling_rate: 0.25")); + assert!(enabled.contains(" service_name: \"grid-edge-site-a\"")); + assert!(enabled.contains(" batch_interval_secs: 3")); + assert!( + !enabled.contains("otlp_headers"), + "credentials must not be rendered into the ConfigMap" + ); + assert!( + !enabled.contains("collector-secret-value"), + "secret values must not appear in generated YAML" + ); + assert!( + !serde_json::to_string(&overlay) + .unwrap_or_else(|_| std::process::abort()) + .contains("telemetry"), + "exporter settings must stay out of the routing overlay" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "covers environment fallback, sampling boundaries, and credential rejection" + )] + fn telemetry_rejects_invalid_sampling_rates_and_credentials_in_endpoint() { + let overlay = simple_overlay(Vec::new()); + let env_only = GatewayTelemetryConfig { + otlp_endpoint: None, + sampling_rate: None, + service_name: None, + service_version: None, + environment: None, + batch_interval_secs: None, + batch_size: None, + }; + let env_config = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + Some(&env_only), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + env_config.contains("telemetry: {}"), + "an environment-only exporter must render as an empty mapping, not null" + ); + + let empty_endpoint = GatewayTelemetryConfig { + otlp_endpoint: Some(String::new()), + ..env_only + }; + assert!( + empty_endpoint.validate().is_ok(), + "an empty endpoint must use the deployment environment fallback" + ); + let empty_config = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + Some(&empty_endpoint), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!(empty_config.contains("telemetry: {}")); + assert!(!empty_config.contains("otlp_endpoint:")); + + for rate in [-0.01, 1.01, f64::NAN, f64::INFINITY] { + let telemetry = GatewayTelemetryConfig { + sampling_rate: Some(rate), + ..valid_telemetry() + }; + let result = generate_consumer_praxis_config_with_telemetry( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + Some(&telemetry), + ); + assert!(result.is_err(), "invalid sampling rate {rate:?} must be rejected"); + } + + let telemetry = GatewayTelemetryConfig { + otlp_endpoint: Some("https://user:password@collector:4317".to_owned()), + ..valid_telemetry() + }; + assert!( + telemetry.validate().is_err(), + "collector authentication must use Secret-backed environment references" + ); + } + + /// Return a minimal valid telemetry setting for validation tests. + fn valid_telemetry() -> GatewayTelemetryConfig { + GatewayTelemetryConfig { + otlp_endpoint: Some("http://collector:4317".to_owned()), + sampling_rate: Some(0.5), + service_name: None, + service_version: None, + environment: None, + batch_interval_secs: None, + batch_size: None, + } + } + #[test] fn plain_candidates_produce_intelligent_route_and_load_balancer() { let overlay = simple_overlay(vec![plain_candidate( diff --git a/scripts/verify-helm-chart.sh b/scripts/verify-helm-chart.sh index 4e734d95..54e8c8f9 100755 --- a/scripts/verify-helm-chart.sh +++ b/scripts/verify-helm-chart.sh @@ -538,6 +538,21 @@ try_template "$GW_DIR" "subchart keys (gw)" "${GW_REQ[@]}" --set enabled=true -- try_reject "$GW_DIR" "runAsNonRoot override" "${GW_REQ[@]}" --set podSecurityContext.runAsNonRoot=false try_reject "$GW_DIR" "overlay enabled no name" "${GW_REQ[@]}" --set overlay.enabled=true try_reject "$GW_DIR" "tls enabled no secret" "${GW_REQ[@]}" --set tls.enabled=true --set tls.existingSecret="" +try_reject_msg "$GW_DIR" "telemetry sampling rate above one" \ + "gatewayConfig[./]telemetry[./]samplingRate.*(less than or equal to 1|maximum: got 1\\.01)" \ + --set gatewayConfig.telemetry.enabled=true --set-json gatewayConfig.telemetry.samplingRate=1.01 +try_reject_msg "$GW_DIR" "telemetry sampling rate below zero" \ + "gatewayConfig[./]telemetry[./]samplingRate.*(greater than or equal to 0|minimum: got -0\\.01)" \ + --set gatewayConfig.telemetry.enabled=true --set-json gatewayConfig.telemetry.samplingRate=-0.01 +try_reject_msg "$GW_DIR" "telemetry endpoint userinfo credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://user:password@collector:4317' +try_reject_msg "$GW_DIR" "telemetry endpoint query credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://collector:4317?api_key=sentinel' +try_reject_msg "$GW_DIR" "telemetry endpoint fragment credentials" \ + "gatewayConfig[./]telemetry[./]otlpEndpoint.*([Dd]oes not match pattern|does not match the regex)" \ + "${GW_REQ[@]}" --set-string 'gatewayConfig.telemetry.otlpEndpoint=https://collector:4317/otlp#token=sentinel' # ── Secure gateway config (render) ────────────────────────────────── GW_RENDER=(--set gatewayConfig.render=true --set gatewayConfig.localSite=hub --set gatewayConfig.model=q --set gatewayConfig.auth.mode=none